From 32d7afc08e46b0d008c700f534ffca02dd326820 Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Wed, 24 Jun 2026 12:54:50 -0700 Subject: [PATCH 001/385] =?UTF-8?q?[claude/vc-workflow]=20feat(assistive):?= =?UTF-8?q?=20Collapsible=20Tool=20Evidence=20=E2=80=94=20friendly=20tool?= =?UTF-8?q?=20names,=20compact=20one-line=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop rendering raw MCP tool calls as paired "Tool call started/finished: mcp__brave-search__brave_web_search" chat cards that swamp the conversation. apply_agent_ui_event now maps the raw wire name to a human-readable label (friendly_tool_name), shows transient activity in the status pill only ("Searching web…"), and emits a single compact evidence line on completion ("Web search completed · 10 results"). Raw name + full summary go to the debug log — raw tool output is debug, not chat. Pure mapping/format helpers are unit-tested (friendly names, status copy, evidence line). Authored-By: claude session_id: 14fa846e-bf98-4d35-b96c-ed95fd7c5968 date: 2026-06-24T12:54:50 PDT runtime: headless Co-Authored-By: Claude Opus 4.8 (1M context) --- app/controller/helpers.rs | 152 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 145 insertions(+), 7 deletions(-) diff --git a/app/controller/helpers.rs b/app/controller/helpers.rs index c6248ff4..e92102fe 100644 --- a/app/controller/helpers.rs +++ b/app/controller/helpers.rs @@ -429,6 +429,87 @@ async fn stream_final_text_to_chat_locally(text: &str) { } } +/// Title-case a `snake_case` / `kebab-case` identifier into readable words. +/// `brave_web_search` -> `Brave Web Search`. +fn prettify_identifier(s: &str) -> String { + let cleaned = s.replace(['_', '-'], " "); + let mut out = String::with_capacity(cleaned.len()); + for (i, word) in cleaned.split_whitespace().enumerate() { + if i > 0 { + out.push(' '); + } + let mut chars = word.chars(); + if let Some(first) = chars.next() { + out.extend(first.to_uppercase()); + out.push_str(chars.as_str()); + } + } + if out.is_empty() { s.to_string() } else { out } +} + +/// Map a raw tool identifier (often `mcp____`) to a concise, +/// human-readable label for the conversation timeline. +/// +/// Collapsible Tool Evidence: raw MCP wire names like +/// `mcp__brave-search__brave_web_search` are transport noise in a conversation — +/// the user wants to read "Web search", not the addressing scheme. This is a pure +/// function so the mapping is unit-testable without a running UI. +pub(crate) fn friendly_tool_name(raw: &str) -> String { + match raw { + "mcp__brave-search__brave_web_search" | "brave_web_search" => return "Web search".into(), + "mcp__brave-search__brave_local_search" | "brave_local_search" => { + return "Local search".into(); + } + "mcp__brave-search__brave_news_search" | "brave_news_search" => { + return "News search".into(); + } + "mcp__brave-search__brave_image_search" | "brave_image_search" => { + return "Image search".into(); + } + "mcp__brave-search__brave_video_search" | "brave_video_search" => { + return "Video search".into(); + } + "mcp__brave-search__brave_summarizer" | "brave_summarizer" => return "Summarize".into(), + _ => {} + } + if let Some(rest) = raw.strip_prefix("mcp__") { + let mut parts = rest.splitn(2, "__"); + let server = parts.next().unwrap_or(""); + let tool = parts.next().unwrap_or(server); + let tool_pretty = prettify_identifier(tool); + if server.is_empty() || tool == server { + return tool_pretty; + } + return format!("{tool_pretty} · {}", prettify_identifier(server)); + } + prettify_identifier(raw) +} + +/// Status-pill copy while a tool is running. Conversation timeline stays for +/// conversation; transient activity lives in the status pill. +pub(crate) fn tool_running_status(friendly: &str) -> String { + let lower = friendly.to_lowercase(); + if lower.contains("web search") { + "Searching web…".to_string() + } else if lower.contains("search") { + "Searching…".to_string() + } else { + format!("Running {friendly}…") + } +} + +/// Compact one-line evidence entry for a completed tool call. Replaces the old +/// pair of raw "Tool call started/finished: mcp__…" cards with a single readable +/// line. The full raw name + summary still goes to the debug log. +pub(crate) fn tool_evidence_line(friendly: &str, summary: &str) -> String { + let summary = summary.trim(); + if summary.is_empty() { + format!("{friendly} completed") + } else { + format!("{friendly} completed · {summary}") + } +} + async fn apply_agent_ui_event(event: AgentUiEvent, overlay_state: &mut AgentUiOverlayState) { match event { AgentUiEvent::TextDelta(delta) => { @@ -461,15 +542,22 @@ async fn apply_agent_ui_event(event: AgentUiEvent, overlay_state: &mut AgentUiOv crate::ui::voice_chat::append_voice_chat_reasoning_delta(&delta); } AgentUiEvent::ToolExecuting { name, .. } => { - crate::ui::voice_chat::update_voice_chat_status(&format!("Tool running: {name}")); - crate::ui::voice_chat::add_voice_chat_system_message(&format!( - "Tool call started: {name}" - )); + // Collapsible Tool Evidence: no per-start chat card. The status pill + // communicates activity; the timeline stays for conversation. The raw + // wire name is debug-only. + let friendly = friendly_tool_name(&name); + debug!("Tool executing: {name} -> {friendly}"); + crate::ui::voice_chat::update_voice_chat_status(&tool_running_status(&friendly)); } AgentUiEvent::ToolResult { name, summary, .. } => { - crate::ui::voice_chat::update_voice_chat_status("Thinking... (70%)"); - crate::ui::voice_chat::add_voice_chat_system_message(&format!( - "Tool call finished: {name} ({summary})" + // One compact, human-readable evidence line per completed call instead + // of the raw "Tool call finished: mcp__…" card. Raw name + full summary + // are kept in the debug log (raw tool output is debug, not chat). + let friendly = friendly_tool_name(&name); + debug!("Tool result: {name} -> {friendly} | raw summary: {summary}"); + crate::ui::voice_chat::update_voice_chat_status("Thinking…"); + crate::ui::voice_chat::add_voice_chat_system_message(&tool_evidence_line( + &friendly, &summary, )); } AgentUiEvent::Done => {} @@ -1094,6 +1182,56 @@ mod tests { assert_eq!(chunks.concat(), "Zażółć gęślą jaźń"); } + // ── Collapsible Tool Evidence: friendly tool-name mapping ─────────────── + + #[test] + fn friendly_tool_name_maps_known_brave_tools() { + assert_eq!( + friendly_tool_name("mcp__brave-search__brave_web_search"), + "Web search" + ); + assert_eq!(friendly_tool_name("brave_web_search"), "Web search"); + assert_eq!( + friendly_tool_name("mcp__brave-search__brave_news_search"), + "News search" + ); + } + + #[test] + fn friendly_tool_name_prettifies_unknown_mcp_tools() { + // Unknown mcp__server__tool falls back to " · " — never the + // raw wire name in the conversation timeline. + assert_eq!( + friendly_tool_name("mcp__github__create_issue"), + "Create Issue · Github" + ); + // Bare snake_case identifier is title-cased. + assert_eq!(friendly_tool_name("read_file"), "Read File"); + // The raw mcp__ wire form must never survive verbatim. + assert!(!friendly_tool_name("mcp__github__create_issue").contains("mcp__")); + } + + #[test] + fn tool_running_status_is_human_readable() { + assert_eq!(tool_running_status("Web search"), "Searching web…"); + assert_eq!(tool_running_status("Local search"), "Searching…"); + assert_eq!( + tool_running_status("Create Issue · Github"), + "Running Create Issue · Github…" + ); + } + + #[test] + fn tool_evidence_line_is_compact() { + assert_eq!( + tool_evidence_line("Web search", "10 results"), + "Web search completed · 10 results" + ); + assert_eq!(tool_evidence_line("Summarize", ""), "Summarize completed"); + // No raw transport noise leaks into the evidence line. + assert!(!tool_evidence_line("Web search", "10 results").contains("mcp__")); + } + struct NoopTestProvider; #[async_trait] From b1413b1cac66b289c997d300203f482d99fc2bd5 Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Wed, 24 Jun 2026 12:55:09 -0700 Subject: [PATCH 002/385] =?UTF-8?q?[claude/vc-workflow]=20feat(theme):=20L?= =?UTF-8?q?ight=20Mode=20Semantic=20Contrast=20=E2=80=94=20deterministic?= =?UTF-8?q?=20token=20foundation=20+=20WCAG=20proof?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a contrast token module with the operator's canonical Light/Dark palettes. The existing ui_colors surfaces are vibrancy-derived (controlBackgroundColor over a HUDWindow material that reads dark regardless of appearance) — in Light Mode the conversation body goes dark while labelColor stays near-black, the reported low-contrast bug. These tokens are deterministic and appearance-resolved (is_dark_appearance reads NSApplication.effectiveAppearance): blur/vibrancy never decides text contrast. Tokens are plain hex data, so the acceptance contract is proven by pure WCAG unit tests (primary/secondary/input text >= 4.5:1, muted/placeholder/toolbar icons >= 3:1, light-is-light/dark-is-dark) with no running UI. AppKit view wiring is intentionally NOT done here — it needs visual GUI verification and is specced as a follow-up in the workflow report. Authored-By: claude session_id: 14fa846e-bf98-4d35-b96c-ed95fd7c5968 date: 2026-06-24T12:55:09 PDT runtime: headless Co-Authored-By: Claude Opus 4.8 (1M context) --- app/ui/shared/helpers/mod.rs | 327 +++++++++++++++++++++++++++++++++++ pico.save | 87 ---------- 2 files changed, 327 insertions(+), 87 deletions(-) delete mode 100644 pico.save diff --git a/app/ui/shared/helpers/mod.rs b/app/ui/shared/helpers/mod.rs index b70eee3d..3fc0e89a 100644 --- a/app/ui/shared/helpers/mod.rs +++ b/app/ui/shared/helpers/mod.rs @@ -417,6 +417,333 @@ pub mod ui_colors { } } +// ============================================================================ +// Light Mode Semantic Contrast +// ============================================================================ +// +// The existing `ui_colors` surfaces are vibrancy-derived (controlBackgroundColor +// + alpha over a HUDWindow material). That material reads *dark regardless of the +// system appearance*, so in Light Mode the conversation body is dark while +// `labelColor` is near-black → dark text on a dark surface (the reported bug). +// +// This module provides DETERMINISTIC, appearance-resolved tokens for text-bearing +// regions: blur/vibrancy must never decide text contrast. Tokens are plain data +// (hex), so the WCAG contrast contract is proven by pure unit tests without a +// running UI. The NSColor accessors are thin wrappers that pick the palette from +// the live `NSAppearance`. + +pub mod contrast { + use super::Id; + use super::color_rgba; + use objc::runtime::Class; + use objc::{msg_send, sel, sel_impl}; + + /// One semantic color: 24-bit RGB plus an alpha (for surfaces meant to sit on + /// `window_background`). Text tokens are fully opaque (`alpha == 1.0`). + #[derive(Clone, Copy, Debug)] + pub struct Token { + pub rgb: u32, + pub alpha: f64, + } + + impl Token { + const fn opaque(rgb: u32) -> Self { + Token { rgb, alpha: 1.0 } + } + const fn with_alpha(rgb: u32, alpha: f64) -> Self { + Token { rgb, alpha } + } + /// Split into linear-space-free sRGB components in 0.0..=1.0. + pub fn srgb(self) -> (f64, f64, f64) { + let r = ((self.rgb >> 16) & 0xFF) as f64 / 255.0; + let g = ((self.rgb >> 8) & 0xFF) as f64 / 255.0; + let b = (self.rgb & 0xFF) as f64 / 255.0; + (r, g, b) + } + } + + /// The full deterministic palette for one appearance. + #[derive(Clone, Copy, Debug)] + pub struct Palette { + pub window_background: Token, + pub conversation_surface: Token, + pub message_surface: Token, + pub message_border: Token, + pub primary_text: Token, + pub secondary_text: Token, + pub muted_text: Token, + pub input_background: Token, + pub input_border: Token, + pub input_text: Token, + pub placeholder_text: Token, + pub toolbar_background: Token, + pub toolbar_icon: Token, + pub toolbar_icon_muted: Token, + pub accent_purple: Token, + pub accent_warm: Token, + } + + /// Light Mode: light readable surfaces, dark text. Values are the operator's + /// canonical token table. + pub const LIGHT: Palette = Palette { + window_background: Token::opaque(0xF6F3EE), + conversation_surface: Token::with_alpha(0xFFFFFF, 0.902), // #FFFFFFE6 + message_surface: Token::opaque(0xFFFFFF), + message_border: Token::opaque(0xD8D2C8), + primary_text: Token::opaque(0x1C1B18), + secondary_text: Token::opaque(0x5F5A52), + muted_text: Token::opaque(0x7A746C), + input_background: Token::with_alpha(0xFFFFFF, 0.949), // #FFFFFFF2 + input_border: Token::opaque(0xD0CBC2), + input_text: Token::opaque(0x1C1B18), + placeholder_text: Token::opaque(0x756F67), + toolbar_background: Token::with_alpha(0xF3EFE8, 0.902), // #F3EFE8E6 + toolbar_icon: Token::opaque(0x38342F), + toolbar_icon_muted: Token::opaque(0x6E675F), + accent_purple: Token::opaque(0x9B4DAD), + accent_warm: Token::opaque(0xC96F4A), + }; + + /// Dark Mode: dark readable surfaces, light text. + pub const DARK: Palette = Palette { + window_background: Token::opaque(0x191919), + conversation_surface: Token::with_alpha(0x202020, 0.949), // #202020F2 + message_surface: Token::opaque(0x252525), + message_border: Token::opaque(0x3A3A3A), + primary_text: Token::opaque(0xF3F0EA), + secondary_text: Token::opaque(0xC8C1B8), + muted_text: Token::opaque(0x918A82), + input_background: Token::opaque(0x2B2B2B), + input_border: Token::opaque(0x444444), + input_text: Token::opaque(0xF3F0EA), + placeholder_text: Token::opaque(0xAAA39B), + toolbar_background: Token::with_alpha(0x202020, 0.902), // #202020E6 + toolbar_icon: Token::opaque(0xD8D3CC), + toolbar_icon_muted: Token::opaque(0x8E8880), + accent_purple: Token::opaque(0xD26BE8), + accent_warm: Token::opaque(0xE08A64), + }; + + /// Is the application currently rendering in a Dark appearance? + /// + /// Reads `NSApplication.effectiveAppearance.name` and checks for "Dark" + /// (covers `NSAppearanceNameDarkAqua` and the vibrant-dark variants). Falls + /// back to Light when AppKit is unavailable (e.g. headless test process). + pub fn is_dark_appearance() -> bool { + unsafe { + let Some(app_cls) = Class::get("NSApplication") else { + return false; + }; + let app: Id = msg_send![app_cls, sharedApplication]; + if app.is_null() { + return false; + } + let appearance: Id = msg_send![app, effectiveAppearance]; + if appearance.is_null() { + return false; + } + let name: Id = msg_send![appearance, name]; + if name.is_null() { + return false; + } + let utf8: *const std::os::raw::c_char = msg_send![name, UTF8String]; + if utf8.is_null() { + return false; + } + std::ffi::CStr::from_ptr(utf8) + .to_string_lossy() + .contains("Dark") + } + } + + /// The palette resolved against the live system appearance. + pub fn palette() -> Palette { + if is_dark_appearance() { DARK } else { LIGHT } + } + + fn ns_color(token: Token) -> Id { + let (r, g, b) = token.srgb(); + color_rgba(r, g, b, token.alpha) + } + + // Deterministic NSColor accessors for text-bearing regions. These do NOT + // depend on vibrancy — they are exact, appearance-resolved colors. + pub fn primary_text() -> Id { + ns_color(palette().primary_text) + } + pub fn secondary_text() -> Id { + ns_color(palette().secondary_text) + } + pub fn muted_text() -> Id { + ns_color(palette().muted_text) + } + pub fn conversation_surface() -> Id { + ns_color(palette().conversation_surface) + } + pub fn message_surface() -> Id { + ns_color(palette().message_surface) + } + pub fn message_border() -> Id { + ns_color(palette().message_border) + } + pub fn input_background() -> Id { + ns_color(palette().input_background) + } + pub fn input_border() -> Id { + ns_color(palette().input_border) + } + pub fn input_text() -> Id { + ns_color(palette().input_text) + } + pub fn placeholder_text() -> Id { + ns_color(palette().placeholder_text) + } + pub fn toolbar_icon() -> Id { + ns_color(palette().toolbar_icon) + } + pub fn toolbar_icon_muted() -> Id { + ns_color(palette().toolbar_icon_muted) + } + + // ── WCAG contrast math (pure; the proof that the tokens are readable) ────── + + fn linearize(c: f64) -> f64 { + if c <= 0.03928 { + c / 12.92 + } else { + ((c + 0.055) / 1.055).powf(2.4) + } + } + + fn relative_luminance((r, g, b): (f64, f64, f64)) -> f64 { + 0.2126 * linearize(r) + 0.7152 * linearize(g) + 0.0722 * linearize(b) + } + + /// Composite a (possibly translucent) token over an opaque background. + pub fn composite_over(fg: Token, bg: Token) -> (f64, f64, f64) { + let (fr, fg_, fb) = fg.srgb(); + let (br, bg_, bb) = bg.srgb(); + let a = fg.alpha; + ( + fr * a + br * (1.0 - a), + fg_ * a + bg_ * (1.0 - a), + fb * a + bb * (1.0 - a), + ) + } + + /// WCAG 2.x contrast ratio between two opaque sRGB colors (1.0..=21.0). + pub fn contrast_ratio(fg: (f64, f64, f64), bg: (f64, f64, f64)) -> f64 { + let l1 = relative_luminance(fg); + let l2 = relative_luminance(bg); + let (hi, lo) = if l1 >= l2 { (l1, l2) } else { (l2, l1) }; + (hi + 0.05) / (lo + 0.05) + } + + /// Contrast of an opaque text token placed on a surface token that is itself + /// composited over `window_background`. + pub fn text_on_surface(text: Token, surface: Token, window: Token) -> f64 { + let surface_rgb = composite_over(surface, window); + contrast_ratio(text.srgb(), surface_rgb) + } + + #[cfg(test)] + mod tests { + use super::*; + + // Acceptance contract (operator): + // primary / secondary / input text >= 4.5:1 + // muted / placeholder / toolbar icon >= 3:1 + fn assert_palette_contrast(p: Palette, label: &str) { + let win = p.window_background; + + let primary = text_on_surface(p.primary_text, p.conversation_surface, win); + assert!( + primary >= 4.5, + "{label}: primary text on conversation surface = {primary:.2}:1 (< 4.5)" + ); + + let primary_msg = text_on_surface(p.primary_text, p.message_surface, win); + assert!( + primary_msg >= 4.5, + "{label}: primary text on message surface = {primary_msg:.2}:1 (< 4.5)" + ); + + let secondary = text_on_surface(p.secondary_text, p.message_surface, win); + assert!( + secondary >= 4.5, + "{label}: secondary text on message surface = {secondary:.2}:1 (< 4.5)" + ); + + let input = text_on_surface(p.input_text, p.input_background, win); + assert!( + input >= 4.5, + "{label}: input text on input background = {input:.2}:1 (< 4.5)" + ); + + let muted = text_on_surface(p.muted_text, p.message_surface, win); + assert!( + muted >= 3.0, + "{label}: muted text on message surface = {muted:.2}:1 (< 3.0)" + ); + + let placeholder = text_on_surface(p.placeholder_text, p.input_background, win); + assert!( + placeholder >= 3.0, + "{label}: placeholder on input background = {placeholder:.2}:1 (< 3.0)" + ); + + let icon = text_on_surface(p.toolbar_icon, p.toolbar_background, win); + assert!( + icon >= 3.0, + "{label}: toolbar icon on toolbar background = {icon:.2}:1 (< 3.0)" + ); + + let icon_muted = text_on_surface(p.toolbar_icon_muted, p.toolbar_background, win); + assert!( + icon_muted >= 3.0, + "{label}: muted toolbar icon on toolbar background = {icon_muted:.2}:1 (< 3.0)" + ); + } + + #[test] + fn light_palette_meets_wcag_contract() { + assert_palette_contrast(LIGHT, "Light"); + } + + #[test] + fn dark_palette_meets_wcag_contract() { + assert_palette_contrast(DARK, "Dark"); + } + + #[test] + fn light_is_light_dark_is_dark() { + // The whole point: Light Mode uses a light surface + dark text; + // Dark Mode the inverse. Guard against accidentally swapping tables. + let light_surface = relative_luminance(LIGHT.message_surface.srgb()); + let light_text = relative_luminance(LIGHT.primary_text.srgb()); + assert!( + light_surface > light_text, + "Light Mode surface must be lighter than its text" + ); + + let dark_surface = relative_luminance(DARK.message_surface.srgb()); + let dark_text = relative_luminance(DARK.primary_text.srgb()); + assert!( + dark_text > dark_surface, + "Dark Mode text must be lighter than its surface" + ); + } + + #[test] + fn contrast_ratio_extremes_are_correct() { + let white = (1.0, 1.0, 1.0); + let black = (0.0, 0.0, 0.0); + assert!((contrast_ratio(white, black) - 21.0).abs() < 0.01); + assert!((contrast_ratio(white, white) - 1.0).abs() < 0.0001); + } + } +} + /// Apply Tafla surface treatment to a CALayer: corner radius + optional border. /// Shadows off by default — Tafla is flat pane, not bubble soup. /// diff --git a/pico.save b/pico.save deleted file mode 100644 index dda07e08..00000000 --- a/pico.save +++ /dev/null @@ -1,87 +0,0 @@ -Jesteś agentem „Codex” w trybie **read mode + plan mode**. Funkcjonalności, o których mowa, **już są wpisane w aplikację**, ale nie wszystkie są poprawnie „spięte” (integracja/przepływy). Masz wyjaśnić **dlaczego pewnych rzeczy nie spinamy** (albo czemu nie warto ich spinać „po staremu”) oraz podać **plan dalszych kroków**, żeby dopiąć minimalny zestaw funkcji znany z poprzedniej wersji — **bez tworzenia duplikatów i bez legacy structów**. - -## 1) Opis zadania (z transkryptu) -- Zidentyfikuj minimalne funkcjonalności, które były w poprzedniej wersji i powinny działać teraz. -- Sprawdź, które z nich są już obecne w kodzie, ale nie są połączone (UI ↔ backend ↔ kernel/serwisy). -- Wyjaśnij, dlaczego **nie spinamy** niektórych rzeczy (np. bo są duplikatem, bo istnieje nowy moduł robiący to samo, bo stary przepływ był błędny, bo powoduje legacy). -- Przygotuj plan działań (kolejność kroków/PR) żeby dopiąć MVP funkcji **bez duplikacji** i bez utrwalania starych struktur danych. - -## 2) Kontekst wstępny -- Projekt jest w trakcie iteracyjnego porządkowania: są stare ścieżki/duplikacje. -- Priorytet: „single source of truth”, brak dublowania logiki, brak przepisywania pod legacy. -- Celem jest spięcie minimalnych funkcji, a nie rekonstrukcja poprzedniej architektury. - ---- - -## 3) Actionable TODO (checklista) - -### Zbadaj kod -- [ ] Wypisz listę „minimalnych funkcjonalności” (MVP) na podstawie poprzedniej wersji: ekrany/komendy/flow. -- [ ] Dla każdej funkcji sprawdź w aktualnym kodzie: - - gdzie jest logika (moduł/serwis), - - gdzie jest wywołanie (UI/command/router), - - gdzie jest brakujące ogniwo (np. brak rejestracji command, brak eventu, brak wstrzyknięcia zależności). -- [ ] Zidentyfikuj obecne moduły, które już realizują te funkcje i mogą być „źródłem prawdy”. -- [ ] Znajdź legacy structy / stare DTO / stare eventy, które kuszą, żeby ich użyć ponownie. -- [ ] Zmapuj duplikacje: jeśli istnieją dwie ścieżki do tego samego celu, opisz je i wskaż, która powinna zostać. - -### Przeprowadź implementację (TYLKO PLAN – bez kodu w tym zadaniu) -- [ ] Dla każdej funkcji MVP zaproponuj **jedną** docelową ścieżkę integracji (konkret: jakie moduły, jakie API/commands, jakie typy danych). -- [ ] Ustal zasady „nie spinamy X, bo…”: - - bo to dubluje nowy moduł, - - bo prowadzi do legacy structów, - - bo nowy kontrakt jest inny i trzeba adapter, nie rewire. -- [ ] Zaproponuj plan kolejnych PR-ów: - 1) doprecyzowanie kontraktów (typy/DTO) i usunięcie nieużywanych struktur, - 2) spięcie 1. funkcji end-to-end, - 3) spięcie kolejnych funkcji, każda osobnym PR, - 4) usunięcie duplikatów i dead code po każdym etapie. -- [ ] Zaproponuj minimalny mechanizm kompatybilności, jeśli trzeba (adaptery), ale bez utrwalania legacy. - -### Sprawdź integralność (format, lint) -- [ ] Określ komendy format/lint (wg stacku) i dodaj jako „definition of done” dla PR-ów. -- [ ] W planie uwzględnij czyszczenie importów, nieużywanego kodu, dead feature flags. - -### Testy (jeśli brak to obowiązkowo napisz) -- [ ] Dla każdej funkcji MVP zaproponuj co najmniej jeden test integracyjny/e2e potwierdzający spięcie end-to-end. -- [ ] Dodaj testy kontraktowe dla granic modułów (żeby nie wróciła duplikacja). -- [ ] Zaproponuj testy regresji tam, gdzie usuwamy starą ścieżkę. - ---- - -## 4) Expected deliverables -1) **Raport „dlaczego nie spinamy X”**: lista decyzji z uzasadnieniem (duplikacja/legacy/błędny przepływ). -2) **Mapa funkcji MVP**: tabela (Funkcja → gdzie jest logika → gdzie brakuje integracji → docelowa ścieżka). -3) **Plan PR-ów**: kolejność, kryteria „done”, jakie pliki/obszary dotykamy. -4) **Plan testów**: co testujemy i jak uruchamiamy. - ---- - -## 5) Report expectation -- Konkretne ścieżki w repo i nazwy modułów. -- Jasne wskazanie „single source of truth”. -- Zero przenoszenia starych struktur danych, jeśli są zbędne — zamiast tego adapter lub nowy kontrakt. - ---- - -## Appendix: tooling + zasady anty–technical debt -**Tooling:** -- `rg` do znajdowania punktów integracji -- `loctree` (jeśli dostępne) do mapy katalogów -- Rust: `cargo fmt`, `cargo clippy`, `cargo test` -- TS: `eslint`, `prettier`, `pnpm test` (jeśli dotyczy) - -**Zasady anty–technical debt:** -- Nie dokładamy nowej warstwy „na stare” tylko dlatego, że kiedyś działało. -- Jedna funkcja = jedno źródło prawdy; reszta to wywołania/adaptory. -- Po spięciu funkcji usuń starą ścieżkę i martwe struktury. -- Brak „tymczasowych” duplikatów bez planu usunięcia. - ---- - -**Call to action:** -Przeanalizuj aktualny kod, wskaż które funkcje MVP są już obecne i gdzie brakuje spięcia, wyjaśnij które rewire są zbędne i przygotuj plan dopięcia MVP bez duplikatów i legacy structów. - -======================= -**„Duplikat w kodzie to jak drugi pacjent z tą samą kartą — niby szybciej, a potem nikt nie wie, który wynik jest prawdziwy (งಠ_ಠ)ง”** -======================= From 030e075fc054d53370277498501781824d5037b9 Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Wed, 24 Jun 2026 14:18:29 -0700 Subject: [PATCH 003/385] [claude/vc-workflow] fix(lexicon): deterministic protected-term preservation across STT + LLM pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Custom-dictionary preservation regressed: proper nouns were corrupted in dictation (Loctree -> Luxury, malformed product/tool names). Root cause was twofold — (1) the protected vocabulary had no acoustic-homophone coverage and most operator/tool/agent names were absent from the builtin lexicon, and (2) the AI formatting/assistive pass consumed lexicon'd text but its OUTPUT was never re-run through the lexicon, so the LLM could silently rewrite proper nouns with no deterministic guard after it. Repair (smallest safe, central — repairs the existing mechanism, no parallel system): - assets/protected_terms.jsonl: curated proper-noun source (CodeScribe, VetCoders, Vibecrafted, AICX, MCP, GitHub, Loctree, Fn Shift, Living Intent Queue, Assistive Talk Anytime, Collapsible Tool Evidence, Light Mode Semantic Contrast, Claude, Codex, ...). Loaded via a case-normalizing loader so "aicx" -> "AICX" works; hand-vetted so ordinary words (rust/rest/diesel) are never capitalized. - assets/programming.jsonl: loctree entry canonicalized to brand "Loctree" and given acoustic homophones (luxury, luxery, lux tree) for the reported bug. - stream_postprocess.rs: store protected canonicals; expose pub apply_lexicon() (deterministic, idempotent) and protected_terms_lost() for downstream guards. - ai_formatting.rs: re-apply the lexicon to accepted LLM output (formatting + assistive) so the AI pass can no longer silently corrupt operator vocabulary. - qube_report.rs: quality-loop check flags protected terms present pre-AI but lost post-AI, surfacing technical-name corruption to the operator. - Regression tests: Loctree-not-Luxury, brand casing (CodeScribe/AICX/MCP/ GitHub), multiword phrases (Fn Shift/Living Intent Queue/Assistive Talk Anytime), no-overcorrection of ordinary language, protected_terms_lost, and apply_lexicon idempotency. Gates: cargo fmt --check, cargo check --workspace, cargo clippy -p codescribe-core -- -D warnings, cargo test stream_postprocess (22) + ai_formatting (9) — all green. Authored-By: claude session_id: 24879cce-e545-4219-8e05-5958e372e7d7 date: 2026-06-24T14:18:07 PDT runtime: headless --- assets/programming.jsonl | 2 +- assets/protected_terms.jsonl | 21 +++ core/llm/ai_formatting.rs | 8 +- core/pipeline/stream_postprocess.rs | 248 +++++++++++++++++++++++++++- core/quality/qube_report.rs | 14 ++ 5 files changed, 289 insertions(+), 4 deletions(-) create mode 100644 assets/protected_terms.jsonl diff --git a/assets/programming.jsonl b/assets/programming.jsonl index b6a0cf9a..bd0043a4 100644 --- a/assets/programming.jsonl +++ b/assets/programming.jsonl @@ -73,7 +73,7 @@ {"term": "idempotent", "mispronunciations": ["ajdempotent", "idem potent", "idem potent"], "category": "net"} {"term": "jq", "mispronunciations": ["dżej ku", "jot ku", "dżej kiu"], "category": "cmd"} {"term": "latency", "mispronunciations": ["lejten si", "lejten sji", "lej ten si"], "category": "net"} -{"term": "loctree", "mispronunciations": ["log tree", "log three", "log 3", "log3", "loc tree", "lock tree", "locktree", "lock three", "logTree", "logtri", "locktri", "loktree", "loktri", "eloksy", "e loksy", "eloksji", "lock3", "LOCK3", "lok 3"], "category": "tools"} +{"term": "Loctree", "mispronunciations": ["loctree", "log tree", "log three", "log 3", "log3", "loc tree", "lock tree", "locktree", "lock three", "logTree", "logtri", "locktri", "loktree", "loktri", "eloksy", "e loksy", "eloksji", "lock3", "LOCK3", "lok 3", "luxury", "luxery", "luxujri", "lux tree", "luxe tree", "lakszeri", "lukstri"], "category": "tools"} {"term": "merge", "mispronunciations": ["merdż", "merdżu"], "category": "git"} {"term": "mlx_embeddings", "mispronunciations": ["em el eks embeddings", "mlx embeddings", "emeliks embeddings"], "category": "ai"} {"term": "multipart/form-data", "mispronunciations": ["multi part form data", "multi part formdejta", "multipart form data"], "category": "net"} diff --git a/assets/protected_terms.jsonl b/assets/protected_terms.jsonl new file mode 100644 index 00000000..5d5c7bf8 --- /dev/null +++ b/assets/protected_terms.jsonl @@ -0,0 +1,21 @@ +{"term": "Loctree", "mispronunciations": ["loctree", "loktree", "luxury", "luxery", "lux tree"], "category": "protected"} +{"term": "CodeScribe", "mispronunciations": ["code scribe", "codescribe", "Codescribe", "codeScribe", "kod scribe", "kod skrajb", "kodeskrajb", "kod-scribe"], "category": "protected"} +{"term": "VetCoders", "mispronunciations": ["vet coders", "vetcoders", "Vetcoders", "vet koders", "wetkoders", "vetkoderzy"], "category": "protected"} +{"term": "Vibecrafted", "mispronunciations": ["vibe crafted", "vibecrafted", "vibekrafted", "wibe krafted", "vajbkrafted", "wajbkrafted"], "category": "protected"} +{"term": "AICX", "mispronunciations": ["a i c x", "aicx", "Aicx", "ai cx", "ai-cx", "a-i-c-x", "ajsiks", "ej aj si eks"], "category": "protected"} +{"term": "MCP", "mispronunciations": ["m c p", "mcp", "Mcp", "m-c-p", "em see pee", "em si pi"], "category": "protected"} +{"term": "GitHub", "mispronunciations": ["git hub", "github", "Github", "git-hub", "githab", "git hab"], "category": "protected"} +{"term": "Brave Search", "mispronunciations": ["brave search", "brave serch", "brejw search", "brave-search"], "category": "protected"} +{"term": "Responses API", "mispronunciations": ["responses api", "response api", "responses a p i", "responses ej pi aj"], "category": "protected"} +{"term": "Fn Shift", "mispronunciations": ["fn shift", "fun shift", "fan shift", "effen shift", "ef en shift", "f n shift"], "category": "protected"} +{"term": "Living Intent Queue", "mispronunciations": ["living intent queue", "living intent kju", "livingintent queue", "living intent kju"], "category": "protected"} +{"term": "Assistive Talk Anytime", "mispronunciations": ["assistive talk anytime", "asistive talk anytime", "assistive talk any time", "assistiv talk anytime"], "category": "protected"} +{"term": "Collapsible Tool Evidence", "mispronunciations": ["collapsible tool evidence", "collapsable tool evidence", "collapsible tool evidens"], "category": "protected"} +{"term": "Light Mode Semantic Contrast", "mispronunciations": ["light mode semantic contrast", "light-mode semantic contrast", "light mode semantik contrast"], "category": "protected"} +{"term": "Emil", "mispronunciations": ["emil", "emyl", "emiel"], "category": "protected"} +{"term": "Emilio", "mispronunciations": ["emilio", "emiljo", "emilijo"], "category": "protected"} +{"term": "Claude", "mispronunciations": ["claude", "klod", "klałd", "kload", "klat", "klołd"], "category": "protected"} +{"term": "Codex", "mispronunciations": ["kodeks", "kodex", "codeks", "codex'a", "codeksu"], "category": "protected"} +{"term": "Grok", "mispronunciations": ["grock", "grok ej aj", "grok ai"], "category": "protected"} +{"term": "Gemini", "mispronunciations": ["gemini", "Gemini ej aj", "dżemini", "dzemini", "dżeminaj"], "category": "protected"} +{"term": "Whisper", "mispronunciations": ["łisper", "łysper", "uisper", "łispr"], "category": "protected"} diff --git a/core/llm/ai_formatting.rs b/core/llm/ai_formatting.rs index 8d107570..ce2eaebc 100644 --- a/core/llm/ai_formatting.rs +++ b/core/llm/ai_formatting.rs @@ -1020,7 +1020,13 @@ pub async fn format_text_with_status_channels( }; if let Some(output) = result_opt { - let formatted = output.assistant_text; + // Deterministic protected-vocabulary pass AFTER the LLM. The model can + // silently corrupt proper nouns ("Loctree" -> "Luxury") or drop + // operator/tool/agent names while rewriting prose; re-applying the + // lexicon restores any registered mispronunciation to its canonical + // form. Safe + idempotent: it only rewrites known variants, never + // ordinary language. Applies to both formatting and assistive modes. + let formatted = crate::stream_postprocess::apply_lexicon(&output.assistant_text); let reasoning_text = output.reasoning_text; // Detect AI refusal responses (OpenAI content policy) diff --git a/core/pipeline/stream_postprocess.rs b/core/pipeline/stream_postprocess.rs index 0aad358b..837ed748 100644 --- a/core/pipeline/stream_postprocess.rs +++ b/core/pipeline/stream_postprocess.rs @@ -16,6 +16,13 @@ const BUILTIN_LEXICONS: &[(&str, &str)] = &[( include_str!("../../assets/programming.jsonl"), )]; const SEED_JSONL: &str = include_str!("../../assets/seed.jsonl"); +/// Curated proper-noun / operator-vocabulary lexicon. Unlike the generic +/// programming/seed sources, entries here are case-normalizing: a variant that +/// differs from the canonical only by casing (e.g. "aicx" -> "AICX") still +/// produces a rewrite rule. The list is hand-vetted so capitalization is always +/// correct for these terms — generic English words (rust, rest, diesel) are NOT +/// in this file, so they never get capitalized. +const PROTECTED_TERMS_JSONL: &str = include_str!("../../assets/protected_terms.jsonl"); const DEFAULT_SIMILARITY_THRESHOLD: f32 = 0.93; const DEFAULT_NOVELTY_THRESHOLD: f32 = 0.12; @@ -87,6 +94,10 @@ struct Lexicon { custom_rules: Vec, custom_path: PathBuf, custom_mtime: Option, + /// Canonical forms of curated protected terms (proper nouns, operator + /// vocabulary). Used by `protected_terms_lost` to flag when an LLM or other + /// downstream pass silently drops or mutates a protected term. + protected_canonicals: Vec, } static GLOBAL_LEXICON: LazyLock> = LazyLock::new(|| { @@ -114,6 +125,16 @@ impl Lexicon { let seed_count = load_seed_jsonl(SEED_JSONL, "seed", &mut builtin_rules); let seed_ms = t_seed.elapsed().as_millis(); + // Protected terms load LAST among builtin sources so their brand casing + // wins over any generic earlier rule that produced a lower-cased form. + let mut protected_canonicals = Vec::new(); + let protected_count = load_protected_jsonl( + PROTECTED_TERMS_JSONL, + "protected", + &mut builtin_rules, + &mut protected_canonicals, + ); + let custom_path = Config::config_dir().join("lexicon.custom.jsonl"); let custom_mtime = fs::metadata(&custom_path) .ok() @@ -131,13 +152,15 @@ impl Lexicon { if total > 0 { info!( - "Loaded {} lexicon rules in {}ms (legacy={} in {}ms, seed={} in {}ms, custom={} in {}ms, custom_path={})", + "Loaded {} lexicon rules in {}ms (legacy={} in {}ms, seed={} in {}ms, protected={} terms={}, custom={} in {}ms, custom_path={})", total, total_ms, legacy_count, legacy_ms, seed_count, seed_ms, + protected_count, + protected_canonicals.len(), custom_count, custom_ms, custom_path.display(), @@ -154,6 +177,7 @@ impl Lexicon { custom_rules, custom_path, custom_mtime, + protected_canonicals, } } @@ -224,6 +248,51 @@ fn apply_global_lexicon(text: &str) -> String { lexicon.apply(text) } +/// Deterministically apply the global lexicon (builtin + seed + protected + +/// custom) to `text`, hot-reloading the custom file if it changed. +/// +/// This is the single deterministic protected-vocabulary pass. It is safe to run +/// at any layer (it only rewrites registered mispronunciations to their +/// canonical form) and is idempotent for canonical output. Use it to re-assert +/// operator vocabulary AFTER a non-deterministic stage such as an LLM +/// formatting/assistive pass, which can otherwise silently corrupt proper nouns +/// (e.g. "Loctree" -> "Luxury"). +pub fn apply_lexicon(text: &str) -> String { + maybe_reload_global_lexicon(); + apply_global_lexicon(text) +} + +/// Whole-word, case-insensitive containment check for a (possibly multi-word) +/// term. Mirrors the lexicon's own matching: internal whitespace is treated +/// flexibly so "Fn Shift" matches across variable spacing. +fn contains_term_ci(haystack: &str, term: &str) -> bool { + build_word_regex(term) + .map(|re| re.is_match(haystack)) + .unwrap_or(false) +} + +/// Report curated protected terms that were present in `before` but are missing +/// from `after` — i.e. silently dropped or mutated by a downstream stage +/// (typically an LLM formatting/assistive pass). Returns canonical forms in a +/// stable, deduplicated order so the quality loop and operator can see exactly +/// which operator vocabulary was lost. +pub fn protected_terms_lost(before: &str, after: &str) -> Vec { + let canonicals = { + let lexicon = GLOBAL_LEXICON + .read() + .expect("global lexicon read lock poisoned"); + lexicon.protected_canonicals.clone() + }; + + let mut lost = Vec::new(); + for term in canonicals { + if contains_term_ci(before, &term) && !contains_term_ci(after, &term) { + lost.push(term); + } + } + lost +} + fn load_legacy_jsonl(source: &str, label: &str, rules: &mut Vec) -> usize { let mut added = 0usize; for (idx, line) in source.lines().enumerate() { @@ -270,6 +339,69 @@ fn load_legacy_jsonl(source: &str, label: &str, rules: &mut Vec) -> added } +/// Load curated protected-term entries (legacy `term`+`mispronunciations` shape). +/// +/// Differs from [`load_legacy_jsonl`] in two deliberate ways: +/// 1. A variant is skipped only when it is *exactly* equal to the canonical, so +/// case-only variants ("aicx" -> "AICX") still produce a normalization rule. +/// This is safe ONLY because the source file is hand-vetted to proper nouns. +/// 2. Each canonical is recorded in `canonicals` so the quality loop can detect +/// when a protected term is lost downstream (e.g. by an LLM rewrite). +fn load_protected_jsonl( + source: &str, + label: &str, + rules: &mut Vec, + canonicals: &mut Vec, +) -> usize { + let mut added = 0usize; + for (idx, line) in source.lines().enumerate() { + let line = line.trim(); + if line.is_empty() { + continue; + } + + let entry: LegacyEntry = match serde_json::from_str(line) { + Ok(entry) => entry, + Err(e) => { + warn!( + "Protected lexicon line {} ({}) failed to parse: {}", + idx + 1, + label, + e + ); + continue; + } + }; + + if !canonicals.iter().any(|c| c == &entry.term) { + canonicals.push(entry.term.clone()); + } + + let mut all_mis = entry.mispronunciations; + if let Some(extras) = entry.extras { + all_mis.extend(extras.mispronunciations); + } + + for mis in all_mis.iter() { + // Skip only exact duplicates; case-only differences are intentional + // normalization rules (the whole point of this curated source). + if mis == &entry.term { + continue; + } + + if let Some(pattern) = build_word_regex(mis) { + rules.push(LexiconRule { + pattern, + replacement: entry.term.clone(), + }); + added += 1; + } + } + } + + added +} + fn load_seed_jsonl(source: &str, label: &str, rules: &mut Vec) -> usize { let mut added = 0usize; for (idx, line) in source.lines().enumerate() { @@ -723,7 +855,7 @@ mod tests { assert_eq!( output, - "Bede nagrywal cos o loctree i nagrywanie o loctree." + "Bede nagrywal cos o Loctree i nagrywanie o Loctree." ); } @@ -787,6 +919,7 @@ mod tests { custom_mtime: std::fs::metadata(&custom_path) .ok() .and_then(|m| m.modified().ok()), + protected_canonicals: Vec::new(), }; // No rules yet @@ -828,6 +961,7 @@ mod tests { custom_rules: Vec::new(), custom_path: custom_path.clone(), custom_mtime: None, // Force initial load + protected_canonicals: Vec::new(), }; // First reload loads the rule @@ -864,6 +998,7 @@ mod tests { custom_mtime: std::fs::metadata(&custom_path) .ok() .and_then(|m| m.modified().ok()), + protected_canonicals: Vec::new(), }; // Write custom rule @@ -955,4 +1090,113 @@ mod tests { lexicon.rule_count() ); } + + /// Build a hermetic builtin-only lexicon (programming + seed + protected), + /// with NO operator custom file, so protected-term regression assertions are + /// deterministic regardless of the host's ~/.codescribe/lexicon.custom.jsonl. + fn builtin_only_lexicon() -> Lexicon { + let mut rules = Vec::new(); + for (label, source) in BUILTIN_LEXICONS { + load_legacy_jsonl(source, label, &mut rules); + } + load_seed_jsonl(SEED_JSONL, "seed", &mut rules); + let mut canonicals = Vec::new(); + load_protected_jsonl( + PROTECTED_TERMS_JSONL, + "protected", + &mut rules, + &mut canonicals, + ); + Lexicon { + builtin_rules: rules, + custom_rules: Vec::new(), + custom_path: PathBuf::from("/nonexistent/lexicon.custom.jsonl"), + custom_mtime: None, + protected_canonicals: canonicals, + } + } + + #[test] + fn test_protected_terms_loctree_not_luxury() { + let lex = builtin_only_lexicon(); + // The reported regression: Whisper/LLM emits the acoustic homophone + // "Luxury" for the product name. The lexicon must restore "Loctree". + assert_eq!( + lex.apply("Odpalam luxury na repo"), + "Odpalam Loctree na repo" + ); + assert_eq!(lex.apply("locktree i loktree"), "Loctree i Loctree"); + // Canonical already correct stays correct. + assert_eq!(lex.apply("Loctree daje sight"), "Loctree daje sight"); + } + + #[test] + fn test_protected_terms_preserve_brand_casing() { + let lex = builtin_only_lexicon(); + assert_eq!(lex.apply("vibe crafted"), "Vibecrafted"); + assert_eq!(lex.apply("code scribe"), "CodeScribe"); + assert_eq!(lex.apply("vet coders"), "VetCoders"); + // Case-only normalization (curated protected source only). + assert_eq!(lex.apply("mam aicx w repo"), "mam AICX w repo"); + assert_eq!(lex.apply("przez mcp"), "przez MCP"); + assert_eq!(lex.apply("a i c x"), "AICX"); + assert_eq!(lex.apply("m c p"), "MCP"); + assert_eq!(lex.apply("github"), "GitHub"); + assert_eq!(lex.apply("git hub"), "GitHub"); + } + + #[test] + fn test_protected_terms_multiword_phrases() { + let lex = builtin_only_lexicon(); + assert_eq!(lex.apply("fn shift"), "Fn Shift"); + assert_eq!(lex.apply("fun shift"), "Fn Shift"); + assert_eq!(lex.apply("living intent queue"), "Living Intent Queue"); + assert_eq!( + lex.apply("assistive talk anytime"), + "Assistive Talk Anytime" + ); + // Already-correct phrases are preserved verbatim. + assert_eq!( + lex.apply("Collapsible Tool Evidence"), + "Collapsible Tool Evidence" + ); + } + + #[test] + fn test_protected_terms_do_not_overcorrect_ordinary_language() { + let lex = builtin_only_lexicon(); + // "rest", "harmony", "diesel" exist as case-only variants in + // programming.jsonl but the legacy loader skips case-equal variants, so + // ordinary English/Polish must pass through untouched. + let sentence = "I need some rest in harmony near the diesel engine"; + assert_eq!(lex.apply(sentence), sentence); + let pl = "To jest zwykłe zdanie bez żadnych nazw własnych"; + assert_eq!(lex.apply(pl), pl); + } + + #[test] + fn test_protected_terms_lost_detects_corruption() { + // Uses the GLOBAL lexicon; builtin protected canonicals (Loctree, + // CodeScribe, MCP, ...) are always present regardless of custom file. + let lost = protected_terms_lost("I run Loctree through MCP", "I run Luxury through MCP"); + assert_eq!(lost, vec!["Loctree".to_string()]); + + // Nothing lost when the term survives. + let none = protected_terms_lost("CodeScribe is great", "CodeScribe is wonderful"); + assert!(none.is_empty()); + } + + #[test] + fn test_apply_lexicon_is_idempotent_on_canonical() { + // Re-applying after an LLM pass must reach a fixed point (no oscillation / + // corruption). Uses the GLOBAL lexicon, so we only assert robustness + // properties that an operator custom file cannot flip: the pass converges + // and Loctree/AICX/MCP (which no builtin/operator rule downgrades) survive. + let once = apply_lexicon("Loctree, AICX and MCP keep working"); + let twice = apply_lexicon(&once); + assert_eq!(once, twice, "lexicon apply must be idempotent on canonical"); + assert!(once.contains("Loctree")); + assert!(once.contains("AICX")); + assert!(once.contains("MCP")); + } } diff --git a/core/quality/qube_report.rs b/core/quality/qube_report.rs index e94076e4..745d9c4c 100644 --- a/core/quality/qube_report.rs +++ b/core/quality/qube_report.rs @@ -525,6 +525,20 @@ async fn process_pair( None }; + // Protected-vocabulary audit: flag operator/tool/agent names that survived + // the post-lexicon transcript but were dropped or mutated by the AI pass. + // This makes technical-name corruption visible to the operator instead of + // silently shipping "plausible prose" that lost the intended terms. + if let (Some(post_text), Some(ai_text)) = (post.as_deref(), ai_formatted.as_deref()) { + let lost = crate::stream_postprocess::protected_terms_lost(post_text, ai_text); + if !lost.is_empty() { + errors.push(format!( + "Protected terms lost in AI formatting: {}", + lost.join(", ") + )); + } + } + let cloud = cloud_jobs.take_for(&id, &mut errors).await; let metrics_reference = match config.metrics_reference { From e6c43d611e70d7167d19259194e0f8fa8e0ae33e Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Wed, 24 Jun 2026 14:30:09 -0700 Subject: [PATCH 004/385] [claude/vc-workflow] feat(lexicon): normalize Polish operator/command vocab to code tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whisper's final-pass mangled spoken Polish UI commands — the reported goblin "schowku" -> "schopku". Add a curated operator/command vocabulary lexicon that normalizes spoken Polish UI-command phrases and their phonetic mis-hears to the canonical code token the codebase actually uses. Canonicals confirmed real and high-frequency via `loct occurrences`: clipboard=230, transcript=389, paste=168, agent=171, prompt=110, selection=82, frontmost=33, screenshot=14. - assets/operator_vocabulary.jsonl: 8 commands, seed format; variants written the way Polish actually sounds through Whisper (schowku->schofku->schopku, screenshot->skrinszot, frontmost->frontmołst, clipboard->klipbord), all collapsing to an invariant English code token (sidesteps Polish inflection). - stream_postprocess.rs: loaded via load_seed_jsonl (rules-only) so these common words never enter protected_canonicals / the loss-detection gate; operator= telemetry; wired into the builtin_only_lexicon test helper. - test_polish_ui_command_phrase_preservation: the reported goblin + command coverage + an ordinary-text no-overcorrection guard. Variants are phonetically reasoned — no live Whisper-observed store exists yet; the loct-occurrences induction loop is the follow-up for ground-truth variants. --- assets/operator_vocabulary.jsonl | 8 ++++++ core/pipeline/stream_postprocess.rs | 40 ++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 assets/operator_vocabulary.jsonl diff --git a/assets/operator_vocabulary.jsonl b/assets/operator_vocabulary.jsonl new file mode 100644 index 00000000..22f558e7 --- /dev/null +++ b/assets/operator_vocabulary.jsonl @@ -0,0 +1,8 @@ +{"id": "op-clipboard", "canonical": "clipboard", "semantic_type": "ui_command", "register": "operator", "language": "pl", "normalization": {"enabled": true, "input_variants": ["schowek", "schowku", "schowka", "schofku", "schofka", "schopku", "schopka", "s howek", "klipbord", "klip bord", "klipboard"], "whole_word_only": true, "case_sensitive": false}} +{"id": "op-selection", "canonical": "selection", "semantic_type": "ui_command", "register": "operator", "language": "pl", "normalization": {"enabled": true, "input_variants": ["zaznaczenie", "zaznaczenia", "zaznaczeniu", "zaznaczeniem", "selekszyn", "selekszon"], "whole_word_only": true, "case_sensitive": false}} +{"id": "op-paste", "canonical": "paste", "semantic_type": "ui_command", "register": "operator", "language": "pl", "normalization": {"enabled": true, "input_variants": ["wklej", "wkleić", "wklejone", "fklej", "f klej", "fkleić", "pejst"], "whole_word_only": true, "case_sensitive": false}} +{"id": "op-screenshot", "canonical": "screenshot", "semantic_type": "ui_command", "register": "operator", "language": "pl", "normalization": {"enabled": true, "input_variants": ["skrinszot", "skrin szot", "skrinszota", "zrzut ekranu", "zżut ekranu", "z żut ekranu"], "whole_word_only": true, "case_sensitive": false}} +{"id": "op-transcript", "canonical": "transcript", "semantic_type": "ui_command", "register": "operator", "language": "pl", "normalization": {"enabled": true, "input_variants": ["transkrypt", "transkryptu", "transkrypcie", "transkrypcja", "trans skrypt", "transkript"], "whole_word_only": true, "case_sensitive": false}} +{"id": "op-frontmost", "canonical": "frontmost", "semantic_type": "ui_command", "register": "operator", "language": "pl", "normalization": {"enabled": true, "input_variants": ["frontmołst", "front mołst", "front most", "frontmoust"], "whole_word_only": true, "case_sensitive": false}} +{"id": "op-prompt", "canonical": "prompt", "semantic_type": "ui_command", "register": "operator", "language": "pl", "normalization": {"enabled": true, "input_variants": ["prompta", "promptem", "prompcie", "promt"], "whole_word_only": true, "case_sensitive": false}} +{"id": "op-agent", "canonical": "agent", "semantic_type": "ui_command", "register": "operator", "language": "pl", "normalization": {"enabled": true, "input_variants": ["agenta", "agentem", "agentowi", "ejdżent", "ejdzent"], "whole_word_only": true, "case_sensitive": false}} diff --git a/core/pipeline/stream_postprocess.rs b/core/pipeline/stream_postprocess.rs index 837ed748..8da008fd 100644 --- a/core/pipeline/stream_postprocess.rs +++ b/core/pipeline/stream_postprocess.rs @@ -16,6 +16,14 @@ const BUILTIN_LEXICONS: &[(&str, &str)] = &[( include_str!("../../assets/programming.jsonl"), )]; const SEED_JSONL: &str = include_str!("../../assets/seed.jsonl"); +/// Curated operator/command vocabulary. Spoken Polish UI-command phrases and +/// their Whisper mis-hears normalize to the canonical *code token* the codebase +/// actually uses (e.g. "schowek"/"schowku"/"schowka"/"schopku" -> "clipboard"). +/// Loaded rules-only via `load_seed_jsonl` (seed format gives whole-word + +/// case control), so these common words never enter `protected_canonicals` and +/// never trip the downstream loss-detection gate. Canonicals were confirmed +/// real and high-frequency via `loct occurrences` before being chosen. +const OPERATOR_VOCAB_JSONL: &str = include_str!("../../assets/operator_vocabulary.jsonl"); /// Curated proper-noun / operator-vocabulary lexicon. Unlike the generic /// programming/seed sources, entries here are case-normalizing: a variant that /// differs from the canonical only by casing (e.g. "aicx" -> "AICX") still @@ -125,6 +133,11 @@ impl Lexicon { let seed_count = load_seed_jsonl(SEED_JSONL, "seed", &mut builtin_rules); let seed_ms = t_seed.elapsed().as_millis(); + // Operator/command vocabulary: spoken Polish UI commands + their + // mis-hears normalize to the canonical code token. Seed format (rules + // only) keeps these common words out of `protected_canonicals`. + let operator_count = load_seed_jsonl(OPERATOR_VOCAB_JSONL, "operator", &mut builtin_rules); + // Protected terms load LAST among builtin sources so their brand casing // wins over any generic earlier rule that produced a lower-cased form. let mut protected_canonicals = Vec::new(); @@ -152,13 +165,14 @@ impl Lexicon { if total > 0 { info!( - "Loaded {} lexicon rules in {}ms (legacy={} in {}ms, seed={} in {}ms, protected={} terms={}, custom={} in {}ms, custom_path={})", + "Loaded {} lexicon rules in {}ms (legacy={} in {}ms, seed={} in {}ms, operator={}, protected={} terms={}, custom={} in {}ms, custom_path={})", total, total_ms, legacy_count, legacy_ms, seed_count, seed_ms, + operator_count, protected_count, protected_canonicals.len(), custom_count, @@ -1100,6 +1114,7 @@ mod tests { load_legacy_jsonl(source, label, &mut rules); } load_seed_jsonl(SEED_JSONL, "seed", &mut rules); + load_seed_jsonl(OPERATOR_VOCAB_JSONL, "operator", &mut rules); let mut canonicals = Vec::new(); load_protected_jsonl( PROTECTED_TERMS_JSONL, @@ -1174,6 +1189,29 @@ mod tests { assert_eq!(lex.apply(pl), pl); } + #[test] + fn test_polish_ui_command_phrase_preservation() { + // Regression class: Polish UI command phrases (and their Whisper + // mis-hears) must normalize to the canonical code token, never leak the + // garbage mutant. The reported goblin: "schowku" -> "schopku". + let lex = builtin_only_lexicon(); + // The reported mutant and the whole "schowek" inflection family collapse + // to the invariant code token (clipboard never inflects in Polish). + assert_eq!(lex.apply("wrzuć do schopku"), "wrzuć do clipboard"); + assert_eq!(lex.apply("otwórz schowek"), "otwórz clipboard"); + assert_eq!(lex.apply("wrzuć do schowka"), "wrzuć do clipboard"); + assert_eq!(lex.apply("zajrzyj do schowku"), "zajrzyj do clipboard"); + // Other operator commands normalize to their code token. + assert_eq!(lex.apply("zrób skrinszot"), "zrób screenshot"); + assert_eq!(lex.apply("zrób zrzut ekranu"), "zrób screenshot"); + assert_eq!(lex.apply("wklej to"), "paste to"); + assert_eq!(lex.apply("pokaż zaznaczenie"), "pokaż selection"); + assert_eq!(lex.apply("zapisz transkrypt"), "zapisz transcript"); + // Ordinary text without command vocabulary is untouched. + let plain = "To jest zwykłe zdanie o kotach i psach"; + assert_eq!(lex.apply(plain), plain); + } + #[test] fn test_protected_terms_lost_detects_corruption() { // Uses the GLOBAL lexicon; builtin protected canonicals (Loctree, From c4c4127631d43dd16fcb1b3c9063a55285ae00fe Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Wed, 24 Jun 2026 15:34:38 -0700 Subject: [PATCH 005/385] =?UTF-8?q?[claude/vc-justdo]=20feat(assistive):?= =?UTF-8?q?=20Talk=20Anytime=20=E2=80=94=20accept=20assistive=20hotkey=20w?= =?UTF-8?q?hile=20agent=20streams?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Open the agent-send hotkey gate for assistive "Talk Anytime" input. While a previously-dispatched agent turn streams in the background (State::Idle, mic free), should_block_hotkey_during_agent_send no longer ignores assistive start events — the new voice intent reaches the recording path and is captured into the existing pending-follow-up buffer (should_capture_pending_followup → get_or_create_pending_followup_index) instead of being dropped. Discriminator is precise: only assistive start events pass; raw dictation starts stay blocked (no barging a live turn), and the gate acts only at Idle so the State::Busy audio/transcription pipeline stays fully protected — no two audio pipelines run concurrently. The predicate is now pure (in-flight flag passed in) and unit-tested over the full matrix. The hold/toggle dictation path keeps State::Busy across its awaited agent turn (serial_lock held), so extending Talk Anytime there needs the agent send detached — documented as the follow-up, not cut blind in a headless run. Authored-By: claude session_id: 3947aa7d-e205-4f49-bb90-dd27562c5494 date: 2026-06-24T15:34:38 PDT runtime: interactive --- app/controller/mod.rs | 53 ++++++++++++++++++-- app/controller/tests.rs | 108 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 151 insertions(+), 10 deletions(-) diff --git a/app/controller/mod.rs b/app/controller/mod.rs index 3d9d5f96..72fd3d07 100644 --- a/app/controller/mod.rs +++ b/app/controller/mod.rs @@ -650,10 +650,39 @@ fn is_hotkey_start_event(event: &HotkeyInput) -> bool { ) } -fn should_block_hotkey_during_agent_send(current_state: State, event: &HotkeyInput) -> bool { +/// An assistive *start* hotkey — FN+Shift hold-down, an assistive toggle press, +/// or any start event flagged `assistive` (Chat / Selection / assistive toggle). +/// These are the "Talk Anytime" inputs the user fires to add a new voice intent +/// while Emil/the agent is still answering. +fn is_assistive_start_event(event: &HotkeyInput) -> bool { + is_hotkey_start_event(event) && event.assistive +} + +/// Block a *new* hotkey start while a previously-dispatched agent turn is still +/// streaming. This fires only at `State::Idle` — the controller has already +/// returned the mic/transcription pipeline; the agent is answering in the +/// background (a detached `tokio::spawn`, see `setup_voice_chat_send_callback`). +/// +/// Exception — **Assistive Talk Anytime**: assistive start events are allowed +/// through so the user can record a *new* voice intent while the agent answers. +/// The resulting utterance is captured into the existing pending-follow-up +/// buffer (`should_capture_pending_followup` → `get_or_create_pending_followup_index`), +/// not dropped — the living intent grows instead of being ignored. Non-assistive +/// (raw) dictation starts stay blocked: barging a raw transcript into a live +/// agent turn is never wanted, and blocking preserves the single-pipeline +/// guarantee for the dictation path. +/// +/// `agent_send_in_flight` is passed in (rather than read from the global) so the +/// decision is a pure function and unit-testable without touching shared state. +fn should_block_hotkey_during_agent_send( + current_state: State, + event: &HotkeyInput, + agent_send_in_flight: bool, +) -> bool { current_state == State::Idle + && agent_send_in_flight && is_hotkey_start_event(event) - && helpers::is_agent_send_in_flight() + && !is_assistive_start_event(event) } fn present_agent_send_hotkey_block() { @@ -1824,7 +1853,11 @@ impl RecordingController { current_state ); - if should_block_hotkey_during_agent_send(current_state, &event) { + if should_block_hotkey_during_agent_send( + current_state, + &event, + helpers::is_agent_send_in_flight(), + ) { present_agent_send_hotkey_block(); return Ok(()); } @@ -1967,7 +2000,19 @@ impl RecordingController { ); } - // Ignore all hotkeys when busy + // Ignore all hotkeys when busy. `State::Busy` covers the active audio + // pipeline: recorder drain → transcription → (for the hold/toggle + // dictation path) the final assistive agent turn, which is awaited while + // `serial_lock` is held. Letting a second start through here would race a + // live audio/transcription pipeline, so it stays blocked unconditionally + // (acceptance: "non-assistive busy/audio/transcription paths remain + // protected; do not run two audio pipelines concurrently"). + // + // Assistive "Talk Anytime" is handled one gate up, at the `Idle` agent- + // send gate (`should_block_hotkey_during_agent_send`): once a turn is + // dispatched in the background the controller returns to `Idle` and the + // mic is free, which is the only state where overlapping a new recording + // is safe. if current_state == State::Busy { info!("App busy; ignoring hotkey event"); return Ok(()); diff --git a/app/controller/tests.rs b/app/controller/tests.rs index 44c85a30..2c10bfe0 100644 --- a/app/controller/tests.rs +++ b/app/controller/tests.rs @@ -523,21 +523,26 @@ fn test_toggle_stop_event_preserves_active_session_identity() { #[tokio::test] #[serial] -async fn test_agent_send_in_flight_blocks_new_hotkey_starts() { +async fn test_agent_send_in_flight_blocks_nonassistive_hotkey_starts() { + // Contract (preserved): a *raw* dictation start fired while a background + // agent turn is still streaming stays blocked — barging a raw transcript + // into a live agent turn is never wanted, and it preserves the single audio + // pipeline. The block lands before the mode-flag section, so the controller + // stays Idle with default flags. let controller = RecordingController::new(); helpers::set_agent_send_in_flight_for_test(true); - let selection_hold = HotkeyInput { + let raw_hold = HotkeyInput { key_type: HotkeyType::Hold, action: HotkeyAction::Down, - assistive: true, - hold_mode: HoldMode::Selection, - force_raw: false, + assistive: false, + hold_mode: HoldMode::Raw, + force_raw: true, force_ai: false, }; controller - .handle_hotkey_event(selection_hold) + .handle_hotkey_event(raw_hold) .await .expect("agent-busy hotkey block should be non-fatal"); @@ -547,6 +552,97 @@ async fn test_agent_send_in_flight_blocks_new_hotkey_starts() { helpers::set_agent_send_in_flight_for_test(false); } +/// Assistive Talk Anytime — the agent-send gate decision is a pure function of +/// (state, event, in-flight flag). These assertions pin the new contract +/// without spawning the heavy async recording machinery. +#[test] +fn test_assistive_talk_anytime_gate_predicate() { + let assistive_chat_hold = HotkeyInput { + key_type: HotkeyType::Hold, + action: HotkeyAction::Down, + assistive: true, + hold_mode: HoldMode::Chat, + force_raw: false, + force_ai: false, + }; + let raw_hold = HotkeyInput { + key_type: HotkeyType::Hold, + action: HotkeyAction::Down, + assistive: false, + hold_mode: HoldMode::Raw, + force_raw: true, + force_ai: false, + }; + let assistive_toggle = HotkeyInput { + key_type: HotkeyType::Toggle, + action: HotkeyAction::Press, + assistive: true, + hold_mode: HoldMode::Raw, + force_raw: false, + force_ai: false, + }; + let release = HotkeyInput { + key_type: HotkeyType::Hold, + action: HotkeyAction::Up, + assistive: true, + hold_mode: HoldMode::Chat, + force_raw: false, + force_ai: false, + }; + + // Classifier: only *start* events flagged assistive are Talk-Anytime starts. + assert!(is_assistive_start_event(&assistive_chat_hold)); + assert!(is_assistive_start_event(&assistive_toggle)); + assert!(!is_assistive_start_event(&raw_hold)); // raw start is not assistive + assert!(!is_assistive_start_event(&release)); // a release is not a start + + // Talk Anytime: an assistive start is ALLOWED through while the agent + // answers in the background (Idle + in-flight) — it must reach the recording + // path so its utterance flows into the pending-follow-up buffer. + assert!( + !should_block_hotkey_during_agent_send(State::Idle, &assistive_chat_hold, true), + "FN+Shift Talk Anytime must not be ignored while an agent turn streams" + ); + assert!( + !should_block_hotkey_during_agent_send(State::Idle, &assistive_toggle, true), + "assistive toggle Talk Anytime must not be ignored while an agent turn streams" + ); + + // Protected: a raw dictation start stays blocked during a streaming turn. + assert!( + should_block_hotkey_during_agent_send(State::Idle, &raw_hold, true), + "raw dictation must not barge a live agent turn" + ); + + // No agent in flight → nothing is gated (normal idle dictation works). + assert!(!should_block_hotkey_during_agent_send( + State::Idle, + &raw_hold, + false + )); + assert!(!should_block_hotkey_during_agent_send( + State::Idle, + &assistive_chat_hold, + false + )); + + // Non-start events (key release) are never blocked by this gate, so a hold + // release can always cancel/finish even mid-turn. + assert!(!should_block_hotkey_during_agent_send( + State::Idle, + &release, + true + )); + + // The gate only acts at Idle: while audio/transcription holds State::Busy the + // separate Busy guard owns the decision (this gate stays out of its way). + assert!(!should_block_hotkey_during_agent_send( + State::Busy, + &raw_hold, + true + )); +} + fn make_final_pass_verdict( text: &str, speech_pct: f32, From c705e5a048bbc66ebf93155765945fa5ebfd07dd Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Thu, 25 Jun 2026 15:48:07 -0700 Subject: [PATCH 006/385] [claude/vc-workflow] feat(assistive): group tool calls into one Tool Activity block per turn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tool results were pushed as standalone System cards, so with several tools per assistant turn they interleaved with the streaming answer — the reader had to reconstruct execution order. Now every tool event of one turn folds into a single, collapsible Tool Activity evidence block, rendered separate from the assistant answer (which stays one uninterrupted bubble). - core: AgentUiEvent::ToolResult carries is_error (the failure signal already computed in session.rs but dropped at the event boundary); summarize_tool_result now surfaces the real error reason (e.g. "empty index") instead of a generic "N error result(s)" so failed lines read compactly. - app: new pure, unit-tested tool_activity module (ToolStatus / ToolActivityEntry / ToolActivityGroup) owns grouping + summary rendering. New ChatRole::ToolActivity + per-turn state (active_tool_activity_index + tool_activity_groups keyed by message index, mirroring message_render_modes). One block per turn; the open pointer resets when the assistant message finalizes or errors, and is cleared on new-chat / teardown. Block is expanded by default (header + compact per-tool lines), click toggles to header-only. Raw mcp__ wire names never reach the timeline; raw payloads stay in the debug log. Tests: 10 pure grouping tests (incl. operator regression sequence), 3 state-level accumulation/rendering tests, updated missing-tool session test for the new is_error field. fmt+clippy(-D warnings) clean; voice_chat 66, core agent 36 green. Authored-By: claude session_id: 79190759-87f8-4ebb-bf61-715aac30001b date: 2026-06-25T15:42:00 CEST runtime: terminal --- app/controller/helpers.rs | 53 ++-- app/ui/voice_chat/api/export.rs | 1 + app/ui/voice_chat/api/lifecycle.rs | 3 + app/ui/voice_chat/api/messages.rs | 156 +++++++++++- app/ui/voice_chat/api/mod.rs | 1 + app/ui/voice_chat/api/send.rs | 4 + app/ui/voice_chat/api/tests.rs | 209 ++++++++++++++++ app/ui/voice_chat/mod.rs | 14 +- app/ui/voice_chat/state.rs | 15 ++ app/ui/voice_chat/tool_activity.rs | 378 +++++++++++++++++++++++++++++ core/agent/event.rs | 4 + core/agent/session.rs | 46 +++- 12 files changed, 831 insertions(+), 53 deletions(-) create mode 100644 app/ui/voice_chat/tool_activity.rs diff --git a/app/controller/helpers.rs b/app/controller/helpers.rs index e92102fe..e7df455b 100644 --- a/app/controller/helpers.rs +++ b/app/controller/helpers.rs @@ -498,18 +498,6 @@ pub(crate) fn tool_running_status(friendly: &str) -> String { } } -/// Compact one-line evidence entry for a completed tool call. Replaces the old -/// pair of raw "Tool call started/finished: mcp__…" cards with a single readable -/// line. The full raw name + summary still goes to the debug log. -pub(crate) fn tool_evidence_line(friendly: &str, summary: &str) -> String { - let summary = summary.trim(); - if summary.is_empty() { - format!("{friendly} completed") - } else { - format!("{friendly} completed · {summary}") - } -} - async fn apply_agent_ui_event(event: AgentUiEvent, overlay_state: &mut AgentUiOverlayState) { match event { AgentUiEvent::TextDelta(delta) => { @@ -541,24 +529,32 @@ async fn apply_agent_ui_event(event: AgentUiEvent, overlay_state: &mut AgentUiOv // (this is what left the overlay silent). Reuses the proven append path. crate::ui::voice_chat::append_voice_chat_reasoning_delta(&delta); } - AgentUiEvent::ToolExecuting { name, .. } => { - // Collapsible Tool Evidence: no per-start chat card. The status pill - // communicates activity; the timeline stays for conversation. The raw + AgentUiEvent::ToolExecuting { name, id } => { + // Grouped Tool Activity: no per-start chat card. The status pill + // communicates live activity; the timeline stays for conversation. + // The tool is folded into this turn's single evidence block. The raw // wire name is debug-only. let friendly = friendly_tool_name(&name); debug!("Tool executing: {name} -> {friendly}"); crate::ui::voice_chat::update_voice_chat_status(&tool_running_status(&friendly)); + crate::ui::voice_chat::record_tool_executing(&name, &friendly, &id); } - AgentUiEvent::ToolResult { name, summary, .. } => { - // One compact, human-readable evidence line per completed call instead - // of the raw "Tool call finished: mcp__…" card. Raw name + full summary - // are kept in the debug log (raw tool output is debug, not chat). + AgentUiEvent::ToolResult { + name, + id, + summary, + is_error, + } => { + // Fold the completed/failed call into this turn's grouped evidence + // block instead of pushing a standalone System card between answer + // chunks. Raw name + full summary stay in the debug log (raw tool + // output is debug, not chat). let friendly = friendly_tool_name(&name); - debug!("Tool result: {name} -> {friendly} | raw summary: {summary}"); + debug!( + "Tool result: {name} -> {friendly} | is_error={is_error} | raw summary: {summary}" + ); crate::ui::voice_chat::update_voice_chat_status("Thinking…"); - crate::ui::voice_chat::add_voice_chat_system_message(&tool_evidence_line( - &friendly, &summary, - )); + crate::ui::voice_chat::record_tool_result(&name, &friendly, &id, &summary, is_error); } AgentUiEvent::Done => {} AgentUiEvent::Error(message) => { @@ -1221,17 +1217,6 @@ mod tests { ); } - #[test] - fn tool_evidence_line_is_compact() { - assert_eq!( - tool_evidence_line("Web search", "10 results"), - "Web search completed · 10 results" - ); - assert_eq!(tool_evidence_line("Summarize", ""), "Summarize completed"); - // No raw transport noise leaks into the evidence line. - assert!(!tool_evidence_line("Web search", "10 results").contains("mcp__")); - } - struct NoopTestProvider; #[async_trait] diff --git a/app/ui/voice_chat/api/export.rs b/app/ui/voice_chat/api/export.rs index 3e8eac80..2938033f 100644 --- a/app/ui/voice_chat/api/export.rs +++ b/app/ui/voice_chat/api/export.rs @@ -67,6 +67,7 @@ pub fn chat_markdown_from_messages(messages: &[ChatMessage], assistant_only: boo ChatRole::Assistant => "Assistant", ChatRole::System => "System", ChatRole::Reasoning => "Reasoning", + ChatRole::ToolActivity => "Tool activity", }; out.push_str(&format!("## {}\n\n", role)); out.push_str(msg.text.trim_end()); diff --git a/app/ui/voice_chat/api/lifecycle.rs b/app/ui/voice_chat/api/lifecycle.rs index 94862501..13075f17 100644 --- a/app/ui/voice_chat/api/lifecycle.rs +++ b/app/ui/voice_chat/api/lifecycle.rs @@ -197,6 +197,9 @@ pub fn clear_overlay_state(state: &mut VoiceChatOverlayState) { state.active_user_stream_index = None; state.active_assistant_stream_index = None; state.active_reasoning_stream_index = None; + // Close any open tool-activity turn; the block messages persist with the + // transcript and re-render from their groups (keyed by message index). + state.active_tool_activity_index = None; state.is_sending = false; state.scroll_pinned = true; state.manual_draft.clear(); diff --git a/app/ui/voice_chat/api/messages.rs b/app/ui/voice_chat/api/messages.rs index cc5cdcec..382ed5be 100644 --- a/app/ui/voice_chat/api/messages.rs +++ b/app/ui/voice_chat/api/messages.rs @@ -89,6 +89,7 @@ fn add_voice_chat_error_message_impl(text: &str) { state.active_assistant_stream_index = None; state.active_reasoning_stream_index = None; clear_agent_thinking_state(&mut state); + close_tool_activity_turn(&mut state); let mode = message_mode_label(&state); state.messages.push(ChatMessage { role: ChatRole::System, @@ -130,6 +131,140 @@ fn add_voice_chat_system_message_impl(text: &str) { update_chat_view_with_state(&mut state, true); } +/// Record a "tool started" event into the current assistant turn's grouped Tool +/// Activity block. +/// +/// Grouped Tool Activity: tool calls belong to the turn, not between the words. +/// Instead of pushing a standalone card per tool, every tool event of one turn +/// folds into a single evidence block so the assistant answer streams as one +/// uninterrupted message. `raw_name` is debug-only; `display_name` is the +/// human-readable label shown in the timeline. +pub fn record_tool_executing(raw_name: &str, display_name: &str, id: &str) { + let raw = raw_name.to_string(); + let display = display_name.to_string(); + let id = id.to_string(); + Queue::main().exec_async(move || { + run_when_overlay_unlocked(move || record_tool_executing_impl(&raw, &display, &id)); + }); +} + +fn record_tool_executing_impl(raw_name: &str, display_name: &str, id: &str) { + let mut state = OVERLAY_STATE.lock().unwrap_or_else(|e| e.into_inner()); + ensure_agent_tab_visible(&mut state); + let idx = ensure_tool_activity_block(&mut state); + if let Some(group) = state.tool_activity_groups.get_mut(&idx) { + group.mark_running(id, raw_name, display_name); + } + refresh_tool_activity_message(&mut state, idx); + update_chat_view_with_state(&mut state, true); +} + +/// Record a "tool completed/failed" event into the current turn's grouped Tool +/// Activity block. `summary` is the compact result text (or failure reason); +/// `is_error` flips the entry to a compact `· failed · ` line. The full +/// payload stays in the debug log, never in the conversation timeline. +pub fn record_tool_result( + raw_name: &str, + display_name: &str, + id: &str, + summary: &str, + is_error: bool, +) { + let raw = raw_name.to_string(); + let display = display_name.to_string(); + let id = id.to_string(); + let summary = summary.to_string(); + Queue::main().exec_async(move || { + run_when_overlay_unlocked(move || { + record_tool_result_impl(&raw, &display, &id, &summary, is_error) + }); + }); +} + +fn record_tool_result_impl( + raw_name: &str, + display_name: &str, + id: &str, + summary: &str, + is_error: bool, +) { + let mut state = OVERLAY_STATE.lock().unwrap_or_else(|e| e.into_inner()); + ensure_agent_tab_visible(&mut state); + let idx = ensure_tool_activity_block(&mut state); + if let Some(group) = state.tool_activity_groups.get_mut(&idx) { + group.mark_result(id, raw_name, display_name, summary, is_error); + } + refresh_tool_activity_message(&mut state, idx); + update_chat_view_with_state(&mut state, true); +} + +/// Return the current turn's Tool Activity message index, creating the block (a +/// single `ToolActivity` message + its empty group) if this turn has none yet. +/// +/// One block per turn: `active_tool_activity_index` is reset to `None` when the +/// assistant message finalizes, so the next turn's first tool event opens a new +/// block. The block is a separate message from the assistant answer, so folding +/// tools into it never splits the streamed answer. +pub fn ensure_tool_activity_block(state: &mut VoiceChatOverlayState) -> usize { + if let Some(idx) = state.active_tool_activity_index + && state + .messages + .get(idx) + .map(|msg| msg.role == ChatRole::ToolActivity) + .unwrap_or(false) + { + return idx; + } + + let mode = message_mode_label(state); + state.messages.push(ChatMessage { + role: ChatRole::ToolActivity, + text: String::new(), + is_streaming: false, + // Expanded by default: the per-tool lines are already compact, and + // failures stay visible without a click. Clicking collapses to header. + is_collapsed: false, + is_error: false, + timestamp: SystemTime::now(), + mode: Some(mode), + is_pending_followup: false, + }); + let idx = state.messages.len() - 1; + state + .tool_activity_groups + .insert(idx, ToolActivityGroup::default()); + state.active_tool_activity_index = Some(idx); + idx +} + +/// Recompute a Tool Activity block's cached `text` from its group, honoring the +/// block's collapsed state. The group is the source of truth; `text` is the +/// rendered cache the bubble layer reads. +pub fn refresh_tool_activity_message(state: &mut VoiceChatOverlayState, idx: usize) { + let collapsed = state + .messages + .get(idx) + .map(|msg| msg.is_collapsed) + .unwrap_or(false); + let Some(text) = state + .tool_activity_groups + .get(&idx) + .map(|group| group.render(collapsed)) + else { + return; + }; + if let Some(msg) = state.messages.get_mut(idx) { + msg.text = text; + } +} + +/// Close the current turn's Tool Activity block. The block message + its group +/// stay in the log (rendered as finished); only the "open" pointer resets so the +/// next assistant turn starts a fresh block. +pub fn close_tool_activity_turn(state: &mut VoiceChatOverlayState) { + state.active_tool_activity_index = None; +} + /// Add a user message to the chat pub fn add_voice_chat_user_message(text: &str) { let text_owned = text.to_string(); @@ -234,6 +369,13 @@ pub fn handle_message_bubble_click_from_recognizer(sender: Id) { message.is_collapsed = !message.is_collapsed; update_chat_view_with_state(&mut state, false); } + ChatRole::ToolActivity => { + // Toggle compact header-only ↔ expanded per-tool lines, then + // re-render the cached block text from its group. + message.is_collapsed = !message.is_collapsed; + refresh_tool_activity_message(&mut state, index); + update_chat_view_with_state(&mut state, false); + } ChatRole::Assistant if !message.text.is_empty() => { let text = message.text.clone(); drop(state); @@ -560,7 +702,7 @@ pub fn message_render_role(message: &ChatMessage) -> BubbleRole { match message.role { ChatRole::User => BubbleRole::User, ChatRole::Assistant => BubbleRole::Assistant, - ChatRole::System | ChatRole::Reasoning => BubbleRole::System, + ChatRole::System | ChatRole::Reasoning | ChatRole::ToolActivity => BubbleRole::System, } } @@ -603,6 +745,7 @@ pub fn message_role_label(role: ChatRole) -> &'static str { ChatRole::Assistant => "Assistant", ChatRole::System => "System", ChatRole::Reasoning => "Reasoning", + ChatRole::ToolActivity => "Tool activity", } } @@ -973,6 +1116,7 @@ pub fn finalize_assistant_message_impl(text: &str, is_error: bool) { msg.is_error = is_error; } clear_agent_thinking_state(&mut state); + close_tool_activity_turn(&mut state); state.is_sending = false; update_chat_view_with_state(&mut state, true); update_send_button_with_state(&mut state); @@ -992,6 +1136,7 @@ pub fn finalize_assistant_message_state_only_impl(is_error: bool) { last.is_error = is_error; } clear_agent_thinking_state(&mut state); + close_tool_activity_turn(&mut state); state.is_sending = false; update_chat_view_with_state(&mut state, true); update_send_button_with_state(&mut state); @@ -1073,7 +1218,7 @@ pub fn active_stream_index_mut( ChatRole::User => Some(&mut state.active_user_stream_index), ChatRole::Assistant => Some(&mut state.active_assistant_stream_index), ChatRole::Reasoning => Some(&mut state.active_reasoning_stream_index), - ChatRole::System => None, + ChatRole::System | ChatRole::ToolActivity => None, } } @@ -1082,7 +1227,7 @@ pub fn active_stream_index(state: &VoiceChatOverlayState, role: ChatRole) -> Opt ChatRole::User => state.active_user_stream_index, ChatRole::Assistant => state.active_assistant_stream_index, ChatRole::Reasoning => state.active_reasoning_stream_index, - ChatRole::System => None, + ChatRole::System | ChatRole::ToolActivity => None, } } @@ -1226,7 +1371,10 @@ pub fn update_chat_view_with_state(state: &mut VoiceChatOverlayState, scroll_to_ copy_action_target: state.action_handler.map(|p| p as Id), measure_cache: Some(measure_cache_ptr), }); - if matches!(message_role, ChatRole::Assistant | ChatRole::Reasoning) { + if matches!( + message_role, + ChatRole::Assistant | ChatRole::Reasoning | ChatRole::ToolActivity + ) { attach_message_bubble_click_recognizer(state, bubble, index); } stack_view_add(container, bubble); diff --git a/app/ui/voice_chat/api/mod.rs b/app/ui/voice_chat/api/mod.rs index cfe76b4f..df62bc7a 100644 --- a/app/ui/voice_chat/api/mod.rs +++ b/app/ui/voice_chat/api/mod.rs @@ -26,6 +26,7 @@ use super::state::{ ChatMessage, ChatRole, ConversationModeState, DrawerEntry, DrawerEntrySource, OVERLAY_STATE, SEND_CALLBACK, Tab, TranscriptionMode, VoiceChatOverlayState, }; +use super::tool_activity::ToolActivityGroup; use crate::ui::shared::status::{UiStatus, status_from_detail}; use crate::ui_helpers::{ BubbleConfig, BubbleMeasureCache, BubbleRole, LabelConfig, NSEdgeInsets, RenderMode, diff --git a/app/ui/voice_chat/api/send.rs b/app/ui/voice_chat/api/send.rs index 5b9f290f..db609f5f 100644 --- a/app/ui/voice_chat/api/send.rs +++ b/app/ui/voice_chat/api/send.rs @@ -131,6 +131,10 @@ pub fn clear_voice_chat_text_impl() { state.active_user_stream_index = None; state.active_assistant_stream_index = None; state.active_reasoning_stream_index = None; + // Grouped tool evidence is keyed by message index; a cleared transcript + // must drop it so stale indices never alias fresh messages. + state.active_tool_activity_index = None; + state.tool_activity_groups.clear(); state.manual_draft.clear(); state.prompt_history_cursor = None; state.is_sending = false; diff --git a/app/ui/voice_chat/api/tests.rs b/app/ui/voice_chat/api/tests.rs index cf673c9c..ffd4f79a 100644 --- a/app/ui/voice_chat/api/tests.rs +++ b/app/ui/voice_chat/api/tests.rs @@ -1230,3 +1230,212 @@ fn assistive_followup_edit_moves_pending_text_to_draft_without_send() { let mut cb = SEND_CALLBACK.lock().unwrap_or_else(|e| e.into_inner()); *cb = None; } + +// ── Grouped Tool Activity ──────────────────────────────────────────────── +// State-level accumulation + summary rendering. Exercises the pure grouping +// helpers (`ensure_tool_activity_block` / `refresh_tool_activity_message` / +// `close_tool_activity_turn`) directly on a windowless state, so no AppKit +// view build runs — the assertion is purely on the message model. + +fn push_chat_message( + state: &mut VoiceChatOverlayState, + role: ChatRole, + text: &str, + streaming: bool, +) { + state.messages.push(ChatMessage { + role, + text: text.to_string(), + is_streaming: streaming, + is_collapsed: false, + is_error: false, + timestamp: SystemTime::now(), + mode: Some("AI".to_string()), + is_pending_followup: false, + }); +} + +fn feed_tool_running(state: &mut VoiceChatOverlayState, id: &str, raw: &str, display: &str) { + let idx = ensure_tool_activity_block(state); + state + .tool_activity_groups + .get_mut(&idx) + .expect("group exists for the open block") + .mark_running(id, raw, display); + refresh_tool_activity_message(state, idx); +} + +fn feed_tool_result( + state: &mut VoiceChatOverlayState, + id: &str, + raw: &str, + display: &str, + summary: &str, + is_error: bool, +) { + let idx = ensure_tool_activity_block(state); + state + .tool_activity_groups + .get_mut(&idx) + .expect("group exists for the open block") + .mark_result(id, raw, display, summary, is_error); + refresh_tool_activity_message(state, idx); +} + +#[test] +fn turn_with_three_tools_renders_one_grouped_block() { + // Operator regression sequence. Assistant text chunks (1 & 5) and tool + // events (2,3,4,6,7,8) interleave on the wire; the model must produce ONE + // assistant message and ONE tool-activity block — never per-tool cards. + let mut state = VoiceChatOverlayState::default(); + push_chat_message(&mut state, ChatRole::User, "Co tam?", false); + // 1. assistant answer starts (single streaming bubble for the whole turn) + push_chat_message( + &mut state, + ChatRole::Assistant, + "Sprawdzam… Wynik jest taki…", + true, + ); + + // 2-3. brave search start + result + feed_tool_running( + &mut state, + "c1", + "mcp__brave-search__brave_web_search", + "Web search", + ); + feed_tool_result( + &mut state, + "c1", + "mcp__brave-search__brave_web_search", + "Web search", + "10 results", + false, + ); + // 4. loctree start (result arrives later, after more assistant text) + feed_tool_running( + &mut state, + "c2", + "mcp__loctree-mcp__context", + "Loctree context", + ); + // 7. aicx start + feed_tool_running( + &mut state, + "c3", + "mcp__aicx-mcp__aicx_intents", + "AICX intents", + ); + // 6. loctree result + feed_tool_result( + &mut state, + "c2", + "mcp__loctree-mcp__context", + "Loctree context", + "", + false, + ); + // 8. aicx failed + feed_tool_result( + &mut state, + "c3", + "mcp__aicx-mcp__aicx_intents", + "AICX intents", + "empty index", + true, + ); + + // Exactly one tool-activity block for the turn. + let blocks: Vec<&ChatMessage> = state + .messages + .iter() + .filter(|m| m.role == ChatRole::ToolActivity) + .collect(); + assert_eq!(blocks.len(), 1, "all turn tools collapse into one block"); + assert_eq!( + blocks[0].text, + "Tool activity · 3 calls · 1 failed\n\ + - Web search · completed · 10 results\n\ + - Loctree context · completed\n\ + - AICX intents · failed · empty index" + ); + + // The assistant answer stays a single, uninterrupted message. + let assistant_count = state + .messages + .iter() + .filter(|m| m.role == ChatRole::Assistant) + .count(); + assert_eq!(assistant_count, 1, "answer is not split by tool cards"); + + // No raw MCP wire name leaks into the primary timeline. + assert!( + state.messages.iter().all(|m| !m.text.contains("mcp__")), + "raw MCP names must never reach the timeline" + ); +} + +#[test] +fn block_is_reused_within_a_turn_and_reopened_next_turn() { + let mut state = VoiceChatOverlayState::default(); + + feed_tool_running( + &mut state, + "a", + "mcp__loctree-mcp__find", + "Loctree occurrences/find", + ); + let first_idx = state + .active_tool_activity_index + .expect("first turn opened a block"); + // Same turn: a second tool reuses the same block index. + feed_tool_running(&mut state, "b", "read_clipboard", "Clipboard read"); + assert_eq!(state.active_tool_activity_index, Some(first_idx)); + assert_eq!( + state + .messages + .iter() + .filter(|m| m.role == ChatRole::ToolActivity) + .count(), + 1 + ); + + // Turn boundary: assistant finalized → pointer closes. + close_tool_activity_turn(&mut state); + assert_eq!(state.active_tool_activity_index, None); + + // Next turn's first tool opens a brand-new block. + feed_tool_running(&mut state, "c", "take_screenshot", "Screenshot"); + assert_ne!(state.active_tool_activity_index, Some(first_idx)); + assert_eq!( + state + .messages + .iter() + .filter(|m| m.role == ChatRole::ToolActivity) + .count(), + 2, + "each turn renders its own block" + ); +} + +#[test] +fn collapse_toggle_rerenders_block_header_only() { + let mut state = VoiceChatOverlayState::default(); + feed_tool_result( + &mut state, + "a", + "mcp__loctree-mcp__context", + "Loctree context", + "", + false, + ); + let idx = state.active_tool_activity_index.expect("block open"); + + // Expanded by default: header + lines. + assert!(state.messages[idx].text.contains('\n')); + + // Collapse → header only. + state.messages[idx].is_collapsed = true; + refresh_tool_activity_message(&mut state, idx); + assert_eq!(state.messages[idx].text, "Tool activity · 1 call completed"); +} diff --git a/app/ui/voice_chat/mod.rs b/app/ui/voice_chat/mod.rs index 25d11892..7792e172 100644 --- a/app/ui/voice_chat/mod.rs +++ b/app/ui/voice_chat/mod.rs @@ -8,6 +8,7 @@ mod api; mod handlers; mod state; +pub(crate) mod tool_activity; // Re-export public API pub use api::{ @@ -17,12 +18,13 @@ pub use api::{ commit_last_user_message, commit_pending_followup_message, dispatch_voice_chat_send, edit_pending_followup_message, filter_drawer, finalize_voice_chat_assistant_message, finalize_voice_chat_user_message, handoff_transcript_to_chat, hide_voice_chat_overlay, - is_auto_send_enabled, is_conversation_active, is_voice_chat_overlay_visible, refresh_drawer, - reset_voice_chat_activity, send_voice_chat_draft, set_voice_chat_agent_thinking, - set_voice_chat_runtime_degraded, set_voice_chat_send_callback, set_voice_chat_sending, - set_voice_chat_target_app, set_voice_chat_text, set_voice_chat_user_text, show_agent_tab, - show_drawer_tab, update_conversation_state, update_drawer_after_save, - update_voice_chat_context_summary, update_voice_chat_status, + is_auto_send_enabled, is_conversation_active, is_voice_chat_overlay_visible, + record_tool_executing, record_tool_result, refresh_drawer, reset_voice_chat_activity, + send_voice_chat_draft, set_voice_chat_agent_thinking, set_voice_chat_runtime_degraded, + set_voice_chat_send_callback, set_voice_chat_sending, set_voice_chat_target_app, + set_voice_chat_text, set_voice_chat_user_text, show_agent_tab, show_drawer_tab, + update_conversation_state, update_drawer_after_save, update_voice_chat_context_summary, + update_voice_chat_status, }; pub use state::{ConversationModeState, VoiceChatOverlayConfig}; diff --git a/app/ui/voice_chat/state.rs b/app/ui/voice_chat/state.rs index e1f3e191..2eb30aba 100644 --- a/app/ui/voice_chat/state.rs +++ b/app/ui/voice_chat/state.rs @@ -9,6 +9,7 @@ use std::time::{Instant, SystemTime}; use codescribe_core::attachment::Attachment; +use super::tool_activity::ToolActivityGroup; use crate::ui::shared::status::UiStatus; use crate::ui_helpers::{BubbleMeasureCache, RenderMode}; @@ -45,6 +46,10 @@ pub enum ChatRole { /// Live reasoning summary streamed from the agent (its own lane, rendered /// as a collapsible "thinking" entry — NOT mixed into the assistant text). Reasoning, + /// Grouped tool evidence for one assistant turn (its own lane, rendered as a + /// single compact/collapsible block — NEVER interleaved between answer + /// chunks). Backed by a [`ToolActivityGroup`] keyed by message index. + ToolActivity, } /// A single chat message @@ -187,6 +192,14 @@ pub struct VoiceChatOverlayState { pub active_assistant_stream_index: Option, /// Active streaming reasoning-summary message index (if any). pub active_reasoning_stream_index: Option, + /// Message index of the current assistant turn's grouped tool-activity block + /// (if one is open). `None` between turns — the next tool event opens a fresh + /// block so each turn renders exactly one evidence block. + pub active_tool_activity_index: Option, + /// Grouped tool evidence per turn, keyed by the `ToolActivity` message index + /// (parallel to `message_render_modes`). The block message's `text` is a + /// rendered cache of its group; the group is the source of truth. + pub tool_activity_groups: HashMap, pub manual_draft: String, /// Manual prompts sent from the Agent input, newest at the end. pub prompt_history: Vec, @@ -281,6 +294,8 @@ impl Default for VoiceChatOverlayState { active_user_stream_index: None, active_assistant_stream_index: None, active_reasoning_stream_index: None, + active_tool_activity_index: None, + tool_activity_groups: HashMap::new(), manual_draft: String::new(), prompt_history: Vec::new(), prompt_history_cursor: None, diff --git a/app/ui/voice_chat/tool_activity.rs b/app/ui/voice_chat/tool_activity.rs new file mode 100644 index 00000000..d64fa985 --- /dev/null +++ b/app/ui/voice_chat/tool_activity.rs @@ -0,0 +1,378 @@ +//! Grouped Tool Activity model for the Assistive conversation timeline. +//! +//! Product rule: the primary timeline is for **conversation**. Tool calls are +//! **evidence/activity**, raw tool output is **debug**. An assistant answer must +//! never be interrupted by individual tool-call log cards. +//! +//! Before this module, each completed tool emitted its own `System` chat bubble +//! (`tool_evidence_line`). With several tools per turn those single cards +//! interleaved with the streaming assistant answer, forcing the reader to +//! mentally reconstruct execution order. Here we accumulate every tool event of +//! one assistant turn into a single [`ToolActivityGroup`] and render it as one +//! compact block, separate from the assistant answer. +//! +//! This module is intentionally pure (no AppKit, no global state): the grouping +//! and summary rendering are unit-testable without a running UI. The voice-chat +//! state layer owns one group per assistant turn and feeds it raw events; the +//! rendered string is what the timeline shows. + +/// Lifecycle of a single tool call within a turn. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ToolStatus { + /// Dispatched, result not yet received. + Running, + /// Finished with a usable result. + Completed, + /// Finished in error. + Failed, +} + +impl ToolStatus { + fn word(self) -> &'static str { + match self { + ToolStatus::Running => "running", + ToolStatus::Completed => "completed", + ToolStatus::Failed => "failed", + } + } +} + +/// One tool call's evidence within a turn. `raw_name` is the transport wire name +/// (`mcp__server__tool`) kept only for debug/telemetry; `display_name` is the +/// human-readable label shown in the timeline. +#[derive(Debug, Clone)] +pub struct ToolActivityEntry { + pub id: String, + pub display_name: String, + pub raw_name: String, + pub status: ToolStatus, + /// Short result summary for a completed call (already truncated upstream). + pub summary: String, + /// Error reason for a failed call (compact; full payload stays in the log). + pub error: String, + /// Optional structured result count when the upstream summary exposes one. + pub result_count: Option, +} + +/// Maximum characters for a per-entry suffix so the block stays compact and no +/// raw payload / stack dump leaks into the conversation timeline. +const ENTRY_SUFFIX_MAX_CHARS: usize = 80; + +fn clamp_suffix(text: &str) -> String { + let trimmed = text.trim(); + if trimmed.chars().count() <= ENTRY_SUFFIX_MAX_CHARS { + return trimmed.to_string(); + } + let clipped: String = trimmed + .chars() + .take(ENTRY_SUFFIX_MAX_CHARS.saturating_sub(1)) + .collect(); + format!("{}…", clipped.trim_end()) +} + +impl ToolActivityEntry { + /// One compact line, e.g. `Web search · completed · 10 results` or + /// `AICX intents · failed · empty index`. Never contains the raw wire name. + pub fn line(&self) -> String { + let mut line = format!("{} · {}", self.display_name, self.status.word()); + let suffix = match self.status { + ToolStatus::Completed => { + if let Some(count) = self.result_count { + let noun = if count == 1 { "result" } else { "results" }; + format!("{count} {noun}") + } else { + clamp_suffix(&self.summary) + } + } + ToolStatus::Failed => clamp_suffix(&self.error), + ToolStatus::Running => String::new(), + }; + if !suffix.is_empty() { + line.push_str(" · "); + line.push_str(&suffix); + } + line + } +} + +/// All tool evidence for one assistant turn. Owned per-turn by the voice-chat +/// state; rendered once as a single timeline block. +#[derive(Debug, Clone, Default)] +pub struct ToolActivityGroup { + pub entries: Vec, +} + +impl ToolActivityGroup { + /// Record a tool that just started. Idempotent on `id`: a repeated start for + /// the same call updates its labels but keeps its place. + pub fn mark_running(&mut self, id: &str, raw_name: &str, display_name: &str) { + if let Some(entry) = self.entries.iter_mut().find(|e| e.id == id) { + entry.raw_name = raw_name.to_string(); + entry.display_name = display_name.to_string(); + return; + } + self.entries.push(ToolActivityEntry { + id: id.to_string(), + display_name: display_name.to_string(), + raw_name: raw_name.to_string(), + status: ToolStatus::Running, + summary: String::new(), + error: String::new(), + result_count: None, + }); + } + + /// Record a tool result. Matches the running entry by `id`; if no start was + /// seen (events can be dropped/coalesced) it inserts a finished entry so the + /// evidence is never lost. + pub fn mark_result( + &mut self, + id: &str, + raw_name: &str, + display_name: &str, + summary: &str, + is_error: bool, + ) { + let status = if is_error { + ToolStatus::Failed + } else { + ToolStatus::Completed + }; + let summary_owned = summary.trim().to_string(); + if let Some(entry) = self.entries.iter_mut().find(|e| e.id == id) { + entry.raw_name = raw_name.to_string(); + entry.display_name = display_name.to_string(); + entry.status = status; + if is_error { + entry.error = summary_owned; + } else { + entry.summary = summary_owned; + } + return; + } + self.entries.push(ToolActivityEntry { + id: id.to_string(), + display_name: display_name.to_string(), + raw_name: raw_name.to_string(), + status, + summary: if is_error { + String::new() + } else { + summary_owned.clone() + }, + error: if is_error { + summary_owned + } else { + String::new() + }, + result_count: None, + }); + } + + fn counts(&self) -> (usize, usize, usize) { + let mut completed = 0; + let mut failed = 0; + let mut running = 0; + for entry in &self.entries { + match entry.status { + ToolStatus::Completed => completed += 1, + ToolStatus::Failed => failed += 1, + ToolStatus::Running => running += 1, + } + } + (completed, failed, running) + } + + /// One-line header summarizing the whole turn's tool activity, e.g. + /// `Tool activity · 3 calls completed`, `Tool activity · 3 calls · 1 failed`, + /// or `Tool activity · 2 calls · 1 running`. Failures are always visible here + /// even when the block is collapsed. + pub fn compact_header(&self) -> String { + let total = self.entries.len(); + if total == 0 { + return "Tool activity".to_string(); + } + let noun = if total == 1 { "call" } else { "calls" }; + let (_, failed, running) = self.counts(); + + if running == total { + return format!("Tool activity · {total} {noun} running"); + } + + let mut header = format!("Tool activity · {total} {noun}"); + if running > 0 { + header.push_str(&format!(" · {running} running")); + } + if failed > 0 { + header.push_str(&format!(" · {failed} failed")); + } + if running == 0 && failed == 0 { + // Clean turn: read as "3 calls completed". + header.push_str(" completed"); + } + header + } + + /// Render the block for the timeline. Collapsed → header only (failures still + /// counted in the header). Expanded (default) → header + one compact line per + /// tool. Raw payloads are never included — they stay in the debug log. + pub fn render(&self, collapsed: bool) -> String { + let header = self.compact_header(); + if collapsed || self.entries.is_empty() { + return header; + } + let mut out = header; + for entry in &self.entries { + out.push('\n'); + out.push_str("- "); + out.push_str(&entry.line()); + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn group_from_regression() -> ToolActivityGroup { + // Mirrors the operator's regression event sequence (interleaved with + // assistant text chunks, which this module never sees): + // ToolExecuting brave -> ToolResult brave (10 results) + // ToolExecuting loctree -> ToolResult loctree + // ToolExecuting aicx -> ToolResult failed (empty index) + let mut group = ToolActivityGroup::default(); + group.mark_running("c1", "mcp__brave-search__brave_web_search", "Web search"); + group.mark_result( + "c1", + "mcp__brave-search__brave_web_search", + "Web search", + "10 results", + false, + ); + group.mark_running("c2", "mcp__loctree-mcp__context", "Loctree context"); + group.mark_running("c3", "mcp__aicx-mcp__aicx_intents", "AICX intents"); + group.mark_result( + "c2", + "mcp__loctree-mcp__context", + "Loctree context", + "", + false, + ); + group.mark_result( + "c3", + "mcp__aicx-mcp__aicx_intents", + "AICX intents", + "empty index", + true, + ); + group + } + + #[test] + fn groups_all_turn_tools_into_one_block() { + let group = group_from_regression(); + assert_eq!( + group.entries.len(), + 3, + "three calls collapse into one group" + ); + } + + #[test] + fn header_surfaces_failures_compactly() { + let group = group_from_regression(); + // 2 completed + 1 failed → failure visible in the header. + assert_eq!(group.compact_header(), "Tool activity · 3 calls · 1 failed"); + } + + #[test] + fn header_reads_completed_when_clean() { + let mut group = ToolActivityGroup::default(); + group.mark_result( + "a", + "mcp__loctree-mcp__context", + "Loctree context", + "", + false, + ); + group.mark_result("b", "read_clipboard", "Clipboard read", "ok", false); + assert_eq!(group.compact_header(), "Tool activity · 2 calls completed"); + } + + #[test] + fn header_reports_running_progress() { + let mut group = ToolActivityGroup::default(); + group.mark_running("a", "mcp__brave-search__brave_web_search", "Web search"); + assert_eq!(group.compact_header(), "Tool activity · 1 call running"); + group.mark_running("b", "mcp__loctree-mcp__context", "Loctree context"); + group.mark_result( + "a", + "mcp__brave-search__brave_web_search", + "Web search", + "10 results", + false, + ); + assert_eq!( + group.compact_header(), + "Tool activity · 2 calls · 1 running" + ); + } + + #[test] + fn expanded_render_lists_each_tool_readably() { + let group = group_from_regression(); + let expected = "Tool activity · 3 calls · 1 failed\n\ + - Web search · completed · 10 results\n\ + - Loctree context · completed\n\ + - AICX intents · failed · empty index"; + assert_eq!(group.render(false), expected); + } + + #[test] + fn collapsed_render_is_header_only() { + let group = group_from_regression(); + assert_eq!(group.render(true), "Tool activity · 3 calls · 1 failed"); + } + + #[test] + fn raw_wire_names_never_leak_into_rendered_block() { + let group = group_from_regression(); + let rendered = group.render(false); + assert!( + !rendered.contains("mcp__"), + "raw MCP names must not reach the timeline" + ); + } + + #[test] + fn result_for_unseen_start_is_still_recorded() { + // A ToolResult with no prior ToolExecuting (dropped/coalesced start) + // must still produce evidence rather than vanish. + let mut group = ToolActivityGroup::default(); + group.mark_result("z", "take_screenshot", "Screenshot", "saved", false); + assert_eq!(group.entries.len(), 1); + assert_eq!(group.entries[0].status, ToolStatus::Completed); + } + + #[test] + fn duplicate_start_does_not_duplicate_entry() { + let mut group = ToolActivityGroup::default(); + group.mark_running("c1", "mcp__loctree-mcp__find", "Loctree occurrences/find"); + group.mark_running("c1", "mcp__loctree-mcp__find", "Loctree occurrences/find"); + assert_eq!(group.entries.len(), 1); + } + + #[test] + fn long_failure_suffix_is_clamped_not_a_stack_dump() { + let mut group = ToolActivityGroup::default(); + let huge = "panic at line 42: ".repeat(40); + group.mark_result("c1", "mcp__x__y", "Tool", &huge, true); + let line = group.entries[0].line(); + assert!( + line.chars().count() < 120, + "failed line must stay compact: {line}" + ); + assert!(line.ends_with('…'), "clamped suffix is elided"); + } +} diff --git a/core/agent/event.rs b/core/agent/event.rs index b8473163..cf8e6a63 100644 --- a/core/agent/event.rs +++ b/core/agent/event.rs @@ -47,6 +47,10 @@ pub enum AgentUiEvent { name: String, id: String, summary: String, + /// True when the tool dispatch returned an error result. Carried so the + /// UI can render a failed tool compactly (`· failed · `) without + /// string-sniffing the summary. The reason already lives in `summary`. + is_error: bool, }, Done, Error(String), diff --git a/core/agent/session.rs b/core/agent/session.rs index b0e08e08..c1356a61 100644 --- a/core/agent/session.rs +++ b/core/agent/session.rs @@ -414,6 +414,7 @@ impl AgentSession { name: tool_name, id: call_id, summary, + is_error, }, ) .await; @@ -465,6 +466,7 @@ fn summarize_tool_result(outputs: &[ToolResultContent]) -> String { const SUMMARY_MAX_CHARS: usize = 120; let mut first_text: Option = None; + let mut first_error: Option = None; let mut image_count = 0usize; let mut error_count = 0usize; @@ -476,7 +478,12 @@ fn summarize_tool_result(outputs: &[ToolResultContent]) -> String { } } ToolResultContent::Image { .. } | ToolResultContent::ImageAsset(_) => image_count += 1, - ToolResultContent::Error(_) => error_count += 1, + ToolResultContent::Error(message) => { + error_count += 1; + if first_error.is_none() { + first_error = Some(message.trim().to_string()); + } + } } } @@ -491,8 +498,14 @@ fn summarize_tool_result(outputs: &[ToolResultContent]) -> String { return format!("{image_count} image result(s)"); } - if error_count > 0 { - return format!("{error_count} error result(s)"); + // Surface the real failure reason (e.g. "empty index") so the grouped Tool + // Activity block can show it compactly. Fall back to a count when the error + // carries no message. The full payload still goes to the debug log. + if let Some(error) = first_error { + if error.is_empty() { + return format!("{error_count} error result(s)"); + } + return truncate_summary(&error, SUMMARY_MAX_CHARS); } "No tool output".to_string() @@ -1006,13 +1019,28 @@ mod tests { }), "expected ToolExecuting event, got {ui_events:?}" ); + let tool_result_event = ui_events + .iter() + .find_map(|event| match event { + AgentUiEvent::ToolResult { + name, + id, + summary, + is_error, + } => Some((name.clone(), id.clone(), summary.clone(), *is_error)), + _ => None, + }) + .expect("expected a ToolResult UI event"); + assert_eq!(tool_result_event.0, "missing_tool"); + assert_eq!(tool_result_event.1, "call_missing"); + assert!( + tool_result_event.3, + "missing tool dispatch must flag the UI event as an error, got {ui_events:?}" + ); assert!( - ui_events.contains(&AgentUiEvent::ToolResult { - name: "missing_tool".to_string(), - id: "call_missing".to_string(), - summary: "1 error result(s)".to_string(), - }), - "expected ToolResult fallback summary, got {ui_events:?}" + tool_result_event.2.contains("not registered"), + "expected the failure reason to reach the UI summary, got {:?}", + tool_result_event.2 ); assert!( ui_events From 60bc6798c7f3e9eeb9d964214dcb9acb04ed5d12 Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Thu, 25 Jun 2026 16:02:32 -0700 Subject: [PATCH 007/385] [claude/vc-workflow] feat(assistive): honor operator tool-label table in friendly_tool_name The grouped Tool Activity block (commit c705e5a) already folds every tool event of an assistant turn into one evidence card and keeps the assistant answer one uninterrupted bubble. But the raw->label mapping only special-cased the brave-search family; every other surface fell into the generic mcp__ prefix-strip + prettify fallback, which reads reversed and noisy: mcp__loctree-mcp__context -> "Context . Loctree mcp" (wanted "Loctree context") mcp__aicx-mcp__aicx_intents -> "Aicx Intents . Aicx mcp" (wanted "AICX intents") read_clipboard -> "Read Clipboard" (wanted "Clipboard read") The operator's regression "Expected UI" labels ("Loctree context", "AICX intents") therefore only held in the pure-module test, which hardcodes display_name on input -- the live controller path never produced them. Pin the operator's explicit table in friendly_tool_name (the single mapping hub, on the ToolExecuting/ToolResult event boundary): loctree context/find, aicx intents, vibecrafted observe, and the native clipboard/screenshot/audio tools. Unknown mcp__ tools keep the generic prettify fallback (github etc.), so no raw wire name ever reaches the timeline. Tests: friendly_tool_name_honors_operator_label_table (8 entries) + regression_sequence_raw_names_produce_expected_runtime_labels (proves the brave/loctree/aicx raw names map to the exact grouped-block labels at runtime, not just in the hardcoded pure-module fixture). fmt + clippy(-D warnings) clean; 3 friendly + 1 regression + 10 tool_activity grouping tests green. Authored-By: claude session_id: 9c4efcf7-4338-4a46-aa19-b52cd46a02c3 date: 2026-06-25T16:01:48 PDT runtime: terminal --- app/controller/helpers.rs | 63 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/app/controller/helpers.rs b/app/controller/helpers.rs index e7df455b..17eb5194 100644 --- a/app/controller/helpers.rs +++ b/app/controller/helpers.rs @@ -470,6 +470,19 @@ pub(crate) fn friendly_tool_name(raw: &str) -> String { return "Video search".into(); } "mcp__brave-search__brave_summarizer" | "brave_summarizer" => return "Summarize".into(), + // Structural / intent / fleet MCP surfaces the operator named explicitly: + // the generic `mcp__` fallback would read "Context · Loctree mcp", which is + // both reversed and noisy. Pin the exact human labels here. + "mcp__loctree-mcp__context" => return "Loctree context".into(), + "mcp__loctree-mcp__find" => return "Loctree occurrences/find".into(), + "mcp__aicx-mcp__aicx_intents" => return "AICX intents".into(), + "mcp__vibecrafted-mcp__vc_run_observe" => return "Vibecrafted observe".into(), + // Native (non-mcp) tools: the bare snake_case prettifies to a reversed, + // verbose label ("Read Clipboard"); the operator wants noun-first copy. + "read_clipboard" => return "Clipboard read".into(), + "write_clipboard" => return "Clipboard write".into(), + "take_screenshot" => return "Screenshot".into(), + "transcribe_audio" => return "Audio transcription".into(), _ => {} } if let Some(rest) = raw.strip_prefix("mcp__") { @@ -1207,6 +1220,56 @@ mod tests { assert!(!friendly_tool_name("mcp__github__create_issue").contains("mcp__")); } + #[test] + fn friendly_tool_name_honors_operator_label_table() { + // The operator's explicit raw→label table. Before this mapping these all + // fell into the generic `mcp__` / prettify fallback and read reversed or + // noisy (e.g. "Context · Loctree mcp", "Read Clipboard"). + assert_eq!( + friendly_tool_name("mcp__loctree-mcp__context"), + "Loctree context" + ); + assert_eq!( + friendly_tool_name("mcp__loctree-mcp__find"), + "Loctree occurrences/find" + ); + assert_eq!( + friendly_tool_name("mcp__aicx-mcp__aicx_intents"), + "AICX intents" + ); + assert_eq!( + friendly_tool_name("mcp__vibecrafted-mcp__vc_run_observe"), + "Vibecrafted observe" + ); + assert_eq!(friendly_tool_name("read_clipboard"), "Clipboard read"); + assert_eq!(friendly_tool_name("write_clipboard"), "Clipboard write"); + assert_eq!(friendly_tool_name("take_screenshot"), "Screenshot"); + assert_eq!( + friendly_tool_name("transcribe_audio"), + "Audio transcription" + ); + } + + #[test] + fn regression_sequence_raw_names_produce_expected_runtime_labels() { + // Operator regression scenario: the grouped block must show exactly these + // labels at runtime — not just in the pure-module test that hardcodes the + // display_name. This proves the controller maps the raw wire names the same + // way the timeline expects. + assert_eq!( + friendly_tool_name("mcp__brave-search__brave_web_search"), + "Web search" + ); + assert_eq!( + friendly_tool_name("mcp__loctree-mcp__context"), + "Loctree context" + ); + assert_eq!( + friendly_tool_name("mcp__aicx-mcp__aicx_intents"), + "AICX intents" + ); + } + #[test] fn tool_running_status_is_human_readable() { assert_eq!(tool_running_status("Web search"), "Searching web…"); From 1ee7a2e0d8c2b8c0339ab8a1f95fa265e319577d Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Thu, 25 Jun 2026 20:26:53 -0700 Subject: [PATCH 008/385] fix(attachments): deliver images as real vision input on agent path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent send path called `session.send(text, Vec::new(), ...)`, so image attachments reached the model only as text file paths under the `ATTACHMENTS (image paths)` marker — never as vision input. The legacy fallback (`ai_formatting`) already parsed the marker and encoded base64, so images "worked" only when the request fell back to legacy. Fix: the agent path now strips the marker and loads each listed image into an `ImageAttachment` via a shared `core::attachment` contract, so `AgentSession::send` emits real `input_image` blocks. Clipboard, drag&drop and file-picker are source-agnostic and all go through this path identically. - core/attachment.rs: canonical marker parsing + MIME + image loading (`parse_image_attachment_block`, `image_media_type`, `load_image_for_vision`, `MAX_VISION_IMAGE_BYTES`) with unit tests. - core/llm/ai_formatting.rs: legacy path delegates to the same contract (DRY). - app/ui/voice_chat/api/send.rs: only claim "will be sent as vision input" for vision-supported images within the size cap; oversized/unsupported get an honest message and are not listed. - app/controller/helpers.rs: load images on the agent path, log forwarded count, surface dropped images to the user instead of silently dropping. - docs/architecture/attachment-vision-pipeline.md: developer note. Verified e2e: model describes screenshot contents via both clipboard and file-picker; logs show "forwarding N image(s) as vision input" on the agent path with no legacy fallback. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/controller/helpers.rs | 145 +++++++++++++- app/ui/voice_chat/api/send.rs | 25 ++- core/attachment.rs | 182 ++++++++++++++++++ core/llm/ai_formatting.rs | 85 +------- .../attachment-vision-pipeline.md | 53 +++++ 5 files changed, 400 insertions(+), 90 deletions(-) create mode 100644 docs/architecture/attachment-vision-pipeline.md diff --git a/app/controller/helpers.rs b/app/controller/helpers.rs index c6248ff4..c1ec8c4d 100644 --- a/app/controller/helpers.rs +++ b/app/controller/helpers.rs @@ -8,13 +8,13 @@ use std::sync::Mutex as StdMutex; use std::sync::OnceLock; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use tokio::sync::{Mutex as TokioMutex, RwLock, mpsc}; -use tracing::{debug, warn}; +use tracing::{debug, info, warn}; use crate::config::{Config, default_assistive_model}; use anyhow::{Context, Result}; use codescribe_core::agent::{ - AgentSession, AgentUiEvent, ContentBlock, Message, Role, StreamOptions, Thread, ThreadMessage, - ThreadStore, ToolRegistry, + AgentSession, AgentUiEvent, ContentBlock, ImageAttachment, Message, Role, StreamOptions, + Thread, ThreadMessage, ThreadStore, ToolRegistry, }; use serde_json::json; @@ -680,6 +680,70 @@ fn agent_send_error_is_transient(error: &anyhow::Error) -> bool { .any(|pattern| message.contains(pattern)) } +/// Maximum number of image attachments forwarded to the model per message. +/// Matches the legacy (`ai_formatting`) cap so both send paths behave alike. +const MAX_AGENT_VISION_IMAGES: usize = 4; + +/// Split an outgoing payload into its visible text and the loaded image +/// attachments referenced by the `ATTACHMENTS (image paths)` marker. +/// +/// This is the fix for the attachment pipeline: the voice-chat send path appends +/// image paths to the payload as *text* (`build_attachments_block`). Without this +/// step the agent path forwarded them as plain text and the model never received +/// real vision input. Here we strip the marker block from the text and load each +/// image as bytes so `AgentSession::send` can emit proper `input_image` blocks. +/// +/// Returns `(cleaned_text, loaded_images, dropped_names)`. `dropped_names` lists +/// images that could not be forwarded (missing/unreadable/too large) so the +/// caller can surface a visible attachment error instead of silently continuing. +fn build_image_attachments_from_text(text: &str) -> (String, Vec, Vec) { + let (cleaned, mut paths) = codescribe_core::attachment::parse_image_attachment_block(text); + + if paths.is_empty() { + return (cleaned, Vec::new(), Vec::new()); + } + + let mut dropped: Vec = Vec::new(); + + if paths.len() > MAX_AGENT_VISION_IMAGES { + for extra in &paths[MAX_AGENT_VISION_IMAGES..] { + dropped.push(file_label(extra)); + } + warn!( + "Too many image attachments ({}); forwarding first {} as vision input", + paths.len(), + MAX_AGENT_VISION_IMAGES + ); + paths.truncate(MAX_AGENT_VISION_IMAGES); + } + + let mut attachments = Vec::with_capacity(paths.len()); + for path in &paths { + match codescribe_core::attachment::load_image_for_vision( + path, + codescribe_core::attachment::MAX_VISION_IMAGE_BYTES, + ) { + Some((data, media_type)) => attachments.push(ImageAttachment { data, media_type }), + None => { + warn!( + "Dropping image attachment (unsupported, unreadable, or too large): {}", + path.display() + ); + dropped.push(file_label(path)); + } + } + } + + (cleaned, attachments, dropped) +} + +/// Short, user-facing label for an attachment path (file name, path fallback). +fn file_label(path: &std::path::Path) -> String { + path.file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| path.to_string_lossy().to_string()) +} + async fn run_agent_send_path( runtime_state: &mut AgentRuntimeState, runtime_generation: u64, @@ -706,7 +770,22 @@ async fn run_agent_send_path( let send_result = { let (session, ui_rx) = (&mut runtime.session, &mut runtime.ui_rx); - let send_future = session.send(text, Vec::new(), &stream_options); + let (user_text, image_attachments, dropped_images) = + build_image_attachments_from_text(&text); + if !image_attachments.is_empty() { + info!( + "Agent send: forwarding {} image(s) as vision input", + image_attachments.len() + ); + } + if !dropped_images.is_empty() { + crate::ui::voice_chat::add_voice_chat_system_message(&format!( + "⚠️ Could not attach {} image(s) as vision input: {}", + dropped_images.len(), + dropped_images.join(", ") + )); + } + let send_future = session.send(user_text, image_attachments, &stream_options); tokio::pin!(send_future); let result = loop { @@ -1644,4 +1723,62 @@ mod tests { assert!(runtime_state.runtime_degraded); assert!(runtime_state.runtime.is_none()); } + + #[test] + fn test_build_image_attachments_passthrough_without_marker() { + let text = "plain message, no attachments"; + let (cleaned, images, dropped) = build_image_attachments_from_text(text); + assert_eq!(cleaned, text); + assert!(images.is_empty()); + assert!(dropped.is_empty()); + } + + #[test] + fn test_build_image_attachments_loads_real_image_and_reports_dropped() { + let dir = std::env::temp_dir().join(format!("cs_helpers_vision_{}", std::process::id())); + let _ = std::fs::create_dir_all(&dir); + let img = dir.join("shot.png"); + std::fs::write(&img, b"\x89PNG\r\n\x1a\nfake").unwrap(); + let missing = dir.join("gone.png"); + + let text = format!( + "describe these\n\n---\nATTACHMENTS (image paths)\n- {}\n- {}\n", + img.display(), + missing.display() + ); + let (cleaned, images, dropped) = build_image_attachments_from_text(&text); + + // Marker block and raw paths are gone from the model-visible text. + assert!(!cleaned.contains("ATTACHMENTS (image paths)")); + assert!(!cleaned.contains(&img.display().to_string())); + assert!(cleaned.contains("describe these")); + + // Only the readable image becomes a real vision attachment; the missing + // one is reported as dropped (visible error), never forwarded as text. + assert_eq!(images.len(), 1); + assert_eq!(images[0].media_type, "image/png"); + assert!(!images[0].data.is_empty()); + assert_eq!(dropped, vec!["gone.png".to_string()]); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn test_build_image_attachments_caps_and_reports_overflow() { + let dir = std::env::temp_dir().join(format!("cs_helpers_cap_{}", std::process::id())); + let _ = std::fs::create_dir_all(&dir); + let mut lines = String::from("multi\n\nATTACHMENTS (image paths)\n"); + for i in 0..(MAX_AGENT_VISION_IMAGES + 2) { + let p = dir.join(format!("img{i}.png")); + std::fs::write(&p, b"\x89PNG\r\n\x1a\nfake").unwrap(); + lines.push_str(&format!("- {}\n", p.display())); + } + let (_cleaned, images, dropped) = build_image_attachments_from_text(&lines); + + // Cap honored, overflow surfaced (not silently dropped). + assert_eq!(images.len(), MAX_AGENT_VISION_IMAGES); + assert_eq!(dropped.len(), 2); + + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/app/ui/voice_chat/api/send.rs b/app/ui/voice_chat/api/send.rs index 5b9f290f..0e6cd820 100644 --- a/app/ui/voice_chat/api/send.rs +++ b/app/ui/voice_chat/api/send.rs @@ -1098,15 +1098,24 @@ Tools (optional): `brew install poppler ocrmypdf tesseract-lang`.)\n", let _ = (&mut f).take(MAX_FILE_BYTES as u64).read_to_end(&mut buf); let Ok(mut s) = String::from_utf8(buf) else { - let is_image = matches!( - ext.as_str(), - "png" | "jpg" | "jpeg" | "webp" | "gif" | "bmp" | "tif" | "tiff" - ); - if is_image { - out.push_str("(image detected; will be sent as vision input)\n"); - image_paths.push(display.to_string()); + // Only claim "will be sent as vision input" for images the model can + // actually receive: a vision-supported format within the byte cap. + // Anything else gets an honest message and is NOT added to the image + // paths list, so the agent send path never sees an unsendable image. + if codescribe_core::attachment::image_media_type(path).is_some() { + let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + if size > codescribe_core::attachment::MAX_VISION_IMAGE_BYTES { + out.push_str(&format!( + "(image too large for vision input: {} bytes > {} max; not sent)\n", + size, + codescribe_core::attachment::MAX_VISION_IMAGE_BYTES + )); + } else { + out.push_str("(image detected; will be sent as vision input)\n"); + image_paths.push(display.to_string()); + } } else { - out.push_str("(skipped: not UTF-8 text)\n"); + out.push_str("(skipped: unsupported image format or not UTF-8 text)\n"); } continue; }; diff --git a/core/attachment.rs b/core/attachment.rs index 72dd6b07..e1c00d5f 100644 --- a/core/attachment.rs +++ b/core/attachment.rs @@ -194,6 +194,118 @@ fn kind_from_extension(path: &Path) -> AttachmentKind { } } +// ═══════════════════════════════════════════════════════════ +// Vision attachment parsing (shared by agent + legacy send paths) +// ═══════════════════════════════════════════════════════════ + +/// Marker line emitted by `build_attachments_block` that introduces the list of +/// image file paths appended to a chat payload as text. +pub const IMAGE_PATHS_MARKER: &str = "ATTACHMENTS (image paths)"; + +/// Default per-image byte cap honored when loading images for vision input. +pub const MAX_VISION_IMAGE_BYTES: u64 = 8 * 1024 * 1024; + +/// MIME media type for a vision-supported image, inferred from extension. +/// +/// Returns `None` for extensions the model APIs do not accept as image input +/// (e.g. `svg`, `heic`, `raw`), even though [`AttachmentKind::Image`] classifies +/// them as images for UI purposes. +pub fn image_media_type(path: &Path) -> Option<&'static str> { + let ext = path + .extension() + .and_then(|e| e.to_str()) + .map(|e| e.to_ascii_lowercase()) + .unwrap_or_default(); + match ext.as_str() { + "png" => Some("image/png"), + "jpg" | "jpeg" => Some("image/jpeg"), + "webp" => Some("image/webp"), + "gif" => Some("image/gif"), + "bmp" => Some("image/bmp"), + "tif" | "tiff" => Some("image/tiff"), + _ => None, + } +} + +/// Split a chat payload into its visible text and the image paths listed under +/// the `ATTACHMENTS (image paths)` marker appended by `build_attachments_block`. +/// +/// The marker block (and a dangling `---`/`—` separator directly above it) is +/// removed from the returned text so the model never sees raw file paths where a +/// real vision input belongs. The original payload is returned verbatim in +/// `.0` when no marker is present, so non-attachment messages pass through +/// unchanged. +pub fn parse_image_attachment_block(text: &str) -> (String, Vec) { + let mut out_lines: Vec = Vec::new(); + let mut image_paths: Vec = Vec::new(); + let mut in_block = false; + + for line in text.lines() { + let trimmed = line.trim(); + + if trimmed == IMAGE_PATHS_MARKER { + // Drop a preceding separator if present to avoid leaving a dangling "---". + if out_lines + .last() + .is_some_and(|l| l.trim() == "---" || l.trim() == "—") + { + out_lines.pop(); + } + in_block = true; + continue; + } + + if in_block { + if trimmed.is_empty() { + in_block = false; + continue; + } + if let Some(rest) = trimmed.strip_prefix("- ") { + let p = rest.trim(); + if !p.is_empty() { + image_paths.push(PathBuf::from(p)); + } + continue; + } + // Unexpected line → end block, keep the line. + in_block = false; + out_lines.push(line.to_string()); + continue; + } + + out_lines.push(line.to_string()); + } + + (out_lines.join("\n"), image_paths) +} + +/// Load an image file as `(bytes, media_type)` for vision input. +/// +/// Returns `None` (with a warning) when the extension is not a vision-supported +/// image, the file is unreadable, or it exceeds `max_bytes`. +pub fn load_image_for_vision(path: &Path, max_bytes: u64) -> Option<(Vec, String)> { + let media_type = image_media_type(path)?; + + let meta = std::fs::metadata(path).ok()?; + if meta.len() > max_bytes { + warn!( + "Skipping image attachment (too large, {} bytes > {} max): {}", + meta.len(), + max_bytes, + path.display() + ); + return None; + } + + match std::fs::read(path) { + Ok(bytes) => Some((bytes, media_type.to_string())), + Err(e) => { + warn!("Failed to read image attachment {}: {}", path.display(), e); + None + } + } +} + // ═══════════════════════════════════════════════════════════ // Attachment Store // ═══════════════════════════════════════════════════════════ @@ -449,6 +561,76 @@ mod tests { assert_eq!(sanitize_filename(&many_dots), "abc.txt"); } + #[test] + fn test_image_media_type() { + assert_eq!(image_media_type(Path::new("a.png")), Some("image/png")); + assert_eq!(image_media_type(Path::new("a.JPG")), Some("image/jpeg")); + assert_eq!(image_media_type(Path::new("a.jpeg")), Some("image/jpeg")); + assert_eq!(image_media_type(Path::new("a.webp")), Some("image/webp")); + assert_eq!(image_media_type(Path::new("a.tiff")), Some("image/tiff")); + // Not accepted as vision input despite being "images" for the UI. + assert_eq!(image_media_type(Path::new("a.svg")), None); + assert_eq!(image_media_type(Path::new("a.heic")), None); + assert_eq!(image_media_type(Path::new("a.txt")), None); + assert_eq!(image_media_type(Path::new("noext")), None); + } + + #[test] + fn test_parse_image_attachment_block_passthrough() { + // No marker → text returned unchanged, no paths. + let text = "just a normal message\nwith two lines"; + let (cleaned, paths) = parse_image_attachment_block(text); + assert_eq!(cleaned, text); + assert!(paths.is_empty()); + } + + #[test] + fn test_parse_image_attachment_block_extracts_paths() { + let text = "Look at these\n\n---\nATTACHMENTS (image paths)\n- /tmp/a.png\n- /tmp/b.jpg\n"; + let (cleaned, paths) = parse_image_attachment_block(text); + assert_eq!( + paths, + vec![PathBuf::from("/tmp/a.png"), PathBuf::from("/tmp/b.jpg")] + ); + // The marker block and the dangling separator are stripped. + assert!(!cleaned.contains(IMAGE_PATHS_MARKER)); + assert!(!cleaned.contains("/tmp/a.png")); + assert!(cleaned.contains("Look at these")); + assert!(!cleaned.trim_end().ends_with("---")); + } + + #[test] + fn test_parse_image_attachment_block_stops_at_blank_line() { + let text = "msg\nATTACHMENTS (image paths)\n- /tmp/a.png\n\ntrailing text kept"; + let (cleaned, paths) = parse_image_attachment_block(text); + assert_eq!(paths, vec![PathBuf::from("/tmp/a.png")]); + assert!(cleaned.contains("trailing text kept")); + } + + #[test] + fn test_load_image_for_vision_rejects_oversize_and_nonimage() { + let dir = std::env::temp_dir().join(format!("cs_vision_test_{}", std::process::id())); + let _ = std::fs::create_dir_all(&dir); + + let png = dir.join("small.png"); + std::fs::write(&png, b"\x89PNG\r\n\x1a\nfake").unwrap(); + let loaded = load_image_for_vision(&png, MAX_VISION_IMAGE_BYTES); + assert!(loaded.is_some()); + let (bytes, mt) = loaded.unwrap(); + assert_eq!(mt, "image/png"); + assert!(!bytes.is_empty()); + + // Oversize → rejected. + assert!(load_image_for_vision(&png, 2).is_none()); + + // Non-vision extension → rejected even if file exists. + let txt = dir.join("note.txt"); + std::fs::write(&txt, b"hello").unwrap(); + assert!(load_image_for_vision(&txt, MAX_VISION_IMAGE_BYTES).is_none()); + + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn test_paths_extraction() { let attachments = vec![ diff --git a/core/llm/ai_formatting.rs b/core/llm/ai_formatting.rs index 8d107570..107768fc 100644 --- a/core/llm/ai_formatting.rs +++ b/core/llm/ai_formatting.rs @@ -22,7 +22,6 @@ use anyhow::{Context, Result}; use reqwest::Client; use serde::{Deserialize, Serialize}; use std::env; -use std::path::PathBuf; use std::sync::{Arc, OnceLock, RwLock}; use std::time::Duration; use tracing::{debug, info, trace, warn}; @@ -448,84 +447,13 @@ enum InputContent { Image { image_url: String }, } -fn image_mime_from_path(path: &std::path::Path) -> Option<&'static str> { - let ext = path - .extension() - .map(|e| e.to_string_lossy().to_ascii_lowercase()) - .unwrap_or_default(); - match ext.as_str() { - "png" => Some("image/png"), - "jpg" | "jpeg" => Some("image/jpeg"), - "webp" => Some("image/webp"), - "gif" => Some("image/gif"), - "bmp" => Some("image/bmp"), - "tif" | "tiff" => Some("image/tiff"), - _ => None, - } -} - -fn strip_image_attachments(user_message: &str) -> (String, Vec) { - let mut out_lines: Vec = Vec::new(); - let mut image_paths: Vec = Vec::new(); - let mut in_block = false; - - for line in user_message.lines() { - let trimmed = line.trim(); - - if trimmed == "ATTACHMENTS (image paths)" { - // Drop a preceding separator if present to avoid leaving a dangling "---". - if out_lines - .last() - .is_some_and(|l| l.trim() == "---" || l.trim() == "—") - { - out_lines.pop(); - } - in_block = true; - continue; - } - - if in_block { - if trimmed.is_empty() { - in_block = false; - continue; - } - if let Some(rest) = trimmed.strip_prefix("- ") { - let p = rest.trim(); - if !p.is_empty() { - image_paths.push(PathBuf::from(p)); - } - continue; - } - // Unexpected line → end block, keep the line. - in_block = false; - out_lines.push(line.to_string()); - continue; - } - - out_lines.push(line.to_string()); - } - - (out_lines.join("\n"), image_paths) -} - -fn encode_image_as_data_url(path: &PathBuf) -> Option { +fn encode_image_as_data_url(path: &std::path::Path) -> Option { use base64::{Engine, engine::general_purpose::STANDARD as BASE64}; - const MAX_IMAGE_BYTES: u64 = 8 * 1024 * 1024; // 8MB per image - - let mime = image_mime_from_path(path)?; - - let meta = std::fs::metadata(path).ok()?; - if meta.len() > MAX_IMAGE_BYTES { - warn!( - "Skipping image attachment (too large, {} bytes): {}", - meta.len(), - path.display() - ); - return None; - } - - let bytes = std::fs::read(path).ok()?; + // Marker parsing, MIME mapping and the size cap are shared with the agent + // send path via `crate::attachment` so both routes honor one contract. + let (bytes, mime) = + crate::attachment::load_image_for_vision(path, crate::attachment::MAX_VISION_IMAGE_BYTES)?; let b64 = BASE64.encode(bytes); Some(format!("data:{mime};base64,{b64}")) } @@ -533,7 +461,8 @@ fn encode_image_as_data_url(path: &PathBuf) -> Option { fn build_responses_user_content(user_message: &str) -> Vec { const MAX_IMAGES: usize = 4; - let (mut cleaned, mut image_paths) = strip_image_attachments(user_message); + let (mut cleaned, mut image_paths) = + crate::attachment::parse_image_attachment_block(user_message); if image_paths.len() > MAX_IMAGES { warn!( "Too many image attachments ({}); keeping first {}", diff --git a/docs/architecture/attachment-vision-pipeline.md b/docs/architecture/attachment-vision-pipeline.md new file mode 100644 index 00000000..35dec6b0 --- /dev/null +++ b/docs/architecture/attachment-vision-pipeline.md @@ -0,0 +1,53 @@ +# Attachment & vision input pipeline + +How image attachments travel from the chat UI to the model as real vision input. + +## Internal representation + +- UI attachments are `codescribe_core::attachment::Attachment` (path + kind + + source + display name). Clipboard, drag&drop and file-picker all produce the + same `Attachment` and land in `VoiceChatOverlayState::attachments` — the send + path is **source-agnostic**. +- The model-facing message is `codescribe_core::agent::Message` with a + `Vec`. Images are `ContentBlock::Image { data, media_type }` + (raw bytes + MIME), converted to provider JSON in + `app/agent/openai_provider.rs` as an `input_image` data URI. + +## Flow + +1. On send, `build_attachments_block` (`app/ui/voice_chat/api/send.rs`) inlines + text files and appends image **paths** under the + `ATTACHMENTS (image paths)` marker. It only lists an image (and only then + claims "will be sent as vision input") when the file is a vision-supported + format within `attachment::MAX_VISION_IMAGE_BYTES`. Oversized / unsupported + images get an honest message and are **not** listed. +2. The payload is a single `String` (the send callback is `Fn(String)`). +3. Agent path: `run_agent_send_path` (`app/controller/helpers.rs`) calls + `build_image_attachments_from_text`, which strips the marker block from the + text and loads each listed image into an `ImageAttachment` via + `attachment::load_image_for_vision`. These go to `AgentSession::send` as real + image blocks. Images that fail to load are reported to the user (not silently + dropped) and the cap is `MAX_AGENT_VISION_IMAGES`. +4. Legacy path: `ai_formatting::build_responses_user_content` uses the same + `attachment::parse_image_attachment_block` + `load_image_for_vision` contract, + so both routes behave identically. + +## Image support / capability + +The assistive backend is assumed to be vision-capable (OpenAI Responses +`input_image`). There is no per-model capability probe — if a future text-only +backend is introduced, gate image forwarding here and surface a clear +"backend does not support image analysis" message instead of attaching. + +## Marker contract (single source of truth) + +`core/attachment.rs` owns the marker constant and parsing: +`IMAGE_PATHS_MARKER`, `parse_image_attachment_block`, `image_media_type`, +`load_image_for_vision`, `MAX_VISION_IMAGE_BYTES`. Both send paths depend on it; +change the marker in one place only. + +## Adding another image-capable provider + +Implement `AgentProvider::build_image_block` for the provider (see +`openai_provider.rs`) to emit its native image block; the rest of the pipeline +(`ContentBlock::Image`) is provider-agnostic. From 0043ab461efe5d2b2d20091c0d125a4822072b30 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Thu, 25 Jun 2026 20:41:01 -0700 Subject: [PATCH 009/385] feat(attachments): auto-clear after send with re-attach previous Attachments stayed in the input chip strip after sending; only a fingerprint prevented resending the identical set. Adding another image then resent the whole accumulated set, so a follow-up "send 1 image" silently shipped the earlier one too. Now a successful send clears the chip strip and stashes the sent set in `last_sent_attachments`. The attach menu gains "Re-attach previous (N)" so the user can resend the same files without re-picking them. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/ui/voice_chat/api/send.rs | 19 +++++++++++ app/ui/voice_chat/handlers/attachments.rs | 25 ++++++++++++++ app/ui/voice_chat/handlers/classes.rs | 4 +++ app/ui/voice_chat/handlers/menus.rs | 41 ++++++++++++++++------- app/ui/voice_chat/state.rs | 4 +++ 5 files changed, 80 insertions(+), 13 deletions(-) diff --git a/app/ui/voice_chat/api/send.rs b/app/ui/voice_chat/api/send.rs index 0e6cd820..c03a091b 100644 --- a/app/ui/voice_chat/api/send.rs +++ b/app/ui/voice_chat/api/send.rs @@ -153,6 +153,21 @@ pub fn clear_voice_chat_text_impl() { update_attach_button_ui(btn_ptr, 0, Vec::new()); } +/// After a send consumes the current attachments, clear them from the input bar +/// but stash a copy in `last_sent_attachments` so the user can re-attach them via +/// the attach menu instead of re-picking each file. +fn clear_attachments_after_send_locked(state: &mut VoiceChatOverlayState) { + if state.attachments.is_empty() { + return; + } + state.last_sent_attachments = state.attachments.clone(); + state.attachments.clear(); + state.attachments_last_sent = None; + let btn_ptr = state.agent_attach_button; + render_attachment_chips_locked(state); + update_attach_button_ui(btn_ptr, 0, Vec::new()); +} + /// Send the draft message (called from handlers) pub fn send_draft_message_impl() { let callback = { @@ -195,6 +210,8 @@ pub fn send_draft_message_impl() { mode: Some(mode), is_pending_followup: false, }); + // Sent once: clear the chip strip, keep a copy for re-attach. + clear_attachments_after_send_locked(&mut state); } push_prompt_history_locked(&mut state, &draft); follow_latest_after_manual_send_locked(&mut state); @@ -277,6 +294,8 @@ pub fn commit_last_user_message_impl() { mode: Some(mode), is_pending_followup: false, }); + // Sent once: clear the chip strip, keep a copy for re-attach. + clear_attachments_after_send_locked(&mut state); } state.is_sending = true; update_chat_view_with_state(&mut state, true); diff --git a/app/ui/voice_chat/handlers/attachments.rs b/app/ui/voice_chat/handlers/attachments.rs index 025f676a..faca731d 100644 --- a/app/ui/voice_chat/handlers/attachments.rs +++ b/app/ui/voice_chat/handlers/attachments.rs @@ -252,6 +252,31 @@ pub extern "C" fn on_attach_clear(_this: &Object, _cmd: Sel, _sender: Id) { }; update_attach_button_ui(btn_ptr, count, names); } + +/// Restore the attachments cleared by the previous send so the user can resend +/// them without re-picking each file. Duplicates (same path) are skipped. +pub extern "C" fn on_attach_reattach(_this: &Object, _cmd: Sel, _sender: Id) { + let (btn_ptr, count, names) = { + let mut state = OVERLAY_STATE.lock().unwrap_or_else(|e| e.into_inner()); + if state.last_sent_attachments.is_empty() { + return; + } + for attachment in state.last_sent_attachments.clone() { + if !state.attachments.iter().any(|a| a.path == attachment.path) { + state.attachments.push(attachment); + } + } + state.attachments_last_sent = None; + render_attachment_chips(&mut state); + let names: Vec = state + .attachments + .iter() + .map(|a| a.display_name.clone()) + .collect(); + (state.agent_attach_button, state.attachments.len(), names) + }; + update_attach_button_ui(btn_ptr, count, names); +} // ═══════════════════════════════════════════════════════════ // Attachment Chip Handlers // ═══════════════════════════════════════════════════════════ diff --git a/app/ui/voice_chat/handlers/classes.rs b/app/ui/voice_chat/handlers/classes.rs index 1322e062..41f19f23 100644 --- a/app/ui/voice_chat/handlers/classes.rs +++ b/app/ui/voice_chat/handlers/classes.rs @@ -117,6 +117,10 @@ pub fn action_handler_class() -> *const Class { sel!(onAttachClear:), on_attach_clear as extern "C" fn(&Object, Sel, Id), ); + decl.add_method( + sel!(onAttachReattach:), + on_attach_reattach as extern "C" fn(&Object, Sel, Id), + ); decl.add_method( sel!(onTabDrawer:), on_tab_drawer as extern "C" fn(&Object, Sel, Id), diff --git a/app/ui/voice_chat/handlers/menus.rs b/app/ui/voice_chat/handlers/menus.rs index c6c76034..ddefbd1d 100644 --- a/app/ui/voice_chat/handlers/menus.rs +++ b/app/ui/voice_chat/handlers/menus.rs @@ -44,24 +44,39 @@ pub extern "C" fn on_attach_menu(this: &Object, _cmd: Sel, sender: Id) { let _: () = msg_send![url, setTarget: target]; let _: () = msg_send![menu, addItem: url]; - let count = { + let (count, reattach_count) = { let state = OVERLAY_STATE.lock().unwrap_or_else(|e| e.into_inner()); - state.attachments.len() + (state.attachments.len(), state.last_sent_attachments.len()) }; - if count > 0 { + if count > 0 || reattach_count > 0 { let sep: Id = msg_send![ns_menu_item, separatorItem]; let _: () = msg_send![menu, addItem: sep]; - let clear_title = format!("Clear attachments ({})", count); - let clear: Id = msg_send![ns_menu_item, alloc]; - let clear: Id = msg_send![ - clear, - initWithTitle: ns_string(&clear_title) - action: sel!(onAttachClear:) - keyEquivalent: ns_string("") - ]; - let _: () = msg_send![clear, setTarget: target]; - let _: () = msg_send![menu, addItem: clear]; + if reattach_count > 0 { + let reattach_title = format!("Re-attach previous ({})", reattach_count); + let reattach: Id = msg_send![ns_menu_item, alloc]; + let reattach: Id = msg_send![ + reattach, + initWithTitle: ns_string(&reattach_title) + action: sel!(onAttachReattach:) + keyEquivalent: ns_string("") + ]; + let _: () = msg_send![reattach, setTarget: target]; + let _: () = msg_send![menu, addItem: reattach]; + } + + if count > 0 { + let clear_title = format!("Clear attachments ({})", count); + let clear: Id = msg_send![ns_menu_item, alloc]; + let clear: Id = msg_send![ + clear, + initWithTitle: ns_string(&clear_title) + action: sel!(onAttachClear:) + keyEquivalent: ns_string("") + ]; + let _: () = msg_send![clear, setTarget: target]; + let _: () = msg_send![menu, addItem: clear]; + } } // Pop up anchored at the button. diff --git a/app/ui/voice_chat/state.rs b/app/ui/voice_chat/state.rs index e1f3e191..efedd6e4 100644 --- a/app/ui/voice_chat/state.rs +++ b/app/ui/voice_chat/state.rs @@ -171,6 +171,9 @@ pub struct VoiceChatOverlayState { pub attachments: Vec, /// Fingerprint of the last attachment set sent to the assistant. pub attachments_last_sent: Option, + /// Attachments cleared from the input after the most recent send, kept so the + /// user can re-attach them ("Re-attach previous") instead of re-picking. + pub last_sent_attachments: Vec, /// Chip strip scroll view (horizontal list of attachment chips above input bar). pub attachment_chip_strip: Option, @@ -274,6 +277,7 @@ impl Default for VoiceChatOverlayState { agent_latest_button: None, attachments: Vec::new(), attachments_last_sent: None, + last_sent_attachments: Vec::new(), attachment_chip_strip: None, active_tab: Tab::Drawer, pending_tab: None, From 3ff39f805397fbf1629b05822386fcb1180dc039 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Thu, 25 Jun 2026 20:41:16 -0700 Subject: [PATCH 010/385] fix(ui): guard bubble text against CoreText main-thread spin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pasted/streamed payload with a very long unbroken run (a 12.7k-char base64 line) froze the app: `NSTextFieldCell cellSizeForBounds:` → `CTLineCreateWithAttributedString` spun in OpenType shaping on the main thread. `cap_bubble_display_text` now caps total length and injects breaks into runs longer than `MAX_UNBROKEN_RUN`, so the typesetter wraps instead of spinning. Applied in both the render (`bubble_text_for_message`) and measure (`display_text_for_message`) paths. Normal messages pass through unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/ui/voice_chat/api/messages.rs | 115 ++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 4 deletions(-) diff --git a/app/ui/voice_chat/api/messages.rs b/app/ui/voice_chat/api/messages.rs index cc5cdcec..58380ee3 100644 --- a/app/ui/voice_chat/api/messages.rs +++ b/app/ui/voice_chat/api/messages.rs @@ -532,8 +532,70 @@ pub fn finalize_streaming_reasoning(state: &mut VoiceChatOverlayState) { state.active_reasoning_stream_index = None; } +/// Hard cap on characters rendered/measured per bubble. A backstop against +/// pathological payloads (e.g. a pasted base64 blob) freezing the UI. +const MAX_BUBBLE_DISPLAY_CHARS: usize = 20_000; + +/// Longest run of non-whitespace characters allowed before we inject a break. +/// CoreText's OpenType shaping (`CTLineCreateWithAttributedString`) spins on the +/// main thread for very long unbroken runs (a 12.7k-char base64 line hung the +/// app), so we break them so the typesetter can wrap instead. +const MAX_UNBROKEN_RUN: usize = 200; + +fn has_long_unbroken_run(text: &str, max_run: usize) -> bool { + let mut run = 0usize; + for ch in text.chars() { + if ch.is_whitespace() { + run = 0; + } else { + run += 1; + if run > max_run { + return true; + } + } + } + false +} + +fn break_long_runs(text: &str, max_run: usize) -> String { + let mut out = String::with_capacity(text.len() + text.len() / max_run.max(1) + 1); + let mut run = 0usize; + for ch in text.chars() { + if ch.is_whitespace() { + run = 0; + out.push(ch); + } else { + if run >= max_run { + out.push('\n'); + run = 0; + } + out.push(ch); + run += 1; + } + } + out +} + +/// Make any bubble text safe to render: cap total length and break unbroken runs +/// so CoreText never spins on the main thread. Normal messages pass through +/// unchanged. +pub fn cap_bubble_display_text(text: &str) -> String { + let capped = if text.chars().count() > MAX_BUBBLE_DISPLAY_CHARS { + let mut t: String = text.chars().take(MAX_BUBBLE_DISPLAY_CHARS).collect(); + t.push_str("\n… (truncated for display; full text preserved in the thread)"); + t + } else { + text.to_string() + }; + if has_long_unbroken_run(&capped, MAX_UNBROKEN_RUN) { + break_long_runs(&capped, MAX_UNBROKEN_RUN) + } else { + capped + } +} + pub fn display_text_for_message(message: &ChatMessage) -> String { - if message.role == ChatRole::Reasoning && message.is_collapsed { + let raw = if message.role == ChatRole::Reasoning && message.is_collapsed { reasoning_summary_header(message) } else if message.is_streaming && message.text.is_empty() { "• • •".to_string() @@ -541,15 +603,17 @@ pub fn display_text_for_message(message: &ChatMessage) -> String { format!("{} …", message.text) } else { message.text.clone() - } + }; + cap_bubble_display_text(&raw) } pub fn bubble_text_for_message(message: &ChatMessage) -> String { - if message.role == ChatRole::Reasoning && message.is_collapsed { + let raw = if message.role == ChatRole::Reasoning && message.is_collapsed { reasoning_summary_header(message) } else { message.text.clone() - } + }; + cap_bubble_display_text(&raw) } pub fn bubble_streaming_for_message(message: &ChatMessage) -> bool { @@ -1265,3 +1329,46 @@ pub fn update_chat_view_with_state(state: &mut VoiceChatOverlayState, scroll_to_ } } } + +#[cfg(test)] +mod bubble_display_tests { + use super::{ + MAX_BUBBLE_DISPLAY_CHARS, MAX_UNBROKEN_RUN, break_long_runs, cap_bubble_display_text, + has_long_unbroken_run, + }; + + #[test] + fn normal_text_passes_through_unchanged() { + let text = "A normal multi-line message.\nSecond line with spaces."; + assert_eq!(cap_bubble_display_text(text), text); + } + + #[test] + fn detects_and_breaks_long_unbroken_run() { + // Simulates a pasted base64 blob: one very long run with no whitespace. + let blob = "a".repeat(12_776); + assert!(has_long_unbroken_run(&blob, MAX_UNBROKEN_RUN)); + + let broken = break_long_runs(&blob, MAX_UNBROKEN_RUN); + // No remaining run exceeds the cap → CoreText can wrap instead of spinning. + assert!(!has_long_unbroken_run(&broken, MAX_UNBROKEN_RUN)); + // Content preserved (only newlines injected). + assert_eq!(broken.chars().filter(|c| *c == 'a').count(), 12_776); + } + + #[test] + fn caps_total_length() { + let huge = "x".repeat(MAX_BUBBLE_DISPLAY_CHARS * 2); + let capped = cap_bubble_display_text(&huge); + assert!(capped.contains("truncated for display")); + // Capped well below the original length. + assert!(capped.chars().count() < MAX_BUBBLE_DISPLAY_CHARS + 4_000); + } + + #[test] + fn whitespace_resets_run_counter() { + // Many short words separated by spaces must not be flagged. + let text = "word ".repeat(5_000); + assert!(!has_long_unbroken_run(&text, MAX_UNBROKEN_RUN)); + } +} From f3653273f024167c76857c082b165ee04ecbce93 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Thu, 25 Jun 2026 23:08:53 -0700 Subject: [PATCH 011/385] fix(attachments): raise per-message image cap from 4 to 16 The cap of 4 (inherited from legacy formatting) dropped the 5th image when comparing several variants (e.g. 5 wireframes), surfacing an honest "could not attach" notice but still leaving the image out. Vision-capable backends accept far more and each image is already size-capped individually, so raise the per-message cap to 16 on both the agent and legacy send paths. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/controller/helpers.rs | 6 ++++-- core/llm/ai_formatting.rs | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/app/controller/helpers.rs b/app/controller/helpers.rs index c1ec8c4d..2dee23ed 100644 --- a/app/controller/helpers.rs +++ b/app/controller/helpers.rs @@ -681,8 +681,10 @@ fn agent_send_error_is_transient(error: &anyhow::Error) -> bool { } /// Maximum number of image attachments forwarded to the model per message. -/// Matches the legacy (`ai_formatting`) cap so both send paths behave alike. -const MAX_AGENT_VISION_IMAGES: usize = 4; +/// Kept in sync with the legacy (`ai_formatting`) cap so both send paths behave +/// alike. Sized for real multi-image use (e.g. comparing several wireframes); +/// vision-capable backends accept far more, images are size-capped individually. +const MAX_AGENT_VISION_IMAGES: usize = 16; /// Split an outgoing payload into its visible text and the loaded image /// attachments referenced by the `ATTACHMENTS (image paths)` marker. diff --git a/core/llm/ai_formatting.rs b/core/llm/ai_formatting.rs index 107768fc..a30c1640 100644 --- a/core/llm/ai_formatting.rs +++ b/core/llm/ai_formatting.rs @@ -459,7 +459,8 @@ fn encode_image_as_data_url(path: &std::path::Path) -> Option { } fn build_responses_user_content(user_message: &str) -> Vec { - const MAX_IMAGES: usize = 4; + // Kept in sync with `MAX_AGENT_VISION_IMAGES` in the agent send path. + const MAX_IMAGES: usize = 16; let (mut cleaned, mut image_paths) = crate::attachment::parse_image_attachment_block(user_message); From 02f3d55fdb2a842c5c14eb4e8b8977b1c6badb06 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Thu, 25 Jun 2026 23:21:14 -0700 Subject: [PATCH 012/385] fix(attachments): don't send empty image blocks (thread restore 400) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After restoring a thread, images come back with empty bytes — the thread store persists them with `data_omitted` (no base64). On the next send the provider emitted `data:image/png;base64,` with no payload, and the API rejected the whole request: "empty base64-encoded bytes" (HTTP 400), blocking the conversation. Two guards: - core/attachment.rs: `load_image_for_vision` rejects 0-byte files. - openai_provider.rs `build_input_items`: skip `ContentBlock::Image` with empty data (restored-from-history) instead of emitting an empty image_url. Fresh attachments are unaffected; restored conversations no longer 400 (images from prior turns are simply not re-sent, since their bytes aren't persisted). Co-Authored-By: Claude Opus 4.8 (1M context) --- app/agent/openai_provider.rs | 12 +++++++++++- core/attachment.rs | 13 +++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/app/agent/openai_provider.rs b/app/agent/openai_provider.rs index e67307f8..dd3f957a 100644 --- a/app/agent/openai_provider.rs +++ b/app/agent/openai_provider.rs @@ -9,7 +9,7 @@ use reqwest::Client; use serde::Serialize; use serde_json::{Value, json}; use tokio::sync::{Mutex, mpsc}; -use tracing::info; +use tracing::{info, warn}; use codescribe_core::agent::{ AgentEvent, AgentProvider, ContentBlock, ImageAsset, Message, Role, StreamOptions, @@ -376,6 +376,16 @@ fn build_input_items(messages: &[Message]) -> Result> { } } ContentBlock::Image { data, media_type } => { + // Images restored from the thread store carry no bytes + // (persisted with `data_omitted`). Emitting an empty data URL + // makes the provider reject the whole request + // ("empty base64-encoded bytes"), so skip empty images. + if data.is_empty() { + warn!( + "Skipping image content block with no bytes (likely restored from history)" + ); + continue; + } content.push(json!({ "type": "input_image", "image_url": to_data_uri(data, media_type) diff --git a/core/attachment.rs b/core/attachment.rs index e1c00d5f..dfb92299 100644 --- a/core/attachment.rs +++ b/core/attachment.rs @@ -298,6 +298,13 @@ pub fn load_image_for_vision(path: &Path, max_bytes: u64) -> Option<(Vec, St } match std::fs::read(path) { + Ok(bytes) if bytes.is_empty() => { + // An empty/zero-byte file would encode to an empty base64 string, + // which providers reject ("empty base64-encoded bytes") and which + // fails the whole request. Drop it instead. + warn!("Skipping image attachment (empty file): {}", path.display()); + None + } Ok(bytes) => Some((bytes, media_type.to_string())), Err(e) => { warn!("Failed to read image attachment {}: {}", path.display(), e); @@ -628,6 +635,12 @@ mod tests { std::fs::write(&txt, b"hello").unwrap(); assert!(load_image_for_vision(&txt, MAX_VISION_IMAGE_BYTES).is_none()); + // Empty (0-byte) image → rejected: an empty base64 payload would make + // the provider reject the whole request. + let empty = dir.join("empty.png"); + std::fs::write(&empty, b"").unwrap(); + assert!(load_image_for_vision(&empty, MAX_VISION_IMAGE_BYTES).is_none()); + let _ = std::fs::remove_dir_all(&dir); } From 306e076fc800aaaf3fdbc7d608ad2cb201ae0902 Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Fri, 26 Jun 2026 10:45:27 -0700 Subject: [PATCH 013/385] [claude/vc-justdo] feat(assistive): Evidence Summary as default tool-activity block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per assistant turn the Tool Activity block now defaults to a semantic Evidence Summary that answers what the assistant checked, which sources were involved, the key result, and whether anything failed — instead of a mechanical per-call list. The technical per-tool list becomes the layer-2 detail view, reached by clicking the block; raw payloads stay debug-only. The summary is deterministic (no LLM): calls are grouped by SOURCE (the system behind the wire name, so two Loctree calls read as one "Loctree" row), real result text/counts are echoed, failures surface as warnings, and a single Key finding verdict is extracted. Raw mcp__ wire names never reach either rendered view. - core engine in tool_activity.rs: source_label/source_detail/ evidence_key_finding + ToolActivityGroup::evidence_summary (pure, unit-tested). friendly_tool_name label table (60bc679) left untouched. - refresh_tool_activity_message renders evidence_summary by default and the technical render() on expand (is_collapsed flips the two layers). - 7 new summary tests + reframed two state-level tests to the new default; existing pure grouping/render/header tests unchanged and green. Verified: cargo fmt --all --check clean; cargo clippy -p codescribe --all-targets -D warnings clean; cargo check --workspace clean; tool_activity (17) + voice_chat::api::tests (48) all pass. Authored-By: claude session_id: f12fdb12-6ed7-4560-b718-fab763657c8a date: 2026-06-26T10:52:00 CEST runtime: interactive --- app/ui/voice_chat/api/messages.rs | 32 ++- app/ui/voice_chat/api/tests.rs | 44 +++- app/ui/voice_chat/tool_activity.rs | 395 +++++++++++++++++++++++++++++ 3 files changed, 454 insertions(+), 17 deletions(-) diff --git a/app/ui/voice_chat/api/messages.rs b/app/ui/voice_chat/api/messages.rs index 382ed5be..5e4c36bf 100644 --- a/app/ui/voice_chat/api/messages.rs +++ b/app/ui/voice_chat/api/messages.rs @@ -221,8 +221,9 @@ pub fn ensure_tool_activity_block(state: &mut VoiceChatOverlayState) -> usize { role: ChatRole::ToolActivity, text: String::new(), is_streaming: false, - // Expanded by default: the per-tool lines are already compact, and - // failures stay visible without a click. Clicking collapses to header. + // Evidence Summary by default (`is_collapsed = false`): the operator sees + // what was checked and what mattered, with failures surfaced as warnings. + // Clicking expands to the technical per-tool list. is_collapsed: false, is_error: false, timestamp: SystemTime::now(), @@ -237,20 +238,27 @@ pub fn ensure_tool_activity_block(state: &mut VoiceChatOverlayState) -> usize { idx } -/// Recompute a Tool Activity block's cached `text` from its group, honoring the -/// block's collapsed state. The group is the source of truth; `text` is the -/// rendered cache the bubble layer reads. +/// Recompute a Tool Activity block's cached `text` from its group. The group is +/// the source of truth; `text` is the rendered cache the bubble layer reads. +/// +/// Layering: the **default** primary block is the semantic Evidence Summary +/// (`What I checked · …` — what was checked, which sources, the key result, +/// failures). Clicking the block (`is_collapsed = true`) expands it to the +/// **technical** per-tool list (`render`). The raw tool payload stays in the +/// debug log and never reaches either view. pub fn refresh_tool_activity_message(state: &mut VoiceChatOverlayState, idx: usize) { - let collapsed = state + let show_technical_details = state .messages .get(idx) .map(|msg| msg.is_collapsed) .unwrap_or(false); - let Some(text) = state - .tool_activity_groups - .get(&idx) - .map(|group| group.render(collapsed)) - else { + let Some(text) = state.tool_activity_groups.get(&idx).map(|group| { + if show_technical_details { + group.render(false) + } else { + group.evidence_summary() + } + }) else { return; }; if let Some(msg) = state.messages.get_mut(idx) { @@ -370,7 +378,7 @@ pub fn handle_message_bubble_click_from_recognizer(sender: Id) { update_chat_view_with_state(&mut state, false); } ChatRole::ToolActivity => { - // Toggle compact header-only ↔ expanded per-tool lines, then + // Toggle Evidence Summary (default) ↔ technical per-tool list, then // re-render the cached block text from its group. message.is_collapsed = !message.is_collapsed; refresh_tool_activity_message(&mut state, index); diff --git a/app/ui/voice_chat/api/tests.rs b/app/ui/voice_chat/api/tests.rs index ffd4f79a..2010817b 100644 --- a/app/ui/voice_chat/api/tests.rs +++ b/app/ui/voice_chat/api/tests.rs @@ -1352,8 +1352,28 @@ fn turn_with_three_tools_renders_one_grouped_block() { .filter(|m| m.role == ChatRole::ToolActivity) .collect(); assert_eq!(blocks.len(), 1, "all turn tools collapse into one block"); + // Default primary view is the semantic Evidence Summary: what was checked, + // which sources, the key result, with failures surfaced as warnings. assert_eq!( blocks[0].text, + "What I checked · 3 tools · 1 warning\n\ + - Web search: 10 results.\n\ + - Loctree: scanned code surfaces.\n\ + - AICX: failed — empty index.\n\ + Key finding: AICX check failed: empty index." + ); + + // Expanding the block (a click flips `is_collapsed`) reveals the technical + // per-tool list as the layer-2 detail view. + let block_idx = state + .messages + .iter() + .position(|m| m.role == ChatRole::ToolActivity) + .expect("one tool-activity block"); + state.messages[block_idx].is_collapsed = true; + refresh_tool_activity_message(&mut state, block_idx); + assert_eq!( + state.messages[block_idx].text, "Tool activity · 3 calls · 1 failed\n\ - Web search · completed · 10 results\n\ - Loctree context · completed\n\ @@ -1419,7 +1439,7 @@ fn block_is_reused_within_a_turn_and_reopened_next_turn() { } #[test] -fn collapse_toggle_rerenders_block_header_only() { +fn toggle_switches_between_evidence_summary_and_technical_list() { let mut state = VoiceChatOverlayState::default(); feed_tool_result( &mut state, @@ -1431,11 +1451,25 @@ fn collapse_toggle_rerenders_block_header_only() { ); let idx = state.active_tool_activity_index.expect("block open"); - // Expanded by default: header + lines. - assert!(state.messages[idx].text.contains('\n')); + // Default: the Evidence Summary is the primary view. + assert_eq!( + state.messages[idx].text, + "What I checked · 1 tool\n- Loctree: scanned code surfaces." + ); - // Collapse → header only. + // Click → technical per-tool list. state.messages[idx].is_collapsed = true; refresh_tool_activity_message(&mut state, idx); - assert_eq!(state.messages[idx].text, "Tool activity · 1 call completed"); + assert_eq!( + state.messages[idx].text, + "Tool activity · 1 call completed\n- Loctree context · completed" + ); + + // Click again → back to the Evidence Summary. + state.messages[idx].is_collapsed = false; + refresh_tool_activity_message(&mut state, idx); + assert_eq!( + state.messages[idx].text, + "What I checked · 1 tool\n- Loctree: scanned code surfaces." + ); } diff --git a/app/ui/voice_chat/tool_activity.rs b/app/ui/voice_chat/tool_activity.rs index d64fa985..a6ebb0c8 100644 --- a/app/ui/voice_chat/tool_activity.rs +++ b/app/ui/voice_chat/tool_activity.rs @@ -229,6 +229,257 @@ impl ToolActivityGroup { } out } + + /// Roll the turn's entries up by **source** (the system behind the wire + /// name), preserving first-seen order. Two Loctree calls become one + /// `Loctree` row; counts and the richest result text are folded in. + fn source_summaries(&self) -> Vec { + let mut out: Vec = Vec::new(); + for entry in &self.entries { + let source = source_label(&entry.raw_name, &entry.display_name); + let slot = match out.iter_mut().find(|s| s.source == source) { + Some(slot) => slot, + None => { + out.push(SourceSummary { + source, + completed: 0, + failed: 0, + running: 0, + best_count: None, + best_summary: String::new(), + first_error: String::new(), + }); + out.last_mut().expect("just pushed a source slot") + } + }; + match entry.status { + ToolStatus::Completed => { + slot.completed += 1; + if let Some(count) = entry.result_count { + slot.best_count = Some(slot.best_count.map_or(count, |c| c.max(count))); + } + let summary = entry.summary.trim(); + if summary.chars().count() > slot.best_summary.chars().count() { + slot.best_summary = summary.to_string(); + } + } + ToolStatus::Failed => { + slot.failed += 1; + let error = entry.error.trim(); + if slot.first_error.is_empty() && !error.is_empty() { + slot.first_error = error.to_string(); + } + } + ToolStatus::Running => slot.running += 1, + } + } + out + } + + /// The default operator-facing evidence block for one assistant turn. + /// + /// `What I checked · N tools[ · M warning(s)]`, then one compact line per + /// source (`- Source: detail.`), then a `Key finding:` verdict when one can + /// be extracted. Deterministic and self-contained — it echoes the results it + /// already has, never fabricates prose, and never lets a raw `mcp__` wire + /// name reach the rendered text. This is layer 3 of the product layering + /// (raw payload = debug, technical list = [`render`], summary = default). + pub fn evidence_summary(&self) -> String { + let summaries = self.source_summaries(); + if summaries.is_empty() { + return "What I checked".to_string(); + } + let tools = summaries.len(); + let warnings: usize = summaries.iter().map(|s| s.failed).sum(); + let tool_noun = if tools == 1 { "tool" } else { "tools" }; + let mut header = format!("What I checked · {tools} {tool_noun}"); + if warnings > 0 { + let warn_noun = if warnings == 1 { "warning" } else { "warnings" }; + header.push_str(&format!(" · {warnings} {warn_noun}")); + } + let mut out = header; + for summary in &summaries { + out.push('\n'); + out.push_str(&format!( + "- {}: {}", + summary.source, + finish_clause(&source_detail(summary)) + )); + } + if let Some(key) = evidence_key_finding(&summaries) { + out.push('\n'); + out.push_str(&format!("Key finding: {key}")); + } + out + } +} + +// ---- Evidence Summary helpers (deterministic, no LLM) ------------------------- +// +// The summary groups calls by SOURCE — the system behind the wire name — so it +// answers "which sources did I consult", not "how many wire calls fired". These +// are pure, module-local, and unit-testable. Kept separate from +// `friendly_tool_name` (controller/helpers.rs): that table yields the *specific* +// per-call label ("Loctree context"); this yields the *coarse* system heading +// ("Loctree"). Different granularity, not a duplicate path. + +/// One source's rolled-up evidence within a turn. +#[derive(Debug, Clone)] +struct SourceSummary { + source: String, + completed: usize, + failed: usize, + running: usize, + best_count: Option, + best_summary: String, + first_error: String, +} + +/// Coarse source/system a tool call belongs to. Keyed on the wire `raw_name` +/// (the stable system identity); `display_name` is the fallback for native +/// tools that have no `mcp__server__tool` shape. Never emits a raw `mcp__` name. +fn source_label(raw_name: &str, display_name: &str) -> String { + if let Some(rest) = raw_name.strip_prefix("mcp__") { + let server = rest.split("__").next().unwrap_or(""); + return match server { + "brave-search" => "Web search".to_string(), + "loctree-mcp" => "Loctree".to_string(), + "aicx-mcp" => "AICX".to_string(), + "vibecrafted-mcp" => "Vibecrafted".to_string(), + "" => fallback_source(display_name), + other => prettify_source(other), + }; + } + match raw_name { + "read_clipboard" | "write_clipboard" => "Clipboard".to_string(), + "take_screenshot" => "Screenshot".to_string(), + "transcribe_audio" => "Audio".to_string(), + _ => fallback_source(display_name), + } +} + +/// Use the already-friendly display name when no source mapping applies, so the +/// line still reads cleanly instead of falling back to a wire token. +fn fallback_source(display_name: &str) -> String { + let trimmed = display_name.trim(); + if trimmed.is_empty() { + "Tool".to_string() + } else { + trimmed.to_string() + } +} + +/// Prettify an unknown MCP server segment (`some-server` → `Some server`) so the +/// summary stays readable without leaking the `mcp__` addressing scheme. +fn prettify_source(server: &str) -> String { + let cleaned: String = server + .chars() + .map(|c| if c == '-' || c == '_' { ' ' } else { c }) + .collect(); + let mut chars = cleaned.trim().chars(); + match chars.next() { + Some(first) => first.to_uppercase().collect::() + chars.as_str(), + None => "Tool".to_string(), + } +} + +/// Default action verb for a source when a completed call left no result text, +/// so the line reads as an action ("Loctree: scanned code surfaces") rather than +/// a bare "completed". +fn source_action(source: &str) -> &'static str { + match source { + "Web search" => "searched the web", + "Loctree" => "scanned code surfaces", + "AICX" => "checked intent history", + "Vibecrafted" => "verified run status", + "Clipboard" => "read the clipboard", + "Screenshot" => "captured the screen", + "Audio" => "transcribed audio", + _ => "checked", + } +} + +/// Compact result phrase for one source line. Leads with a failure when the +/// source only failed; otherwise leads with the real result (count or richest +/// summary, or the action verb) and appends running/warning counts. +fn source_detail(summary: &SourceSummary) -> String { + if summary.failed > 0 && summary.completed == 0 && summary.running == 0 { + return if summary.first_error.is_empty() { + "failed".to_string() + } else { + format!("failed — {}", clamp_suffix(&summary.first_error)) + }; + } + + let mut detail = if let Some(count) = summary.best_count { + count_phrase(count) + } else if !summary.best_summary.is_empty() { + clamp_suffix(&summary.best_summary) + } else { + source_action(&summary.source).to_string() + }; + if summary.running > 0 { + detail.push_str(&format!(" · {} running", summary.running)); + } + if summary.failed > 0 { + let noun = if summary.failed == 1 { + "warning" + } else { + "warnings" + }; + detail.push_str(&format!(" · {} {noun}", summary.failed)); + } + detail +} + +/// `N result` / `N results`. +fn count_phrase(count: usize) -> String { + let noun = if count == 1 { "result" } else { "results" }; + format!("{count} {noun}") +} + +/// Deterministic single-line verdict: failures dominate; otherwise echo the most +/// informative completed result. `None` when there is nothing notable to report. +fn evidence_key_finding(summaries: &[SourceSummary]) -> Option { + if let Some(failed) = summaries.iter().find(|s| s.failed > 0) { + return Some(if failed.first_error.is_empty() { + format!("{} check failed.", failed.source) + } else { + format!( + "{} check failed: {}", + failed.source, + finish_clause(&clamp_suffix(&failed.first_error)) + ) + }); + } + + let richest = summaries + .iter() + .filter(|s| !s.best_summary.is_empty() || s.best_count.is_some()) + .max_by_key(|s| s.best_summary.chars().count())?; + let detail = if richest.best_summary.is_empty() { + match richest.best_count { + Some(count) => count_phrase(count), + None => return None, + } + } else { + clamp_suffix(&richest.best_summary) + }; + Some(format!("{} — {}", richest.source, finish_clause(&detail))) +} + +/// Append a sentence period unless the clause already ends in terminal +/// punctuation (including the ellipsis a clamp may add). +fn finish_clause(text: &str) -> String { + let trimmed = text.trim_end(); + if trimmed.is_empty() { + return String::new(); + } + if trimmed.ends_with(['.', '!', '?', '…']) { + trimmed.to_string() + } else { + format!("{trimmed}.") + } } #[cfg(test)] @@ -375,4 +626,148 @@ mod tests { ); assert!(line.ends_with('…'), "clamped suffix is elided"); } + + // ---- Evidence Summary (default operator-facing block) ---------------- + + #[test] + fn evidence_summary_has_semantic_heading_and_one_line_per_source() { + let group = group_from_regression(); + let expected = "What I checked · 3 tools · 1 warning\n\ + - Web search: 10 results.\n\ + - Loctree: scanned code surfaces.\n\ + - AICX: failed — empty index.\n\ + Key finding: AICX check failed: empty index."; + assert_eq!(group.evidence_summary(), expected); + } + + #[test] + fn evidence_summary_collapses_repeat_calls_to_one_source() { + // Two Loctree calls in a turn must read as ONE "Loctree" source, not two + // rows — the summary answers "which sources", not "how many wire calls". + let mut group = ToolActivityGroup::default(); + group.mark_result( + "c1", + "mcp__loctree-mcp__context", + "Loctree context", + "scanned voice_chat", + false, + ); + group.mark_result( + "c2", + "mcp__loctree-mcp__find", + "Loctree occurrences/find", + "clipboard 230 occurrences, schowek 4 occurrences", + false, + ); + let summary = group.evidence_summary(); + assert_eq!( + summary, + "What I checked · 1 tool\n\ + - Loctree: clipboard 230 occurrences, schowek 4 occurrences.\n\ + Key finding: Loctree — clipboard 230 occurrences, schowek 4 occurrences." + ); + assert_eq!( + summary.matches("- Loctree:").count(), + 1, + "two Loctree calls fold into one source line" + ); + } + + #[test] + fn evidence_summary_regression_scenario_three_distinct_sources() { + // The operator's suggested regression event sequence. + let mut group = ToolActivityGroup::default(); + group.mark_running("c1", "mcp__brave-search__brave_web_search", "Web search"); + group.mark_result( + "c1", + "mcp__brave-search__brave_web_search", + "Web search", + "10 results", + false, + ); + group.mark_running("c2", "mcp__loctree-mcp__find", "Loctree occurrences/find"); + group.mark_result( + "c2", + "mcp__loctree-mcp__find", + "Loctree occurrences/find", + "clipboard 230 occurrences, schowek 4 occurrences", + false, + ); + group.mark_running( + "c3", + "mcp__vibecrafted-mcp__vc_run_observe", + "Vibecrafted observe", + ); + group.mark_result( + "c3", + "mcp__vibecrafted-mcp__vc_run_observe", + "Vibecrafted observe", + "run completed", + false, + ); + assert_eq!( + group.evidence_summary(), + "What I checked · 3 tools\n\ + - Web search: 10 results.\n\ + - Loctree: clipboard 230 occurrences, schowek 4 occurrences.\n\ + - Vibecrafted: run completed.\n\ + Key finding: Loctree — clipboard 230 occurrences, schowek 4 occurrences." + ); + } + + #[test] + fn evidence_summary_singular_nouns_for_one_tool_one_warning() { + let mut group = ToolActivityGroup::default(); + group.mark_result( + "c1", + "mcp__aicx-mcp__aicx_intents", + "AICX intents", + "boom", + true, + ); + assert_eq!( + group.evidence_summary(), + "What I checked · 1 tool · 1 warning\n\ + - AICX: failed — boom.\n\ + Key finding: AICX check failed: boom." + ); + } + + #[test] + fn evidence_summary_never_leaks_raw_wire_names() { + let group = group_from_regression(); + assert!( + !group.evidence_summary().contains("mcp__"), + "raw MCP names must never reach the evidence summary" + ); + } + + #[test] + fn evidence_summary_failed_line_is_clamped_not_a_stack_dump() { + let mut group = ToolActivityGroup::default(); + let huge = "panic at line 42: ".repeat(40); + group.mark_result("c1", "mcp__x__y", "Tool", &huge, true); + let summary = group.evidence_summary(); + for line in summary.lines() { + assert!( + line.chars().count() < 160, + "no line may become a stack dump: {line}" + ); + } + } + + #[test] + fn evidence_summary_unknown_server_prettifies_without_wire_scheme() { + let mut group = ToolActivityGroup::default(); + group.mark_result( + "c1", + "mcp__some-other-server__do_thing", + "Do thing · Some other server", + "ok", + false, + ); + let summary = group.evidence_summary(); + assert!(summary.contains("- Some other server: ok.")); + assert!(!summary.contains("mcp__")); + } } From 864c3f8e960eeaf67724fd8f97953a135e1a6d0b Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Fri, 26 Jun 2026 16:10:03 -0700 Subject: [PATCH 014/385] [claude/vc-justdo] feat(onboarding): persist first-run mode choice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the W1-C1 data model + persistence for the first-run operating lane (Basic / Agentic) without any readiness UI yet. - New OnboardingModeChoice enum in onboarding/state.rs mirroring LanguageChoice: Basic is #[default] so a fresh install never lands in the agentic lane by accident; value()/from_value() give a stable settings token with unknown values decoding back to the safe Basic lane. - Persisted in the same style as existing onboarding preferences: a new UserSettings.onboarding_mode field wired through to_v2/from_v2 (SystemV2), set_string("ONBOARDING_MODE"), and PROMOTED_SETTINGS_KEYS — so it survives the flat<->SettingsV2 round-trip and cannot ghost. - initial_onboarding_mode_choice() seeds OnboardingState at window build (load path); save_onboarding_mode() persists on completion in finish_onboarding (save path). Both have real callers, no dead code. - Additive Option field with skip_serializing_if: existing settings.json and setup_done markers are untouched and load unchanged. - Tests: config-layer round-trip + system-section placement (ghosting guard), default-is-None; onboarding-layer default-is-Basic, persisted-Agentic round-trip, and token round-trip with unknown-token fallback. Gates green: fmt, cargo check --workspace, clippy --workspace --all-targets -D warnings. Authored-By: claude session_id: 2db53e33-c286-4709-9579-b828f3f2e36a date: 2026-06-26T16:09:42 PDT runtime: claude --- app/ui/onboarding/actions.rs | 23 ++++++++++++++++ app/ui/onboarding/state.rs | 52 ++++++++++++++++++++++++++++++++++++ app/ui/onboarding/tests.rs | 47 +++++++++++++++++++++++++++++++- app/ui/onboarding/window.rs | 3 ++- core/config/settings.rs | 43 +++++++++++++++++++++++++++++ 5 files changed, 166 insertions(+), 2 deletions(-) diff --git a/app/ui/onboarding/actions.rs b/app/ui/onboarding/actions.rs index ae64da3d..65445316 100644 --- a/app/ui/onboarding/actions.rs +++ b/app/ui/onboarding/actions.rs @@ -464,8 +464,31 @@ fn save_hotkey_mode() { info!("Onboarding: hotkey mode set to {}", mode.label()); } +/// Persist the selected first-run operating lane (Basic / Agentic) into +/// settings.json, mirroring [`save_language_choice`]. No wizard step mutates +/// the lane yet, so a completed onboarding records the safe Basic default; a +/// later readiness-UI cut will set `state.onboarding_mode` before finish. +fn save_onboarding_mode() { + let mode = { + let state = ONBOARDING_STATE.lock().unwrap_or_else(|e| e.into_inner()); + state.onboarding_mode + }; + + let mut settings = UserSettings::load(); + settings.onboarding_mode = Some(mode.value().to_string()); + if let Err(e) = settings.save() { + warn!( + "Onboarding: failed to persist onboarding mode {}: {e}", + mode.value() + ); + } + + info!("Onboarding: mode set to {}", mode.label()); +} + pub(super) fn finish_onboarding(completed: bool) { if completed { + save_onboarding_mode(); reconcile_runtime_after_onboarding_completion(); mark_onboarding_done(); } diff --git a/app/ui/onboarding/state.rs b/app/ui/onboarding/state.rs index ffffe042..776636b4 100644 --- a/app/ui/onboarding/state.rs +++ b/app/ui/onboarding/state.rs @@ -50,6 +50,47 @@ impl HotkeyModeChoice { } } +/// First-run operating lane the user picks during onboarding. +/// +/// `Basic` keeps CodeScribe a plain dictation tool; `Agentic` opts into the +/// dictation-driven orchestration runtime (which a later cut gates on the +/// Vibecrafted + MCP substrate). `Basic` is the safe default so a fresh +/// install never lands in the agentic lane by accident. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(super) enum OnboardingModeChoice { + #[default] + Basic, + Agentic, +} + +impl OnboardingModeChoice { + pub(super) fn label(self) -> &'static str { + match self { + Self::Basic => "Basic", + Self::Agentic => "Agentic", + } + } + + /// Stable token persisted in settings.json. Kept distinct from [`label`] + /// so display copy can change without breaking persisted values. + pub(super) fn value(self) -> &'static str { + match self { + Self::Basic => "basic", + Self::Agentic => "agentic", + } + } + + /// Decode a persisted token. Unknown values fall back to `Basic` — the + /// safe default — so a corrupt or forward-version setting can never force + /// the agentic lane. + pub(super) fn from_value(value: &str) -> Self { + match value { + "agentic" => Self::Agentic, + _ => Self::Basic, + } + } +} + #[derive(Clone, Copy, Default)] pub(super) struct UiRefs { pub(super) sidebar_step_labels: [Option; TOTAL_STEPS], @@ -84,6 +125,7 @@ pub(super) struct OnboardingState { pub(super) step_index: usize, pub(super) language: LanguageChoice, pub(super) hotkey_mode: HotkeyModeChoice, + pub(super) onboarding_mode: OnboardingModeChoice, pub(super) requested_permissions: [bool; 5], pub(super) permission_states: [PermissionUiStatus; 5], pub(super) scheduled_auto_advance_step: Option, @@ -102,6 +144,7 @@ impl Default for OnboardingState { step_index: 0, language: LanguageChoice::default(), hotkey_mode: HotkeyModeChoice::default(), + onboarding_mode: OnboardingModeChoice::default(), requested_permissions: [false; 5], permission_states: [PermissionUiStatus::NotDetermined; 5], scheduled_auto_advance_step: None, @@ -159,3 +202,12 @@ pub(super) fn initial_hotkey_choice() -> HotkeyModeChoice { (false, false) => HotkeyModeChoice::Both, } } + +pub(super) fn initial_onboarding_mode_choice() -> OnboardingModeChoice { + let settings = UserSettings::load(); + settings + .onboarding_mode + .as_deref() + .map(OnboardingModeChoice::from_value) + .unwrap_or_default() +} diff --git a/app/ui/onboarding/tests.rs b/app/ui/onboarding/tests.rs index e421330d..877b8c88 100644 --- a/app/ui/onboarding/tests.rs +++ b/app/ui/onboarding/tests.rs @@ -1,7 +1,7 @@ use serial_test::serial; use tempfile::TempDir; -use crate::config::Config; +use crate::config::{Config, UserSettings}; use crate::os::permissions::PermissionStatus; use super::permission_flow::{ @@ -13,6 +13,7 @@ use super::session::{ setup_done_refresh_target, }; use super::should_show_onboarding; +use super::state::{OnboardingModeChoice, initial_onboarding_mode_choice}; use super::steps::{PermissionKind, PermissionRecoveryStrategy, WizardStep, step_for_index}; fn setup_test_env() -> TempDir { @@ -54,6 +55,50 @@ fn onboarding_progress_round_trips_for_resume() { assert_eq!(load_onboarding_progress(), 3); } +#[test] +#[serial] +fn onboarding_mode_defaults_to_basic_on_fresh_install() { + let _tmp = setup_test_env(); + assert_eq!( + initial_onboarding_mode_choice(), + OnboardingModeChoice::Basic + ); +} + +#[test] +#[serial] +fn onboarding_mode_round_trips_persisted_agentic_choice() { + let _tmp = setup_test_env(); + + let settings = UserSettings { + onboarding_mode: Some(OnboardingModeChoice::Agentic.value().to_string()), + ..Default::default() + }; + settings.save().expect("persist agentic onboarding mode"); + + assert_eq!( + initial_onboarding_mode_choice(), + OnboardingModeChoice::Agentic + ); +} + +#[test] +fn onboarding_mode_value_round_trips_through_token() { + assert_eq!( + OnboardingModeChoice::from_value(OnboardingModeChoice::Basic.value()), + OnboardingModeChoice::Basic + ); + assert_eq!( + OnboardingModeChoice::from_value(OnboardingModeChoice::Agentic.value()), + OnboardingModeChoice::Agentic + ); + // Unknown / forward-version tokens fall back to the safe Basic lane. + assert_eq!( + OnboardingModeChoice::from_value("orchestrator-9000"), + OnboardingModeChoice::Basic + ); +} + fn assert_resume_permission(step: Option, expected: PermissionKind) { assert_eq!( step.map(step_for_index), diff --git a/app/ui/onboarding/window.rs b/app/ui/onboarding/window.rs index 89bc3e58..0c32677e 100644 --- a/app/ui/onboarding/window.rs +++ b/app/ui/onboarding/window.rs @@ -23,7 +23,7 @@ use super::render::render_current_step; use super::session::{load_onboarding_progress, release_onboarding_lock, save_onboarding_progress}; use super::state::{ ONBOARDING_STATE, OnboardingState, UiRefs, initial_hotkey_choice, initial_language_choice, - mode_api_key_configured, + initial_onboarding_mode_choice, mode_api_key_configured, }; use super::steps::TOTAL_STEPS; use super::widgets::{configure_label, create_radio_button, system_secondary_color}; @@ -167,6 +167,7 @@ fn show_onboarding_wizard_impl() -> bool { step_index: resume_step, language: initial_language_choice(), hotkey_mode: initial_hotkey_choice(), + onboarding_mode: initial_onboarding_mode_choice(), requested_permissions: [false; 5], permission_states: [super::permission_flow::PermissionUiStatus::NotDetermined; 5], scheduled_auto_advance_step: None, diff --git a/core/config/settings.rs b/core/config/settings.rs index d1eae505..d8f4c092 100644 --- a/core/config/settings.rs +++ b/core/config/settings.rs @@ -82,6 +82,10 @@ pub struct UserSettings { pub qube_daemon_autostart: Option, #[serde(skip_serializing_if = "Option::is_none")] pub agent_enter_sends: Option, + /// First-run operating lane chosen during onboarding ("basic" | "agentic"). + /// `None` means "not yet chosen" — callers treat that as the safe Basic lane. + #[serde(skip_serializing_if = "Option::is_none")] + pub onboarding_mode: Option, // ── Voice Lab survivors (user-facing UX knobs) ── #[serde(skip_serializing_if = "Option::is_none")] @@ -278,6 +282,8 @@ struct SystemV2 { start_at_login: Option, #[serde(skip_serializing_if = "Option::is_none")] qube_daemon_autostart: Option, + #[serde(skip_serializing_if = "Option::is_none")] + onboarding_mode: Option, } /// Canonical list of env keys that route to `settings.json` (not `.env`). @@ -325,6 +331,7 @@ pub const PROMOTED_SETTINGS_KEYS: &[&str] = &[ "START_AT_LOGIN", "QUBE_DAEMON_AUTOSTART", "AGENT_ENTER_SENDS", + "ONBOARDING_MODE", // Voice Lab survivors "CODESCRIBE_BUFFER_DELAY_MS", "CODESCRIBE_TYPING_CPS", @@ -410,6 +417,7 @@ impl UserSettings { system: Some(SystemV2 { start_at_login: self.start_at_login, qube_daemon_autostart: self.qube_daemon_autostart, + onboarding_mode: self.onboarding_mode.clone(), }), } } @@ -527,6 +535,7 @@ impl UserSettings { quick_notes_save_only: v2.features.as_ref().and_then(|f| f.quick_notes_save_only), start_at_login: v2.system.as_ref().and_then(|s| s.start_at_login), qube_daemon_autostart: v2.system.as_ref().and_then(|s| s.qube_daemon_autostart), + onboarding_mode: v2.system.as_ref().and_then(|s| s.onboarding_mode.clone()), agent_enter_sends: v2.interaction.as_ref().and_then(|i| i.agent_enter_sends), buffer_delay_ms: v2 .speech @@ -793,6 +802,7 @@ impl UserSettings { "AUDIO_INPUT_DEVICE" => self.audio_input_device = Some(value.to_owned()), "SOUND_NAME" => self.sound_name = Some(value.to_owned()), "WHISPER_MODEL" => self.whisper_model = Some(value.to_owned()), + "ONBOARDING_MODE" => self.onboarding_mode = Some(value.to_owned()), other => { warn!("Unknown string setting key: {other}"); return; @@ -1043,6 +1053,39 @@ mod tests { ); } + #[test] + #[serial] + fn test_onboarding_mode_persists_in_v2_system_section() { + // Ghosting guard (W1-C1): onboarding_mode must survive the flat -> + // SettingsV2 -> flat round-trip and land in the V2 `system` section. + let _tmp = setup_isolated_data_dir(); + let mut settings = UserSettings::default(); + settings.set_string("ONBOARDING_MODE", "agentic"); + + let loaded = UserSettings::load(); + assert_eq!(loaded.onboarding_mode.as_deref(), Some("agentic")); + + let path = UserSettings::settings_path(); + let persisted: serde_json::Value = + serde_json::from_str(&fs::read_to_string(path).expect("read persisted settings")) + .expect("parse persisted settings"); + assert_eq!( + persisted + .get("system") + .and_then(|v| v.get("onboarding_mode")) + .and_then(|v| v.as_str()), + Some("agentic") + ); + } + + #[test] + #[serial] + fn test_onboarding_mode_defaults_to_none_when_unset() { + let _tmp = setup_isolated_data_dir(); + let settings = UserSettings::default(); + assert_eq!(settings.onboarding_mode, None); + } + #[test] #[serial] fn test_mode_binding_updates_canonical_contract_only() { From 968d4c11209687521394567113eecbcad618eaa0 Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Fri, 26 Jun 2026 16:20:28 -0700 Subject: [PATCH 015/385] [claude/vc-justdo] feat(onboarding): classify agentic runtime readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W1-C2: adds a mode-aware readiness probe for the Agentic operating lane without disturbing Basic's neutral/optional MCP contract. - New probe_agentic_readiness()/_at() in app/agent/tools/mcp.rs returns an AgenticReadinessReport with a hard ready/not-ready verdict plus one row per prerequisite: Vibecrafted runtime, AICX MCP, Loctree MCP, PRView integration. Reuses McpStatusRow/McpRowTone + the existing runtime-discovery cache — no parallel tone system, no new config parser. - Prereq classification is honest and actionable: not_configured / disabled / failed are blocking (Bad); configured-but-not-yet-discovered is non-blocking (Warn); discovered is Good. Missing mcp.json in Agentic is NOT neutral (that is Basic's contract) — it classifies every prereq as not configured. - PRView: loctree literal scan (356 files) found zero `prview` occurrences anywhere in the repo. We refuse to invent a binary name — classify_prview reports `missing — no PRView MCP server or local command found` and only honours a user-wired server whose name contains "prview". Its absence is surfaced but never fakes-green or blocks core readiness. - Vibecrafted runtime is proven via the `vibecrafted-mcp` server name (verified repo truth: controller/helpers.rs:479, voice_chat/tool_activity.rs:348); a deeper CLI/binary handshake is out of this cut's scope. - Real caller wired: engine_tab.rs build_mcp_section appends the readiness rows only when the persisted onboarding_mode token == "agentic" (unknown/absent decodes to the safe Basic lane). No dead code. - Tests cover the acceptance matrix: Basic-missing-neutral vs Agentic-missing- not-ready contrast, every-prereq-not-configured, configured AICX/Loctree recognized, disabled blocks, PRView missing classified explicitly, parse error not-ready. 12/12 mcp tests green; fmt + cargo check --workspace + clippy --workspace --all-targets -D warnings clean. Authored-By: claude session_id: 3033edc9-6635-494a-bb74-0191533d03af date: 2026-06-26T16:20:11 PDT runtime: claude --- app/agent/tools/mcp.rs | 411 +++++++++++++++++++++++++++++++++- app/ui/settings/engine_tab.rs | 46 +++- 2 files changed, 449 insertions(+), 8 deletions(-) diff --git a/app/agent/tools/mcp.rs b/app/agent/tools/mcp.rs index fb6f3dde..5b1bb492 100644 --- a/app/agent/tools/mcp.rs +++ b/app/agent/tools/mcp.rs @@ -173,6 +173,254 @@ fn probe_mcp_status_at(path: &Path) -> McpStatusReport { } } +/// One Agentic prerequisite the readiness probe classifies, paired with the MCP +/// server name that satisfies it. +/// +/// Repo truth (loctree literal scan, 2026-06-26): these three server names are +/// the only first-party agentic surfaces referenced anywhere in CodeScribe +/// (`app/controller/helpers.rs`, `app/ui/voice_chat/tool_activity.rs`). PRView +/// is deliberately absent here — it has no known MCP server name or local +/// command in repo truth and is handled separately by `classify_prview`. +const AGENTIC_PREREQS: &[(&str, &str)] = &[ + ("Vibecrafted runtime:", "vibecrafted-mcp"), + ("AICX MCP:", "aicx-mcp"), + ("Loctree MCP:", "loctree-mcp"), +]; + +/// Mode-aware readiness verdict for the Agentic operating lane. +/// +/// Distinct from [`McpStatusReport`] (the Basic-mode config probe) so the two +/// lanes can never bleed into each other: Basic stays neutral/optional, Agentic +/// gets a hard ready/not-ready verdict. Reuses [`McpStatusRow`]/[`McpRowTone`] +/// so there is no parallel tone system. +pub struct AgenticReadinessReport { + pub config_path_display: String, + ready: bool, + rows: Vec, +} + +impl AgenticReadinessReport { + pub fn summary_rows(&self) -> &[McpStatusRow] { + &self.rows + } + + /// `true` only when every gating prerequisite (Vibecrafted / AICX / Loctree) + /// is non-blocking. PRView's missing integration is surfaced but never flips + /// this to `false` on its own — otherwise Agentic could never be ready. + pub fn is_ready(&self) -> bool { + self.ready + } +} + +/// Classify one gating prerequisite against config + cached runtime discovery. +/// +/// Returns the display row and whether the prerequisite is *blocking* (i.e. it +/// flips overall readiness to not-ready). The value strings carry a stable +/// keyword (`not configured` / `disabled` / `failed` / `ready` / `configured`) +/// plus an actionable hint, mirroring the honesty of [`probe_mcp_status_at`]. +fn classify_prereq( + label: &str, + server_name: &str, + config: &McpConfigFile, + runtime: &BTreeMap, +) -> (McpStatusRow, bool) { + let configured = config.servers.get(server_name); + let (value, tone, blocking) = match (configured, runtime.get(server_name)) { + // Not present in mcp.json at all — the substrate for this prereq is absent. + (None, _) => ( + format!("not configured — add \"{server_name}\" to ~/.codescribe/mcp.json"), + McpRowTone::Bad, + true, + ), + // Real discovery succeeded — tools are live. + (Some(_), Some(ServerRuntime::Tools(count))) => ( + format!("ready — {count} tool(s) live"), + McpRowTone::Good, + false, + ), + // Configured but discovery failed: surface the concrete reason. + (Some(_), Some(ServerRuntime::Failed(reason))) => { + (format!("failed: {reason}"), McpRowTone::Bad, true) + } + // Present in config but disabled (either via cache or the `enabled` flag): + // a disabled prerequisite blocks the agentic lane. + (Some(cfg), runtime_state) => { + let enabled = cfg.enabled.unwrap_or(true); + if matches!(runtime_state, Some(ServerRuntime::Disabled)) || !enabled { + ( + format!("disabled — set \"enabled\": true for \"{server_name}\""), + McpRowTone::Bad, + true, + ) + } else { + // Config is correct; the agent runtime simply has not run discovery + // yet. Not blocking — the substrate is present. + ( + "configured — agent not started yet".to_string(), + McpRowTone::Warn, + false, + ) + } + } + }; + ( + McpStatusRow { + label: label.to_string(), + value, + tone, + }, + blocking, + ) +} + +/// Classify PRView readiness. +/// +/// Repo truth (loctree literal scan, 2026-06-26): there is NO `prview` MCP +/// server name and no local prview command anywhere in CodeScribe. We refuse to +/// invent a binary name. If a user has manually wired a server whose name +/// contains "prview" we honour that real config; otherwise we report the exact +/// evidence — missing integration — and never fake green readiness. +fn classify_prview( + config: &McpConfigFile, + runtime: &BTreeMap, +) -> McpStatusRow { + let detected = config + .servers + .keys() + .find(|name| name.to_ascii_lowercase().contains("prview")); + match detected { + Some(name) => { + let (value, tone) = match runtime.get(name) { + Some(ServerRuntime::Tools(count)) => ( + format!("ready — {count} tool(s) live (via \"{name}\")"), + McpRowTone::Good, + ), + Some(ServerRuntime::Failed(reason)) => { + (format!("failed: {reason}"), McpRowTone::Bad) + } + Some(ServerRuntime::Disabled) => { + (format!("disabled — \"{name}\""), McpRowTone::Warn) + } + None => ( + format!("configured — agent not started yet (via \"{name}\")"), + McpRowTone::Warn, + ), + }; + McpStatusRow { + label: "PRView integration:".to_string(), + value, + tone, + } + } + None => McpStatusRow { + label: "PRView integration:".to_string(), + // `missing_prview_integration`: no MCP server, no local command. + value: "missing — no PRView MCP server or local command found".to_string(), + tone: McpRowTone::Warn, + }, + } +} + +/// Agentic-lane readiness probe: classifies the agentic substrate prerequisites +/// (Vibecrafted runtime, AICX MCP, Loctree MCP, PRView integration). +/// +/// Only meaningful when the user chose the Agentic operating lane. Basic mode +/// must keep calling [`probe_mcp_status`] so a missing `mcp.json` stays a +/// neutral/optional state rather than a hard not-ready verdict. +pub fn probe_agentic_readiness() -> AgenticReadinessReport { + let path = match codescribe_core::mcp::default_mcp_config_path() { + Ok(path) => path, + Err(error) => { + return AgenticReadinessReport { + config_path_display: "unavailable".to_string(), + ready: false, + rows: vec![McpStatusRow { + label: "Agentic readiness:".to_string(), + value: format!("not ready — config path unavailable: {error}"), + tone: McpRowTone::Bad, + }], + }; + } + }; + probe_agentic_readiness_at(&path) +} + +fn probe_agentic_readiness_at(path: &Path) -> AgenticReadinessReport { + let config_path_display = path.display().to_string(); + + // Missing config in the Agentic lane is NOT neutral (that is Basic's + // contract): the substrate is simply absent, so classify against an empty + // config and every prereq reports "not configured" → not ready. + let config = if !path.exists() { + McpConfigFile { + servers: Default::default(), + } + } else { + match McpConfigFile::load(path) { + Ok(config) => config, + Err(error) => { + return AgenticReadinessReport { + config_path_display, + ready: false, + rows: vec![ + McpStatusRow { + label: "Agentic readiness:".to_string(), + value: "not ready — config error".to_string(), + tone: McpRowTone::Bad, + }, + McpStatusRow { + label: "Config error:".to_string(), + value: anyhow_root_cause(&error), + tone: McpRowTone::Bad, + }, + ], + }; + } + } + }; + + let runtime = runtime_cache() + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .clone(); + + let mut prereq_rows = Vec::with_capacity(AGENTIC_PREREQS.len() + 1); + let mut blocking = 0usize; + for (label, server_name) in AGENTIC_PREREQS { + let (row, is_blocking) = classify_prereq(label, server_name, &config, &runtime); + if is_blocking { + blocking += 1; + } + prereq_rows.push(row); + } + prereq_rows.push(classify_prview(&config, &runtime)); + + let ready = blocking == 0; + let verdict = if ready { + McpStatusRow { + label: "Agentic readiness:".to_string(), + value: "ready — core substrate present (PRView integration unavailable)".to_string(), + tone: McpRowTone::Good, + } + } else { + McpStatusRow { + label: "Agentic readiness:".to_string(), + value: format!("not ready — {blocking} prerequisite(s) missing"), + tone: McpRowTone::Bad, + } + }; + + let mut rows = Vec::with_capacity(prereq_rows.len() + 1); + rows.push(verdict); + rows.extend(prereq_rows); + + AgenticReadinessReport { + config_path_display, + ready, + rows, + } +} + pub fn register(registry: &mut ToolRegistry) { let path = match codescribe_core::mcp::default_mcp_config_path() { Ok(path) => path, @@ -374,7 +622,8 @@ mod tests { use serde_json::json; use super::{ - McpRowTone, probe_mcp_status_at, public_tool_name, register_mcp_tools_from_config_path, + McpRowTone, probe_agentic_readiness_at, probe_mcp_status_at, public_tool_name, + register_mcp_tools_from_config_path, }; #[test] @@ -507,4 +756,164 @@ mod tests { fn repo_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) } + + // --- Agentic readiness probe (W1-C2) ------------------------------------- + // + // These exercise `probe_agentic_readiness_at` with real prereq server names + // (`vibecrafted-mcp` / `aicx-mcp` / `loctree-mcp`). The global runtime cache + // is never populated for those names by any other test, so the readiness + // probe sees them as "configured, agent not started" — the deterministic, + // ordering-safe baseline. + + fn find_row<'a>( + report: &'a super::AgenticReadinessReport, + label: &str, + ) -> &'a super::McpStatusRow { + report + .summary_rows() + .iter() + .find(|row| row.label == label) + .unwrap_or_else(|| panic!("row '{label}' present")) + } + + #[test] + fn basic_probe_stays_neutral_while_agentic_is_not_ready_for_missing_config() { + // Same nonexistent path, two lanes: Basic must remain optional/neutral, + // Agentic must report a hard not-ready verdict. + let temp = tempfile::tempdir().expect("temp dir"); + let path = temp.path().join("mcp.json"); // never created + + let basic = probe_mcp_status_at(&path); + assert_eq!(basic.summary_rows().len(), 1); + assert_eq!(basic.summary_rows()[0].tone, McpRowTone::Neutral); + + let agentic = probe_agentic_readiness_at(&path); + assert!( + !agentic.is_ready(), + "agentic must not be ready without config" + ); + let verdict = find_row(&agentic, "Agentic readiness:"); + assert_eq!(verdict.tone, McpRowTone::Bad); + assert!( + verdict.value.contains("not ready"), + "got: {}", + verdict.value + ); + } + + #[test] + fn agentic_missing_config_classifies_every_prereq_as_not_configured() { + let temp = tempfile::tempdir().expect("temp dir"); + let path = temp.path().join("mcp.json"); // never created + + let report = probe_agentic_readiness_at(&path); + assert!(!report.is_ready()); + for label in ["Vibecrafted runtime:", "AICX MCP:", "Loctree MCP:"] { + let row = find_row(&report, label); + assert_eq!(row.tone, McpRowTone::Bad, "{label} should block"); + assert!( + row.value.contains("not configured"), + "{label} got: {}", + row.value + ); + } + } + + #[test] + fn agentic_recognizes_configured_aicx_and_loctree() { + let temp = tempfile::tempdir().expect("temp dir"); + let path = temp.path().join("mcp.json"); + let config = json!({ + "mcpServers": { + "vibecrafted-mcp": { "command": "vibecrafted-mcp", "enabled": true }, + "aicx-mcp": { "command": "aicx-mcp", "enabled": true }, + "loctree-mcp": { "command": "loctree-mcp", "enabled": true } + } + }); + fs::write(&path, config.to_string()).expect("write config"); + + let report = probe_agentic_readiness_at(&path); + // All three prereqs are configured + enabled (no runtime discovery yet), + // so none block: the core substrate is recognized as present. + assert!( + report.is_ready(), + "configured substrate should be ready: {:?}", + report + .summary_rows() + .iter() + .map(|r| format!("{}={}", r.label, r.value)) + .collect::>() + ); + for label in ["AICX MCP:", "Loctree MCP:"] { + let row = find_row(&report, label); + assert_eq!(row.tone, McpRowTone::Warn); + assert!( + row.value.contains("configured"), + "{label} got: {}", + row.value + ); + } + } + + #[test] + fn agentic_disabled_prereq_blocks_readiness() { + let temp = tempfile::tempdir().expect("temp dir"); + let path = temp.path().join("mcp.json"); + let config = json!({ + "mcpServers": { + "vibecrafted-mcp": { "command": "vibecrafted-mcp", "enabled": true }, + "aicx-mcp": { "command": "aicx-mcp", "enabled": false }, + "loctree-mcp": { "command": "loctree-mcp", "enabled": true } + } + }); + fs::write(&path, config.to_string()).expect("write config"); + + let report = probe_agentic_readiness_at(&path); + assert!(!report.is_ready(), "a disabled prereq must block readiness"); + let row = find_row(&report, "AICX MCP:"); + assert_eq!(row.tone, McpRowTone::Bad); + assert!(row.value.contains("disabled"), "got: {}", row.value); + } + + #[test] + fn agentic_prview_classified_missing_when_no_known_integration() { + let temp = tempfile::tempdir().expect("temp dir"); + let path = temp.path().join("mcp.json"); + // Full agentic substrate present, but no PRView surface anywhere. + let config = json!({ + "mcpServers": { + "vibecrafted-mcp": { "command": "vibecrafted-mcp", "enabled": true }, + "aicx-mcp": { "command": "aicx-mcp", "enabled": true }, + "loctree-mcp": { "command": "loctree-mcp", "enabled": true } + } + }); + fs::write(&path, config.to_string()).expect("write config"); + + let report = probe_agentic_readiness_at(&path); + let row = find_row(&report, "PRView integration:"); + assert!( + row.value.contains("missing") && row.value.contains("PRView"), + "PRView must be explicitly classified as missing, got: {}", + row.value + ); + // PRView's missing integration must NOT, on its own, block readiness — + // otherwise Agentic could never be ready. + assert!( + report.is_ready(), + "missing PRView alone should not block core readiness" + ); + } + + #[test] + fn agentic_parse_error_is_not_ready_with_concrete_reason() { + let temp = tempfile::tempdir().expect("temp dir"); + let path = temp.path().join("mcp.json"); + fs::write(&path, "{ not valid json").expect("write garbage"); + + let report = probe_agentic_readiness_at(&path); + assert!(!report.is_ready()); + let row = find_row(&report, "Config error:"); + assert_eq!(row.tone, McpRowTone::Bad); + assert!(!row.value.is_empty()); + } } diff --git a/app/ui/settings/engine_tab.rs b/app/ui/settings/engine_tab.rs index acf14952..6a23057d 100644 --- a/app/ui/settings/engine_tab.rs +++ b/app/ui/settings/engine_tab.rs @@ -398,14 +398,46 @@ fn build_mcp_section(container: Id, y: &mut f64, pad: f64, content_w: f64) { secondary, ); + let tone_color = |tone: McpRowTone| match tone { + McpRowTone::Good => ui_colors::status_granted(), + McpRowTone::Warn => ui_colors::status_warning(), + McpRowTone::Bad => ui_colors::bubble_error_text(), + McpRowTone::Neutral => secondary, + }; + for row in status.summary_rows() { - let color = match row.tone { - McpRowTone::Good => ui_colors::status_granted(), - McpRowTone::Warn => ui_colors::status_warning(), - McpRowTone::Bad => ui_colors::bubble_error_text(), - McpRowTone::Neutral => secondary, - }; - add_engine_metric_row(container, y, pad, content_w, &row.label, &row.value, color); + add_engine_metric_row( + container, + y, + pad, + content_w, + &row.label, + &row.value, + tone_color(row.tone), + ); + } + + // Agentic lane adds a stricter readiness verdict (Vibecrafted / AICX / + // Loctree / PRView). Basic mode keeps the neutral config probe above and + // never sees this block. The persisted token is the source of truth; an + // unknown/absent value decodes to the safe Basic lane. + let agentic = crate::config::UserSettings::load() + .onboarding_mode + .as_deref() + == Some("agentic"); + if agentic { + let readiness = crate::agent::tools::mcp::probe_agentic_readiness(); + for row in readiness.summary_rows() { + add_engine_metric_row( + container, + y, + pad, + content_w, + &row.label, + &row.value, + tone_color(row.tone), + ); + } } // SAFETY: Settings AppKit main-thread build path. From de4960ea928db274a7184ded50c36ea7da973258 Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Fri, 26 Jun 2026 16:28:26 -0700 Subject: [PATCH 016/385] [claude/vc-justdo] fix(onboarding): require PRView for agentic readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W1-C2b — correct W1-C2's product semantics. PRView MCP/integration is minimum agentic substrate, not optional decoration, so a missing/disabled/ failed PRView surface must now BLOCK Agentic readiness instead of merely warning. - classify_prview now returns (McpStatusRow, bool) matching classify_prereq, with the full state matrix: Tools=live/ok, Failed=block, Disabled or enabled:false=block, configured-not-started=non-blocking. - Missing PRView keeps the honest label (loctree literal prview = 0 production occurrences, no invented binary name) but flips to Bad tone + "(required)" and feeds the blocking counter, so is_ready() is false. - Ready verdict copy updated: "ready" now implies full substrate present. - Tests: missing PRView blocks readiness; a configured/enabled PRView-like server satisfies the prereq; a disabled PRView server blocks. Basic-mode probe_mcp_status untouched (missing mcp.json stays neutral/optional). Authored-By: claude session_id: 4916ab15-cd14-4431-8060-7473cb4aa913 date: 2026-06-26T23:10:00 CEST runtime: terminal --- app/agent/tools/mcp.rs | 165 +++++++++++++++++++++++++++++------------ 1 file changed, 118 insertions(+), 47 deletions(-) diff --git a/app/agent/tools/mcp.rs b/app/agent/tools/mcp.rs index 5b1bb492..0112fad4 100644 --- a/app/agent/tools/mcp.rs +++ b/app/agent/tools/mcp.rs @@ -204,9 +204,10 @@ impl AgenticReadinessReport { &self.rows } - /// `true` only when every gating prerequisite (Vibecrafted / AICX / Loctree) - /// is non-blocking. PRView's missing integration is surfaced but never flips - /// this to `false` on its own — otherwise Agentic could never be ready. + /// `true` only when every gating prerequisite is non-blocking. The agentic + /// substrate is Vibecrafted + AICX + Loctree + PRView; all four are required. + /// A missing/disabled/failed PRView integration flips this to `false` just + /// like any other prerequisite — PRView is minimum substrate, not decoration. pub fn is_ready(&self) -> bool { self.ready } @@ -273,52 +274,78 @@ fn classify_prereq( ) } -/// Classify PRView readiness. +/// Classify PRView readiness as a **required** agentic prerequisite. /// /// Repo truth (loctree literal scan, 2026-06-26): there is NO `prview` MCP /// server name and no local prview command anywhere in CodeScribe. We refuse to /// invent a binary name. If a user has manually wired a server whose name /// contains "prview" we honour that real config; otherwise we report the exact /// evidence — missing integration — and never fake green readiness. +/// +/// Returns the display row and whether the prerequisite is *blocking*, matching +/// [`classify_prereq`]'s contract: a missing/disabled/failed PRView integration +/// blocks Agentic readiness because PRView is minimum substrate, not decoration. fn classify_prview( config: &McpConfigFile, runtime: &BTreeMap, -) -> McpStatusRow { +) -> (McpStatusRow, bool) { let detected = config .servers - .keys() - .find(|name| name.to_ascii_lowercase().contains("prview")); - match detected { - Some(name) => { - let (value, tone) = match runtime.get(name) { - Some(ServerRuntime::Tools(count)) => ( - format!("ready — {count} tool(s) live (via \"{name}\")"), - McpRowTone::Good, - ), - Some(ServerRuntime::Failed(reason)) => { - (format!("failed: {reason}"), McpRowTone::Bad) - } - Some(ServerRuntime::Disabled) => { - (format!("disabled — \"{name}\""), McpRowTone::Warn) + .iter() + .find(|(name, _)| name.to_ascii_lowercase().contains("prview")); + let (value, tone, blocking) = match detected { + Some((name, cfg)) => match runtime.get(name) { + // Real discovery succeeded — PRView tools are live. + Some(ServerRuntime::Tools(count)) => ( + format!("ready — {count} tool(s) live (via \"{name}\")"), + McpRowTone::Good, + false, + ), + // Configured but discovery failed: surface the concrete reason. + Some(ServerRuntime::Failed(reason)) => { + (format!("failed: {reason}"), McpRowTone::Bad, true) + } + // Disabled (via cache or the `enabled` flag) blocks the agentic lane. + Some(ServerRuntime::Disabled) => ( + format!("disabled — set \"enabled\": true for \"{name}\""), + McpRowTone::Bad, + true, + ), + None => { + if cfg.enabled.unwrap_or(true) { + // Config is correct; the agent runtime has not run discovery + // yet. Not blocking — the substrate is present. + ( + format!("configured — agent not started yet (via \"{name}\")"), + McpRowTone::Warn, + false, + ) + } else { + ( + format!("disabled — set \"enabled\": true for \"{name}\""), + McpRowTone::Bad, + true, + ) } - None => ( - format!("configured — agent not started yet (via \"{name}\")"), - McpRowTone::Warn, - ), - }; - McpStatusRow { - label: "PRView integration:".to_string(), - value, - tone, } - } - None => McpStatusRow { + }, + // `missing_prview_integration`: no MCP server, no local command. Honest + // evidence preserved (loctree literal `prview` = 0 production occurrences), + // but PRView is required substrate, so its absence blocks readiness. + None => ( + "missing (required) — no PRView MCP server or local command found".to_string(), + McpRowTone::Bad, + true, + ), + }; + ( + McpStatusRow { label: "PRView integration:".to_string(), - // `missing_prview_integration`: no MCP server, no local command. - value: "missing — no PRView MCP server or local command found".to_string(), - tone: McpRowTone::Warn, + value, + tone, }, - } + blocking, + ) } /// Agentic-lane readiness probe: classifies the agentic substrate prerequisites @@ -393,13 +420,17 @@ fn probe_agentic_readiness_at(path: &Path) -> AgenticReadinessReport { } prereq_rows.push(row); } - prereq_rows.push(classify_prview(&config, &runtime)); + let (prview_row, prview_blocking) = classify_prview(&config, &runtime); + if prview_blocking { + blocking += 1; + } + prereq_rows.push(prview_row); let ready = blocking == 0; let verdict = if ready { McpStatusRow { label: "Agentic readiness:".to_string(), - value: "ready — core substrate present (PRView integration unavailable)".to_string(), + value: "ready — full agentic substrate present".to_string(), tone: McpRowTone::Good, } } else { @@ -820,24 +851,27 @@ mod tests { } #[test] - fn agentic_recognizes_configured_aicx_and_loctree() { + fn agentic_recognizes_configured_full_substrate() { let temp = tempfile::tempdir().expect("temp dir"); let path = temp.path().join("mcp.json"); + // Full required substrate: Vibecrafted + AICX + Loctree + a PRView-like + // server. A user who wires a "prview" server satisfies the prerequisite. let config = json!({ "mcpServers": { "vibecrafted-mcp": { "command": "vibecrafted-mcp", "enabled": true }, "aicx-mcp": { "command": "aicx-mcp", "enabled": true }, - "loctree-mcp": { "command": "loctree-mcp", "enabled": true } + "loctree-mcp": { "command": "loctree-mcp", "enabled": true }, + "vista-prview": { "command": "vista-prview", "enabled": true } } }); fs::write(&path, config.to_string()).expect("write config"); let report = probe_agentic_readiness_at(&path); - // All three prereqs are configured + enabled (no runtime discovery yet), - // so none block: the core substrate is recognized as present. + // All four prereqs are configured + enabled (no runtime discovery yet), + // so none block: the full agentic substrate is recognized as present. assert!( report.is_ready(), - "configured substrate should be ready: {:?}", + "configured full substrate should be ready: {:?}", report .summary_rows() .iter() @@ -853,6 +887,40 @@ mod tests { row.value ); } + // A configured/enabled PRView-like server satisfies the PRView prereq. + let prview = find_row(&report, "PRView integration:"); + assert_eq!(prview.tone, McpRowTone::Warn); + assert!( + prview.value.contains("configured") && prview.value.contains("vista-prview"), + "PRView-like server should satisfy the prerequisite, got: {}", + prview.value + ); + } + + #[test] + fn agentic_disabled_prview_server_blocks_readiness() { + let temp = tempfile::tempdir().expect("temp dir"); + let path = temp.path().join("mcp.json"); + // Core substrate present, but the wired PRView-like server is disabled — + // a disabled required prerequisite must block readiness. + let config = json!({ + "mcpServers": { + "vibecrafted-mcp": { "command": "vibecrafted-mcp", "enabled": true }, + "aicx-mcp": { "command": "aicx-mcp", "enabled": true }, + "loctree-mcp": { "command": "loctree-mcp", "enabled": true }, + "vista-prview": { "command": "vista-prview", "enabled": false } + } + }); + fs::write(&path, config.to_string()).expect("write config"); + + let report = probe_agentic_readiness_at(&path); + assert!( + !report.is_ready(), + "a disabled PRView server must block readiness" + ); + let prview = find_row(&report, "PRView integration:"); + assert_eq!(prview.tone, McpRowTone::Bad); + assert!(prview.value.contains("disabled"), "got: {}", prview.value); } #[test] @@ -876,10 +944,11 @@ mod tests { } #[test] - fn agentic_prview_classified_missing_when_no_known_integration() { + fn agentic_prview_missing_blocks_readiness() { let temp = tempfile::tempdir().expect("temp dir"); let path = temp.path().join("mcp.json"); - // Full agentic substrate present, but no PRView surface anywhere. + // Core substrate present, but no PRView surface anywhere — this is the + // repo-truth baseline (loctree literal `prview` = 0 production hits). let config = json!({ "mcpServers": { "vibecrafted-mcp": { "command": "vibecrafted-mcp", "enabled": true }, @@ -891,16 +960,18 @@ mod tests { let report = probe_agentic_readiness_at(&path); let row = find_row(&report, "PRView integration:"); + // Honest evidence preserved AND now flagged as a hard, blocking gap. + assert_eq!(row.tone, McpRowTone::Bad); assert!( row.value.contains("missing") && row.value.contains("PRView"), "PRView must be explicitly classified as missing, got: {}", row.value ); - // PRView's missing integration must NOT, on its own, block readiness — - // otherwise Agentic could never be ready. + // PRView is required substrate: its absence blocks Agentic readiness even + // when Vibecrafted / AICX / Loctree are all present. assert!( - report.is_ready(), - "missing PRView alone should not block core readiness" + !report.is_ready(), + "missing PRView must block agentic readiness" ); } From a14a0a63d949e866626df23804a1aba56a2e9836 Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Fri, 26 Jun 2026 16:43:19 -0700 Subject: [PATCH 017/385] [claude/vc-justdo] feat(onboarding): add agentic mode readiness step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W1-C3: wire the first-run wizard so users pick Basic vs Agentic and see the right next steps. Two new steps join the fixed STEP_FLOW: a Mode chooser right after Welcome, and an Agentic-only readiness verdict before Done. Keeping both in the fixed array (rather than a mode-dependent flow) preserves resume-safe step indices; the readiness step is navigated *around* in the Basic lane via pure, tested routing helpers (step_is_visible / next_visible_step / prev_visible_step). The readiness table reuses the W1-C2 contract verbatim (probe_agentic_readiness → AgenticReadinessReport): verdict row plus Vibecrafted / AICX / Loctree / PRView prerequisites, each keeping its actionable value text and tone so missing items never collapse into one generic error. Tone→color mapping mirrors the Settings engine tab so the two surfaces never drift. Basic lane never blocks on MCP readiness. FULL_DISK_STEP_INDEX is now derived from STEP_FLOW instead of hardcoded, so inserting the Mode step ahead of it cannot silently break the full-disk polling guards. Sidebar renders lane-hidden steps as dimmed, unmarked entries so Basic never shows a false checkmark on Readiness. Gates: cargo fmt --check, cargo check --workspace, cargo clippy --workspace --all-targets -D warnings all clean; 18 onboarding tests green (5 new flow-routing cases for Basic and Agentic lanes). GUI window not launched (interactive AppKit menu-bar surface; verified via compile + data contract + routing tests). Authored-By: claude session_id: 36b582bd-4172-47d8-8302-fec4e11aa692 date: 2026-06-26T16:42:54 PDT runtime: terminal --- app/ui/onboarding/actions.rs | 79 +++++++++++++++++++++----- app/ui/onboarding/handlers.rs | 21 ++++++- app/ui/onboarding/render.rs | 86 ++++++++++++++++++++++++++-- app/ui/onboarding/state.rs | 6 ++ app/ui/onboarding/steps.rs | 12 +++- app/ui/onboarding/tests.rs | 104 +++++++++++++++++++++++++++++++++- app/ui/onboarding/widgets.rs | 20 ++++++- app/ui/onboarding/window.rs | 84 +++++++++++++++++++++++++++ 8 files changed, 391 insertions(+), 21 deletions(-) diff --git a/app/ui/onboarding/actions.rs b/app/ui/onboarding/actions.rs index 65445316..77d59289 100644 --- a/app/ui/onboarding/actions.rs +++ b/app/ui/onboarding/actions.rs @@ -22,11 +22,55 @@ use super::permission_flow::{ }; use super::render::render_current_step; use super::session::{mark_onboarding_done, release_onboarding_lock, save_onboarding_progress}; -use super::state::{HotkeyModeChoice, ONBOARDING_STATE}; -use super::steps::{PermissionKind, TOTAL_STEPS, WizardStep, step_for_index}; +use super::state::{HotkeyModeChoice, ONBOARDING_STATE, OnboardingModeChoice}; +use super::steps::{PermissionKind, STEP_FLOW, TOTAL_STEPS, WizardStep, step_for_index}; use super::widgets::{get_text_field_string, system_green_color, system_red_color}; -const FULL_DISK_STEP_INDEX: usize = 5; +/// Resolve the flow index of the Full Disk Access step from the canonical +/// [`STEP_FLOW`] rather than hardcoding it — inserting steps ahead of it (the +/// Mode step) must not silently break the polling-reset guards below. +fn full_disk_step_index() -> usize { + STEP_FLOW + .iter() + .position(|step| *step == WizardStep::Permission(PermissionKind::FullDiskAccess)) + .unwrap_or(0) +} + +/// Whether `step` is part of the active flow for the chosen operating lane. +/// +/// The Agentic readiness verdict is only meaningful in the Agentic lane; in the +/// Basic lane it is skipped entirely so a fresh install never blocks on +/// Vibecrafted/MCP prerequisites it does not need. Every other step is shared. +pub(super) fn step_is_visible(step: WizardStep, mode: OnboardingModeChoice) -> bool { + match step { + WizardStep::AgenticReadiness => mode == OnboardingModeChoice::Agentic, + _ => true, + } +} + +/// Next flow index visible in the given lane, skipping mode-hidden steps. +pub(super) fn next_visible_step(index: usize, mode: OnboardingModeChoice) -> Option { + let mut candidate = index + 1; + while candidate < TOTAL_STEPS { + if step_is_visible(step_for_index(candidate), mode) { + return Some(candidate); + } + candidate += 1; + } + None +} + +/// Previous flow index visible in the given lane, skipping mode-hidden steps. +pub(super) fn prev_visible_step(index: usize, mode: OnboardingModeChoice) -> Option { + let mut candidate = index; + while candidate > 0 { + candidate -= 1; + if step_is_visible(step_for_index(candidate), mode) { + return Some(candidate); + } + } + None +} pub(super) fn handle_primary_action() { let step = { @@ -36,6 +80,12 @@ pub(super) fn handle_primary_action() { match step { WizardStep::Welcome => advance_step(), + WizardStep::Mode => { + // Persist the lane on confirm so a mid-wizard quit or resume already + // records the user's choice; finish_onboarding re-saves defensively. + save_onboarding_mode(); + advance_step(); + } WizardStep::Permission(kind) => handle_permission_primary(kind), WizardStep::Language => { save_language_choice(); @@ -50,6 +100,7 @@ pub(super) fn handle_primary_action() { save_hotkey_mode(); advance_step(); } + WizardStep::AgenticReadiness => advance_step(), WizardStep::Done => finish_onboarding(true), } } @@ -173,14 +224,15 @@ fn handle_permission_primary(kind: PermissionKind) { } fn advance_step() { + let full_disk = full_disk_step_index(); let mut should_render = false; let mut new_step = None; { let mut state = ONBOARDING_STATE.lock().unwrap_or_else(|e| e.into_inner()); - if state.step_index + 1 < TOTAL_STEPS { - state.step_index += 1; + if let Some(next) = next_visible_step(state.step_index, state.onboarding_mode) { + state.step_index = next; state.scheduled_auto_advance_step = None; - if state.step_index != FULL_DISK_STEP_INDEX { + if state.step_index != full_disk { state.full_disk_polling = false; } new_step = Some(state.step_index); @@ -197,14 +249,15 @@ fn advance_step() { } fn retreat_step() { + let full_disk = full_disk_step_index(); let mut should_render = false; let mut new_step = None; { let mut state = ONBOARDING_STATE.lock().unwrap_or_else(|e| e.into_inner()); - if state.step_index > 0 { - state.step_index -= 1; + if let Some(prev) = prev_visible_step(state.step_index, state.onboarding_mode) { + state.step_index = prev; state.scheduled_auto_advance_step = None; - if state.step_index != FULL_DISK_STEP_INDEX { + if state.step_index != full_disk { state.full_disk_polling = false; } new_step = Some(state.step_index); @@ -327,7 +380,7 @@ fn start_full_disk_polling() { render_current_step(); } if should_schedule { - maybe_schedule_auto_advance(FULL_DISK_STEP_INDEX); + maybe_schedule_auto_advance(full_disk_step_index()); } }); } @@ -465,9 +518,9 @@ fn save_hotkey_mode() { } /// Persist the selected first-run operating lane (Basic / Agentic) into -/// settings.json, mirroring [`save_language_choice`]. No wizard step mutates -/// the lane yet, so a completed onboarding records the safe Basic default; a -/// later readiness-UI cut will set `state.onboarding_mode` before finish. +/// settings.json, mirroring [`save_language_choice`]. Called both when the user +/// confirms the Mode step and again from [`finish_onboarding`] so the safe Basic +/// default is recorded even if the lane step is somehow bypassed. fn save_onboarding_mode() { let mode = { let state = ONBOARDING_STATE.lock().unwrap_or_else(|e| e.into_inner()); diff --git a/app/ui/onboarding/handlers.rs b/app/ui/onboarding/handlers.rs index 735873ee..fe7c8cea 100644 --- a/app/ui/onboarding/handlers.rs +++ b/app/ui/onboarding/handlers.rs @@ -13,7 +13,9 @@ use super::actions::{ }; use super::render::render_current_step; use super::session::release_onboarding_lock; -use super::state::{HotkeyModeChoice, LanguageChoice, ONBOARDING_STATE, UiRefs}; +use super::state::{ + HotkeyModeChoice, LanguageChoice, ONBOARDING_STATE, OnboardingModeChoice, UiRefs, +}; static ACTION_HANDLER_CLASS: OnceLock<&'static Class> = OnceLock::new(); static WINDOW_DELEGATE_CLASS: OnceLock<&'static Class> = OnceLock::new(); @@ -36,6 +38,10 @@ pub(super) fn action_handler_class() -> &'static Class { sel!(onSkipAction:), on_skip_action as extern "C" fn(&Object, Sel, Id), ); + decl.add_method( + sel!(onModeSelected:), + on_mode_selected as extern "C" fn(&Object, Sel, Id), + ); decl.add_method( sel!(onLanguageSelected:), on_language_selected as extern "C" fn(&Object, Sel, Id), @@ -78,6 +84,19 @@ extern "C" fn on_skip_action(_this: &Object, _sel: Sel, _sender: Id) { handle_skip_action(); } +extern "C" fn on_mode_selected(_this: &Object, _sel: Sel, sender: Id) { + unsafe { + let tag: isize = msg_send![sender, tag]; + let mut state = ONBOARDING_STATE.lock().unwrap_or_else(|e| e.into_inner()); + state.onboarding_mode = if tag == 1 { + OnboardingModeChoice::Agentic + } else { + OnboardingModeChoice::Basic + }; + } + render_current_step(); +} + extern "C" fn on_language_selected(_this: &Object, _sel: Sel, sender: Id) { unsafe { let tag: isize = msg_send![sender, tag]; diff --git a/app/ui/onboarding/render.rs b/app/ui/onboarding/render.rs index 0f4e4bad..f024896d 100644 --- a/app/ui/onboarding/render.rs +++ b/app/ui/onboarding/render.rs @@ -3,21 +3,25 @@ use crate::ui::shared::helpers::color_label; +use super::Id; use super::actions::maybe_schedule_auto_advance; use super::permission_flow::{ PERMISSION_ORDER, PermissionUiStatus, check_permission_state, permission_instruction_text, permission_status_color, permission_status_text, refresh_all_permission_states_locked, should_wait_for_restart, }; -use super::state::{HotkeyModeChoice, LanguageChoice, ONBOARDING_STATE, UiRefs}; +use super::state::{ + HotkeyModeChoice, LanguageChoice, ONBOARDING_STATE, OnboardingModeChoice, UiRefs, +}; use super::steps::{ PermissionKind, PermissionRecoveryStrategy, TOTAL_STEPS, WizardStep, step_for_index, }; use super::widgets::{ set_button_title_if_present, set_hidden_if_present, set_label_color_if_present, - set_text_if_present, sync_hotkey_radios, sync_language_radios, system_green_color, - system_red_color, system_secondary_color, + set_text_if_present, sync_hotkey_radios, sync_language_radios, sync_mode_radios, + system_green_color, system_orange_color, system_red_color, system_secondary_color, }; +use crate::agent::tools::mcp::{McpRowTone, probe_agentic_readiness}; pub(super) fn render_current_step() { let ( @@ -25,6 +29,7 @@ pub(super) fn render_current_step() { step, language, hotkey_mode, + onboarding_mode, api_key_configured, permissions, requested_permissions, @@ -49,6 +54,7 @@ pub(super) fn render_current_step() { step, state.language, state.hotkey_mode, + state.onboarding_mode, state.api_key_configured, state.permission_states, state.requested_permissions, @@ -63,6 +69,8 @@ pub(super) fn render_current_step() { set_hidden_if_present(ui.status_label, true); set_hidden_if_present(ui.instruction_label, true); + set_hidden_if_present(ui.mode_view, true); + set_hidden_if_present(ui.readiness_view, true); set_hidden_if_present(ui.language_view, true); set_hidden_if_present(ui.api_view, true); set_hidden_if_present(ui.hotkey_view, true); @@ -74,9 +82,10 @@ pub(super) fn render_current_step() { set_hidden_if_present(ui.back_button, step_index == 0); + sync_mode_radios(ui, onboarding_mode); sync_language_radios(ui, language); sync_hotkey_radios(ui, hotkey_mode); - update_sidebar_step_labels(ui, step_index, permissions); + update_sidebar_step_labels(ui, step_index, permissions, onboarding_mode); match step { WizardStep::Welcome => { @@ -88,6 +97,16 @@ pub(super) fn render_current_step() { ); set_button_title_if_present(ui.primary_button, "Get Started"); } + WizardStep::Mode => { + set_text_if_present(ui.icon_label, "MODE"); + set_text_if_present(ui.title_label, "Choose Your Lane"); + set_text_if_present( + ui.description_label, + "Basic keeps CodeScribe a local dictation tool. Agentic turns dictation into agent orchestration through Vibecrafted and MCP, with a readiness check next. You can switch later in Settings.", + ); + set_hidden_if_present(ui.mode_view, false); + set_button_title_if_present(ui.primary_button, "Continue"); + } WizardStep::Permission(kind) => { let idx = kind.index(); let status = permissions[idx]; @@ -173,6 +192,19 @@ pub(super) fn render_current_step() { set_hidden_if_present(ui.hotkey_view, false); set_button_title_if_present(ui.primary_button, "Continue"); } + WizardStep::AgenticReadiness => { + set_text_if_present(ui.icon_label, "AGENT"); + set_text_if_present(ui.title_label, "Agentic Runtime"); + // Short copy: the readiness table below is tall; a long description + // would overlap it. The table carries the actionable detail. + set_text_if_present( + ui.description_label, + "Live prerequisites for the Agentic lane — fix any missing item, or continue and finish setup later.", + ); + set_hidden_if_present(ui.readiness_view, false); + update_readiness_view(ui); + set_button_title_if_present(ui.primary_button, "Continue"); + } WizardStep::Done => { set_text_if_present(ui.icon_label, "DONE"); set_text_if_present(ui.title_label, "You're All Set"); @@ -194,6 +226,7 @@ pub(super) fn render_current_step() { fn sidebar_step_title(step: WizardStep) -> &'static str { match step { WizardStep::Welcome => "Welcome", + WizardStep::Mode => "Mode", WizardStep::Permission(PermissionKind::Microphone) => "Microphone", WizardStep::Permission(PermissionKind::Accessibility) => "Accessibility", WizardStep::Permission(PermissionKind::InputMonitoring) => "Input Monitoring", @@ -202,6 +235,7 @@ fn sidebar_step_title(step: WizardStep) -> &'static str { WizardStep::Language => "Language", WizardStep::ApiKey => "API Key", WizardStep::HotkeyMode => "Hotkeys", + WizardStep::AgenticReadiness => "Readiness", WizardStep::Done => "Finish", } } @@ -210,9 +244,20 @@ fn update_sidebar_step_labels( ui: UiRefs, current_step_index: usize, permissions: [PermissionUiStatus; 5], + mode: OnboardingModeChoice, ) { + use super::actions::step_is_visible; for idx in 0..TOTAL_STEPS { let step = step_for_index(idx); + // Steps not part of the active lane (e.g. Readiness in Basic) render as a + // dimmed, unmarked entry so the sidebar never shows a false ✓ for a step + // the user never visits. + if !step_is_visible(step, mode) { + let text = format!("\u{25CB} {}", sidebar_step_title(step)); + set_text_if_present(ui.sidebar_step_labels[idx], &text); + set_label_color_if_present(ui.sidebar_step_labels[idx], system_secondary_color()); + continue; + } let (marker, color) = if idx == current_step_index { if let WizardStep::Permission(kind) = step { let status = permissions[kind.index()]; @@ -286,3 +331,36 @@ fn update_summary_view( ), ); } + +/// Map an [`McpRowTone`] to an AppKit color, mirroring the Settings engine tab so +/// the two surfaces never drift on what "ready / warn / blocked" looks like. +fn readiness_tone_color(tone: McpRowTone) -> Id { + match tone { + McpRowTone::Good => system_green_color(), + McpRowTone::Warn => system_orange_color(), + McpRowTone::Bad => system_red_color(), + McpRowTone::Neutral => system_secondary_color(), + } +} + +/// Fill the readiness table from a live [`probe_agentic_readiness`] run. +/// +/// Re-uses the W1-C2 verdict contract verbatim (verdict row + Vibecrafted / AICX +/// / Loctree / PRView prerequisites), so each row keeps its actionable value text +/// and tone instead of collapsing into a single generic error. Rows beyond the +/// five reserved labels are dropped (the probe never emits more in normal flow); +/// any unused label slots are blanked. +fn update_readiness_view(ui: UiRefs) { + let report = probe_agentic_readiness(); + let rows = report.summary_rows(); + + for (slot, label_ptr) in ui.readiness_row_labels.iter().enumerate() { + match rows.get(slot) { + Some(row) => { + set_text_if_present(*label_ptr, &format!("{} {}", row.label, row.value)); + set_label_color_if_present(*label_ptr, readiness_tone_color(row.tone)); + } + None => set_text_if_present(*label_ptr, ""), + } + } +} diff --git a/app/ui/onboarding/state.rs b/app/ui/onboarding/state.rs index 776636b4..bdc26c7e 100644 --- a/app/ui/onboarding/state.rs +++ b/app/ui/onboarding/state.rs @@ -103,6 +103,12 @@ pub(super) struct UiRefs { pub(super) primary_button: Option, pub(super) back_button: Option, pub(super) skip_button: Option, + pub(super) mode_view: Option, + pub(super) mode_basic_radio: Option, + pub(super) mode_agentic_radio: Option, + pub(super) readiness_view: Option, + pub(super) readiness_heading_label: Option, + pub(super) readiness_row_labels: [Option; 5], pub(super) language_view: Option, pub(super) language_en_radio: Option, pub(super) language_pl_radio: Option, diff --git a/app/ui/onboarding/steps.rs b/app/ui/onboarding/steps.rs index 755db11b..668d99f8 100644 --- a/app/ui/onboarding/steps.rs +++ b/app/ui/onboarding/steps.rs @@ -90,15 +90,24 @@ impl PermissionKind { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum WizardStep { Welcome, + /// First-run operating-lane choice (Basic vs Agentic). Placed right after + /// Welcome so the lane framing is set before the rest of the wizard. + Mode, Permission(PermissionKind), Language, ApiKey, HotkeyMode, + /// Agentic-only readiness verdict (Vibecrafted / AICX / Loctree / PRView). + /// Present in the fixed flow but navigated *around* in the Basic lane — see + /// `actions::step_is_visible`. Keeping it in the array (rather than a + /// mode-dependent flow) preserves stable, resume-safe step indices. + AgenticReadiness, Done, } -pub const STEP_FLOW: [WizardStep; 10] = [ +pub const STEP_FLOW: [WizardStep; 12] = [ WizardStep::Welcome, + WizardStep::Mode, WizardStep::Permission(PermissionKind::Microphone), WizardStep::Permission(PermissionKind::Accessibility), WizardStep::Permission(PermissionKind::InputMonitoring), @@ -107,6 +116,7 @@ pub const STEP_FLOW: [WizardStep; 10] = [ WizardStep::Language, WizardStep::ApiKey, WizardStep::HotkeyMode, + WizardStep::AgenticReadiness, WizardStep::Done, ]; diff --git a/app/ui/onboarding/tests.rs b/app/ui/onboarding/tests.rs index 877b8c88..0dc11708 100644 --- a/app/ui/onboarding/tests.rs +++ b/app/ui/onboarding/tests.rs @@ -4,6 +4,7 @@ use tempfile::TempDir; use crate::config::{Config, UserSettings}; use crate::os::permissions::PermissionStatus; +use super::actions::{next_visible_step, prev_visible_step, step_is_visible}; use super::permission_flow::{ PermissionUiStatus, should_open_settings_after_failed_request, should_refresh_hotkey_runtime_after_grant, should_wait_for_restart, @@ -14,7 +15,9 @@ use super::session::{ }; use super::should_show_onboarding; use super::state::{OnboardingModeChoice, initial_onboarding_mode_choice}; -use super::steps::{PermissionKind, PermissionRecoveryStrategy, WizardStep, step_for_index}; +use super::steps::{ + PermissionKind, PermissionRecoveryStrategy, STEP_FLOW, WizardStep, step_for_index, +}; fn setup_test_env() -> TempDir { let tmp = TempDir::new().expect("tempdir"); @@ -99,6 +102,105 @@ fn onboarding_mode_value_round_trips_through_token() { ); } +fn index_of(target: WizardStep) -> usize { + STEP_FLOW + .iter() + .position(|step| *step == target) + .expect("step present in canonical flow") +} + +/// Walk the wizard forward from Welcome through every lane-visible step. +fn walk_forward(mode: OnboardingModeChoice) -> Vec { + let mut visited = vec![step_for_index(0)]; + let mut idx = 0; + while let Some(next) = next_visible_step(idx, mode) { + visited.push(step_for_index(next)); + idx = next; + } + visited +} + +#[test] +fn mode_step_immediately_follows_welcome() { + for mode in [OnboardingModeChoice::Basic, OnboardingModeChoice::Agentic] { + assert_eq!( + next_visible_step(0, mode).map(step_for_index), + Some(WizardStep::Mode), + "the lane chooser must be the first step after Welcome in {mode:?}" + ); + } +} + +#[test] +fn step_visibility_gates_only_readiness_on_lane() { + assert!(step_is_visible( + WizardStep::AgenticReadiness, + OnboardingModeChoice::Agentic + )); + assert!(!step_is_visible( + WizardStep::AgenticReadiness, + OnboardingModeChoice::Basic + )); + + // Every shared step stays visible in both lanes — only readiness is gated. + for step in [ + WizardStep::Welcome, + WizardStep::Mode, + WizardStep::Permission(PermissionKind::Microphone), + WizardStep::Language, + WizardStep::ApiKey, + WizardStep::HotkeyMode, + WizardStep::Done, + ] { + assert!(step_is_visible(step, OnboardingModeChoice::Basic)); + assert!(step_is_visible(step, OnboardingModeChoice::Agentic)); + } +} + +#[test] +fn basic_lane_flow_skips_agentic_readiness() { + let steps = walk_forward(OnboardingModeChoice::Basic); + assert!(steps.contains(&WizardStep::Mode)); + assert!( + !steps.contains(&WizardStep::AgenticReadiness), + "Basic lane must not route through the agentic readiness step: {steps:?}" + ); + assert_eq!(steps.last(), Some(&WizardStep::Done)); +} + +#[test] +fn agentic_lane_flow_includes_readiness_before_done() { + let steps = walk_forward(OnboardingModeChoice::Agentic); + assert!(steps.contains(&WizardStep::Mode)); + let readiness_pos = steps + .iter() + .position(|step| *step == WizardStep::AgenticReadiness); + let done_pos = steps.iter().position(|step| *step == WizardStep::Done); + assert!( + readiness_pos.is_some(), + "Agentic lane must surface the readiness step: {steps:?}" + ); + assert!( + readiness_pos < done_pos, + "readiness must precede Done in the agentic lane: {steps:?}" + ); +} + +#[test] +fn back_navigation_respects_lane_visibility() { + let done = index_of(WizardStep::Done); + // Going Back from Done lands on HotkeyMode in Basic (readiness skipped) and + // on the readiness step in Agentic. + assert_eq!( + prev_visible_step(done, OnboardingModeChoice::Basic).map(step_for_index), + Some(WizardStep::HotkeyMode) + ); + assert_eq!( + prev_visible_step(done, OnboardingModeChoice::Agentic).map(step_for_index), + Some(WizardStep::AgenticReadiness) + ); +} + fn assert_resume_permission(step: Option, expected: PermissionKind) { assert_eq!( step.map(step_for_index), diff --git a/app/ui/onboarding/widgets.rs b/app/ui/onboarding/widgets.rs index 83cc6d3d..303c9c63 100644 --- a/app/ui/onboarding/widgets.rs +++ b/app/ui/onboarding/widgets.rs @@ -8,7 +8,7 @@ use objc::{msg_send, sel, sel_impl}; use crate::ui::shared::helpers::{ns_string, set_hidden, set_text_field_string}; use super::Id; -use super::state::{HotkeyModeChoice, LanguageChoice, UiRefs}; +use super::state::{HotkeyModeChoice, LanguageChoice, OnboardingModeChoice, UiRefs}; pub(super) fn configure_label(label: Id, centered: bool, multiline: bool) { unsafe { @@ -84,6 +84,17 @@ pub(super) fn sync_language_radios(ui: UiRefs, language: LanguageChoice) { } } +pub(super) fn sync_mode_radios(ui: UiRefs, mode: OnboardingModeChoice) { + unsafe { + if let Some(basic) = ui.mode_basic_radio { + let _: () = msg_send![basic as Id, setState: if mode == OnboardingModeChoice::Basic { 1_isize } else { 0_isize }]; + } + if let Some(agentic) = ui.mode_agentic_radio { + let _: () = msg_send![agentic as Id, setState: if mode == OnboardingModeChoice::Agentic { 1_isize } else { 0_isize }]; + } + } +} + pub(super) fn sync_hotkey_radios(ui: UiRefs, mode: HotkeyModeChoice) { unsafe { if let Some(hold) = ui.hotkey_hold_radio { @@ -112,6 +123,13 @@ pub(super) fn system_red_color() -> Id { } } +pub(super) fn system_orange_color() -> Id { + unsafe { + let ns_color = Class::get("NSColor").unwrap(); + msg_send![ns_color, systemOrangeColor] + } +} + pub(super) fn system_secondary_color() -> Id { unsafe { let ns_color = Class::get("NSColor").unwrap(); diff --git a/app/ui/onboarding/window.rs b/app/ui/onboarding/window.rs index 0c32677e..17bba2c1 100644 --- a/app/ui/onboarding/window.rs +++ b/app/ui/onboarding/window.rs @@ -352,6 +352,40 @@ fn build_onboarding_ui(root: Id, action_handler: Id) -> UiRefs { add_subview(root, instruction_label); ui.instruction_label = Some(instruction_label as usize); + // Operating-lane chooser (Basic vs Agentic). Mirrors the hotkey radio + // group: short descriptive titles, fuller framing in the step + // description. Tags map to OnboardingModeChoice in `onModeSelected:`. + let mode_view: Id = msg_send![ns_view, alloc]; + let mode_view: Id = msg_send![ + mode_view, + initWithFrame: CGRect::new( + &CGPoint::new(content_center - 200.0, 196.0), + &CGSize::new(400.0, 96.0) + ) + ]; + add_subview(root, mode_view); + ui.mode_view = Some(mode_view as usize); + + let mode_basic = create_radio_button( + CGRect::new(&CGPoint::new(0.0, 56.0), &CGSize::new(390.0, 26.0)), + "Basic — local dictation and overlays", + true, + ); + let _: () = msg_send![mode_basic, setTag: 0_isize]; + button_set_action(mode_basic, action_handler, sel!(onModeSelected:)); + add_subview(mode_view, mode_basic); + ui.mode_basic_radio = Some(mode_basic as usize); + + let mode_agentic = create_radio_button( + CGRect::new(&CGPoint::new(0.0, 20.0), &CGSize::new(390.0, 26.0)), + "Agentic — orchestrate agents via Vibecrafted + MCP", + false, + ); + let _: () = msg_send![mode_agentic, setTag: 1_isize]; + button_set_action(mode_agentic, action_handler, sel!(onModeSelected:)); + add_subview(mode_view, mode_agentic); + ui.mode_agentic_radio = Some(mode_agentic as usize); + let language_view: Id = msg_send![ns_view, alloc]; let language_view: Id = msg_send![ language_view, @@ -496,6 +530,56 @@ fn build_onboarding_ui(root: Id, action_handler: Id) -> UiRefs { add_subview(summary_view, summary_config); ui.summary_config_label = Some(summary_config as usize); + // Agentic readiness table (Agentic lane only). One wrapping label per + // McpStatusRow: verdict + Vibecrafted / AICX / Loctree / PRView. Compact + // rows by design (see W1-C3 recovery hint) — each row keeps the probe's + // full actionable value text so prerequisites never collapse into one + // generic error. Render fills text + tone color per step. + const READINESS_HEIGHT: f64 = 280.0; + let readiness_view: Id = msg_send![ns_view, alloc]; + let readiness_view: Id = msg_send![ + readiness_view, + initWithFrame: CGRect::new( + &CGPoint::new(content_left, 56.0), + &CGSize::new(content_width, READINESS_HEIGHT) + ) + ]; + add_subview(root, readiness_view); + ui.readiness_view = Some(readiness_view as usize); + + let readiness_heading = create_label(LabelConfig { + frame: CGRect::new( + &CGPoint::new(0.0, READINESS_HEIGHT - 26.0), + &CGSize::new(content_width, 22.0), + ), + text: "Agentic runtime readiness".to_string(), + font_size: 14.0, + bold: true, + text_color: color_label(), + ..Default::default() + }); + configure_label(readiness_heading, false, false); + add_subview(readiness_view, readiness_heading); + ui.readiness_heading_label = Some(readiness_heading as usize); + + let mut readiness_rows: [Option; 5] = [None; 5]; + let readiness_row_gap = 48.0; + let readiness_first_top = READINESS_HEIGHT - 34.0 - readiness_row_gap; + for (idx, slot) in readiness_rows.iter_mut().enumerate() { + let y = readiness_first_top - (idx as f64 * readiness_row_gap); + let label = create_label(LabelConfig { + frame: CGRect::new(&CGPoint::new(0.0, y), &CGSize::new(content_width, 44.0)), + text: String::new(), + font_size: 12.0, + text_color: color_secondary_label(), + ..Default::default() + }); + configure_label(label, false, true); + add_subview(readiness_view, label); + *slot = Some(label as usize); + } + ui.readiness_row_labels = readiness_rows; + let primary_w = 132.0; let skip_w = 106.0; let back_w = 90.0; From 66dd8ab7c267d9e1102189c5460220334afe2903 Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Fri, 26 Jun 2026 23:57:53 -0700 Subject: [PATCH 018/385] [codex/vc-manual] test(config): preserve legacy LLM host contract --- core/tests/env_validation.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/tests/env_validation.rs b/core/tests/env_validation.rs index 20e38a90..ad598a9c 100644 --- a/core/tests/env_validation.rs +++ b/core/tests/env_validation.rs @@ -111,6 +111,10 @@ fn env_ignores_legacy_llm_host() { let _g3 = EnvGuard::unset("LLM_ENDPOINT"); let mut cfg = Config::default(); + // Isolate the env loader contract: the runtime default endpoint is covered + // by Config::load()/Config::default() tests, while legacy host envs should + // not populate an otherwise-empty llm_endpoint. + cfg.llm_endpoint = None; cfg.load_from_env(); assert!(cfg.llm_endpoint.is_none()); From 390ea74772a0cd54ebbed9f5308c77c958f309ef Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 27 Jun 2026 00:03:40 -0700 Subject: [PATCH 019/385] fix(make): stop target kills full-path codescribe processes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `make stop` used `pkill -f "^codescribe$"`, which only matches a bare `codescribe` argv and missed the real run targets — `/Applications/CodeScribe .app/Contents/MacOS/codescribe` and `~/.cargo/bin/codescribe` — so the app kept running and had to be killed by hand. Match `codescribe$` instead, which covers the bare name and both full paths without touching `qube-daemon`/`qube-report`. Co-Authored-By: Claude Opus 4.8 (1M context) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 260969fc..9e1dc362 100644 --- a/Makefile +++ b/Makefile @@ -163,7 +163,7 @@ start: @echo "CodeScribe started (logs: /tmp/codescribe.log)" stop: - @pkill -f "^codescribe$$" 2>/dev/null || true + @pkill -f "codescribe$$" 2>/dev/null || true @rm -f ~/.codescribe/codescribe.pid 2>/dev/null || true @echo "Stopped" From 0ada61564646638ad3bf558ae189623a6cedf5b4 Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Sat, 27 Jun 2026 11:57:38 -0700 Subject: [PATCH 020/385] [claude/vc-workflow] fix(controller): guard empty tool segment in friendly_tool_name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A raw MCP identifier with a trailing `__` and no tool name (e.g. `mcp__github__`) split to server="github", tool="" and fell through to the `"{tool} · {server}"` formatter, emitting a dangling " · Github" — the separator floating in front of an empty tool label in the Tool Activity timeline. Guard the empty-tool case: fall back to the bare server label, and for the fully degenerate `mcp__` (no server, no tool) prettify the raw rather than return an empty string. Extends the existing unknown-mcp test with the trailing-`__` and degenerate edges so the regression is pinned. Closes the unresolved review thread from PR #33 (malformed MCP tool label); the prior worker fixed GitHub casing only, not this empty-segment edge. Authored-By: claude session_id: 3055637f-6f4f-44c5-97f1-86cd85beeb59 date: 2026-06-27T11:57:03 PDT runtime: terminal --- app/controller/helpers.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/app/controller/helpers.rs b/app/controller/helpers.rs index 5cf16876..8ee246a9 100644 --- a/app/controller/helpers.rs +++ b/app/controller/helpers.rs @@ -489,6 +489,18 @@ pub(crate) fn friendly_tool_name(raw: &str) -> String { let mut parts = rest.splitn(2, "__"); let server = parts.next().unwrap_or(""); let tool = parts.next().unwrap_or(server); + // Trailing `__` with no tool segment (e.g. `mcp__github__`) yields an + // empty `tool`. Without this guard the formatter below emits a dangling + // " · Github" — the separator with nothing in front of it. Fall back to + // the bare server label; if even the server is empty (`mcp__`), prettify + // the raw rather than returning an empty string. + if tool.is_empty() { + return if server.is_empty() { + prettify_identifier(raw) + } else { + prettify_identifier(server) + }; + } let tool_pretty = prettify_identifier(tool); if server.is_empty() || tool == server { return tool_pretty; @@ -1299,6 +1311,15 @@ mod tests { assert_eq!(friendly_tool_name("read_file"), "Read File"); // The raw mcp__ wire form must never survive verbatim. assert!(!friendly_tool_name("mcp__github__create_issue").contains("mcp__")); + // Trailing `__` leaves an empty tool segment (`mcp__github__`). This must + // collapse to the bare server label — never a dangling " · Github" with + // the separator floating in front of nothing. + assert_eq!(friendly_tool_name("mcp__github__"), "Github"); + assert!(!friendly_tool_name("mcp__github__").contains('·')); + assert!(!friendly_tool_name("mcp__github__").starts_with(' ')); + // Fully degenerate `mcp__` (no server, no tool) must not yield an empty + // label either. + assert!(!friendly_tool_name("mcp__").is_empty()); } #[test] From 420888a863eaa00fe09f2aee4bca26c57d7ad9da Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Sat, 27 Jun 2026 12:01:31 -0700 Subject: [PATCH 021/385] [claude/vc-workflow] docs(release): add App Store readiness plan + preflight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Examine→Research→Implement pass for the board-level decision to make the Mac App Store the first-choice lane. Adds a NO-GO-as-single-SKU readiness plan with official Apple citations and a read-only preflight that flags the 3 P0 blockers (no sandbox, no App Store upload path, no privacy manifest) and 6 P1 warnings (incl. the com.codescribe.app vs com.vetcoders.codescribe bundle-id split). No build behavior, entitlements, signing, or PR36 work is touched; sandbox/entitlement changes are deliberately deferred to follow-ups. Authored-By: claude session_id: 77a6977a-4b1a-4abc-ad8f-7ba5bc7862d9 date: 2026-06-27T12:01:31 PDT runtime: terminal --- docs/APP_STORE_READINESS.md | 123 ++++++++++++++++++++++++++++++++ scripts/appstore-preflight.sh | 128 ++++++++++++++++++++++++++++++++++ 2 files changed, 251 insertions(+) create mode 100644 docs/APP_STORE_READINESS.md create mode 100755 scripts/appstore-preflight.sh diff --git a/docs/APP_STORE_READINESS.md b/docs/APP_STORE_READINESS.md new file mode 100644 index 00000000..a1c9cf2b --- /dev/null +++ b/docs/APP_STORE_READINESS.md @@ -0,0 +1,123 @@ +# Mac App Store Readiness + +Status as of `0.12.2` (source) on branch `fix/make-stop-process-match`. + +> **Verdict: NO-GO for the Mac App Store as a single SKU covering the current +> product.** The shipping lane today is **Developer ID + notarization** (outside +> the App Store), and that is the correct lane for the product as it exists. A +> Mac App Store build is feasible only as a **separate, sandbox-clean "Basic" +> SKU** with a materially reduced feature set — and even that carries real App +> Review risk (see Accessibility, below). This document is the plan; it does not +> change the build. + +Run the read-only check any time: + +```bash +./scripts/appstore-preflight.sh # exit 0 = no P0 blockers, 1 = blockers present +``` + +Last run: **3 P0 blockers, 6 P1 warnings**. + +--- + +## Why this matters + +CodeScribe's product direction is a **dictation-driven orchestration agent**, not +a dictation toy. The Agentic mode spawns MCP servers, shells out to external +binaries (Vibecrafted), reads broad file context, controls other apps, and uses +global hotkeys. Those capabilities are the product — and they are exactly what +the App Sandbox forbids. So "ship to the App Store" is not a packaging task; it +is a question of **which product** goes to the store. + +--- + +## The hard constraints (Apple, official) + +| # | Constraint | Source | +|---|------------|--------| +| 1 | **App Sandbox is required** for Mac App Store apps. Builds without `com.apple.security.app-sandbox = true` are rejected at submission. | [App Sandbox](https://developer.apple.com/documentation/security/app-sandbox); rejection text confirmed on [Apple Developer Forums](https://developer.apple.com/forums/thread/41400) | +| 2 | **Developer ID + notarization** is the *outside-the-store* lane; the store lane needs an **Apple Distribution / "3rd Party Mac Developer"** signing identity, a `.pkg` (`productbuild`), and an App Store Connect upload (Transporter). | [Notarizing macOS software](https://developer.apple.com/documentation/security/notarizing-macos-software-before-distribution) | +| 3 | **Privacy manifest (`PrivacyInfo.xcprivacy`)** with declared **required-reason API** usage is mandatory for App Store Connect submissions since **2024-05-01**. | [Privacy updates for App Store submissions](https://developer.apple.com/news/?id=3d8a9yyh), [Reminder: starts May 1](https://developer.apple.com/news/?id=pvszzano) | +| 4 | **App Privacy Details** ("nutrition labels") must be completed in App Store Connect before publishing — including microphone data. | [App Privacy Details](https://developer.apple.com/app-store/app-privacy-details/) | +| 5 | **Apple Events to other apps** under sandbox need a `scripting-targets` entitlement or an `apple-events` **temporary exception** — which is "carefully reviewed, and **most often rejected**" by App Review. | [App Sandbox Temporary Exception Entitlements](https://developer.apple.com/library/archive/documentation/Miscellaneous/Reference/EntitlementKeyReference/Chapters/AppSandboxTemporaryExceptionEntitlements.html), [QA1888](https://developer.apple.com/library/archive/qa/qa1888/_index.html) | +| 6 | **Accessibility used for non-accessibility purposes** (pasting into / driving other apps) is rejected under **Guideline 2.4.5**. | Apple Developer Forums review reports; App Store Review Guidelines 2.4.5 | +| 7 | **Input Monitoring** (CGEventTap *listen-only*, via `CGPreflightListenEventAccess` / `CGRequestListenEventAccess`) **is available to sandboxed Mac App Store apps**. Global-hotkey *detection* can survive sandbox; *controlling other apps* cannot. | [Xojo: Sandboxing to Notarization](https://blog.xojo.com/2024/08/22/macos-apps-from-sandboxing-to-notarization-the-basics/), [Beyond App Sandbox](https://www.appcoda.com/mac-app-sandbox/) | +| 8 | `allow-unsigned-executable-memory` and `allow-jit` are, per Apple's own entitlement docs, **compatible with both the Mac App Store and Developer ID**. The harder conflict is `disable-library-validation`, which fights the sandbox rule that nested code be team-signed. **Mark as uncertain until validated against a real `productbuild` + App Store Connect upload.** | [allow-unsigned-executable-memory](https://developer.apple.com/documentation/bundleresources/entitlements/com.apple.security.cs.allow-unsigned-executable-memory), [disable-library-validation](https://developer.apple.com/documentation/BundleResources/Entitlements/com.apple.security.cs.disable-library-validation) | + +--- + +## Current state vs. Mac App Store requirement + +| Surface | Today (verified in repo) | MAS requirement | Gap | +|---------|--------------------------|-----------------|-----| +| **Sandbox** | `scripts/entitlements.plist` explicitly **disables** App Sandbox (documented as "outside Mac App Store") | `app-sandbox = true` mandatory | **P0** | +| **Entitlements** | `disable-library-validation`, `allow-unsigned-executable-memory`, `allow-dyld-environment-variables` — all required by embedded Whisper/MiniLM dylibs | Sandboxed apps must team-sign nested code; `disable-library-validation` conflicts | **P0/uncertain** | +| **Privacy manifest** | none (`PrivacyInfo.xcprivacy` absent); app uses `std::fs::metadata().modified()` → **FileTimestamp** required-reason category | `PrivacyInfo.xcprivacy` with declared reasons | **P0** | +| **App Privacy Details** | only a written `docs/guide/privacy.md`; no App Store Connect record | Nutrition-label questionnaire completed | **P1** (process, blocked on having an app record) | +| **Purpose strings** | Mic, Accessibility, Input Monitoring, Screen Capture, Apple Events — generated in `Makefile` bundle target (lines 127–131) | Mic + Input Monitoring OK; Accessibility/Apple Events review-risky | **P1** | +| **Basic vs Agentic** | Onboarding has Basic (safe default) + Agentic lanes; Agentic probes MCP readiness | Agentic capabilities are sandbox-incompatible | **architecture** | +| **MCP / Vibecrafted** | Agentic mode shells out, spawns MCP, reads broad files | Forbidden under sandbox | **P0 for Agentic SKU** | +| **Signing/upload** | Developer ID → notarytool → stapler → Gatekeeper (DMG); `.github/workflows/release.yml` | Apple Distribution cert → `.pkg` → App Store Connect | **P0** | +| **Release gates / PRView** | PR35 (release-forward) not yet main-ready; signing secrets unset; live release `v0.8.0` ≪ source `0.12.2` | Green release pipeline | **P1** | +| **Cold install smoke** | DMG drag-install + Gatekeeper drill documented in `PUBLIC_RELEASE_CHECKLIST.md` | TestFlight / store install | **P1** | + +--- + +## Recommended path: two SKUs + +Marked uncertain where Apple's real truth needs a live submission to confirm. + +1. **Developer ID SKU (primary, ships today).** The full product — Agentic + orchestration, MCP, Vibecrafted, global hotkeys, paste-into-other-apps, + embedded ML. Stays exactly as it is. This is where the product lives. + +2. **Mac App Store SKU (Basic, new build profile).** A sandbox-clean dictation + app: microphone → transcript → its own window / clipboard copy. **Drops**: + Accessibility paste, Apple Events focus-restore, Agentic/MCP/Vibecrafted, + broad file access, Full Disk Access probing. **Keeps** (pending validation): + Input Monitoring listen-only hotkeys, embedded Whisper/MiniLM (subject to the + `disable-library-validation` question). Even this SKU risks rejection if the + transcript is delivered by driving another app via Accessibility — keep + delivery in-app or via standard clipboard. + +A **single SKU covering both is not viable**: the sandbox cannot be both on (for +the store) and off (for the agent). Do not attempt one binary for both. + +--- + +## Blockers, ordered + +**P0 — must change before any MAS submission is even possible** + +1. **No App Sandbox** — would require a separate sandboxed build profile; + incompatible with the current feature set. +2. **No App Store distribution path** — no Apple Distribution identity, no + `productbuild`/`.pkg`, no App Store Connect upload step anywhere in the repo. +3. **No privacy manifest** — `PrivacyInfo.xcprivacy` is required and absent; + FileTimestamp required-reason usage is confirmed in code. + +**P1 — required but not the gate** + +4. **Bundle-id split** — `com.codescribe.app` (Makefile/Info.plist) vs + `com.vetcoders.codescribe` (`core/config/keychain.rs:15`, `release.yml:73`). + A store app record needs one canonical id; this split also fragments TCC and + keychain identity today. **Fix is out-of-scope here** (touches the dirty + `Makefile` on this branch) — see follow-up prompt. +5. **Accessibility / Apple Events purpose strings** — review-risky for a store + build; fine for Developer ID. +6. **Release pipeline not green** — PR35 unresolved; signing secrets unset; live + release lags source. (Tracked in `PUBLIC_RELEASE_CHECKLIST.md`.) + +**P2 — only after a sandboxed build exists** + +7. App Privacy Details questionnaire, App Store screenshots/metadata, TestFlight + cold-install smoke. + +--- + +## What this repo change does and does NOT do + +- **Adds** this document and `scripts/appstore-preflight.sh` (read-only check). +- **Does not** enable the sandbox, change entitlements, alter signing, touch the + `Makefile`, or modify any PR36 work. Those are deliberate follow-ups, gated on + the operator's decision to actually stand up a second SKU. diff --git a/scripts/appstore-preflight.sh b/scripts/appstore-preflight.sh new file mode 100755 index 00000000..0087afe4 --- /dev/null +++ b/scripts/appstore-preflight.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# CodeScribe — Mac App Store (MAS) readiness preflight +# +# READ-ONLY. Inspects the repo for state that is known-incompatible with Mac App +# Store distribution and reports each item as a blocker (P0), a warning (P1), or +# OK. It does NOT modify build behavior, entitlements, or any tracked file. +# +# The current shipping lane is Developer ID + notarization (outside the App +# Store). This script does not change that. It exists so an operator can answer +# "how far is the tree from a sandbox-clean MAS build?" without re-deriving it by +# hand. See docs/APP_STORE_READINESS.md for the full plan and citations. +# +# Usage: +# ./scripts/appstore-preflight.sh # human-readable report +# ./scripts/appstore-preflight.sh && echo OK # exit 0 only if no P0 blockers +# +# Exit code: 0 = no P0 blockers found, 1 = at least one P0 blocker. +# +# Created by M&K (c)2026 VetCoders + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(dirname "$SCRIPT_DIR")" +cd "$ROOT" + +ENTITLEMENTS="scripts/entitlements.plist" +MAKEFILE="Makefile" +KEYCHAIN_RS="core/config/keychain.rs" +RELEASE_YML=".github/workflows/release.yml" + +P0=0 +P1=0 + +red() { printf '\033[31m%s\033[0m\n' "$1"; } +green() { printf '\033[32m%s\033[0m\n' "$1"; } +yellow(){ printf '\033[33m%s\033[0m\n' "$1"; } + +blocker() { red " [P0] $1"; P0=$((P0 + 1)); } +warn() { yellow " [P1] $1"; P1=$((P1 + 1)); } +ok() { green " [OK] $1"; } + +echo "═══════════════════════════════════════════════════════════" +echo " CodeScribe — Mac App Store readiness preflight (read-only)" +echo "═══════════════════════════════════════════════════════════" + +# 1. App Sandbox — REQUIRED for MAS, intentionally absent today. +echo "" +echo "▶ App Sandbox entitlement (com.apple.security.app-sandbox)" +if grep -q "com.apple.security.app-sandbox" "$ENTITLEMENTS" 2>/dev/null \ + && grep -A1 "com.apple.security.app-sandbox" "$ENTITLEMENTS" | grep -q ""; then + ok "App Sandbox is enabled." +else + blocker "App Sandbox is NOT enabled in $ENTITLEMENTS. MAS rejects builds without it." +fi + +# 2. Hardened-runtime relaxations that conflict with a clean sandboxed MAS build. +echo "" +echo "▶ Sandbox-hostile hardened-runtime relaxations" +for key in \ + "com.apple.security.cs.disable-library-validation" \ + "com.apple.security.cs.allow-unsigned-executable-memory" \ + "com.apple.security.cs.allow-dyld-environment-variables"; do + if grep -q "$key" "$ENTITLEMENTS" 2>/dev/null; then + warn "$key present (required by embedded ML dylibs; review against MAS — see readiness doc)." + else + ok "$key absent." + fi +done + +# 3. Privacy manifest — required by App Store Connect since 2024-05-01. +echo "" +echo "▶ Privacy manifest (PrivacyInfo.xcprivacy)" +if find . -path ./target -prune -o -name "PrivacyInfo.xcprivacy" -print 2>/dev/null | grep -q .; then + ok "PrivacyInfo.xcprivacy present." +else + blocker "No PrivacyInfo.xcprivacy. Required-reason APIs (FileTimestamp via std::fs metadata) are used; MAS submission needs it." +fi + +# 4. Bundle identifier consistency across the surfaces that must agree. +echo "" +echo "▶ Bundle identifier consistency" +MK_ID="$(grep -E "^CODESCRIBE_BUNDLE_ID \?=" "$MAKEFILE" 2>/dev/null | sed -E 's/.*= *//' | tr -d ' ')" +KC_ID="$(grep -oE "com\.[a-zA-Z0-9.]+codescribe[a-zA-Z0-9.]*" "$KEYCHAIN_RS" 2>/dev/null | head -1)" +YML_ID="$(grep -oE "com\.[a-zA-Z0-9.]+codescribe[a-zA-Z0-9.]*|com\.codescribe\.app" "$RELEASE_YML" 2>/dev/null | head -1)" +echo " Makefile default : ${MK_ID:-}" +echo " keychain.rs : ${KC_ID:-}" +echo " release.yml : ${YML_ID:-}" +if [ -n "$MK_ID" ] && [ "$MK_ID" = "$KC_ID" ] && [ "$MK_ID" = "$YML_ID" ]; then + ok "Bundle id is consistent across Makefile, keychain, and release workflow." +else + warn "Bundle id differs across surfaces. A MAS app record needs one canonical id (TCC + keychain identity)." +fi + +# 5. Apple Events / Accessibility purpose strings — MAS review risk under sandbox. +echo "" +echo "▶ Sandbox-review-sensitive purpose strings (Makefile bundle target)" +if grep -q "NSAppleEventsUsageDescription" "$MAKEFILE" 2>/dev/null; then + warn "NSAppleEventsUsageDescription present. Apple-events temporary exception is 'most often rejected' by App Review." +else + ok "No NSAppleEventsUsageDescription." +fi +if grep -q "NSAccessibilityUsageDescription" "$MAKEFILE" 2>/dev/null; then + warn "NSAccessibilityUsageDescription present. Accessibility for non-accessibility purposes risks Guideline 2.4.5 rejection." +else + ok "No NSAccessibilityUsageDescription." +fi + +# 6. App Store distribution path — none today (Developer ID only). +echo "" +echo "▶ App Store upload path" +if grep -qiE "productbuild|App Store Connect|altool|Transporter|3rd Party Mac Developer|Apple Distribution" "$MAKEFILE" "$RELEASE_YML" 2>/dev/null; then + ok "An App Store distribution path is referenced." +else + blocker "No App Store distribution path (productbuild/pkg, Apple Distribution cert, App Store Connect upload). Only Developer ID + notarization exists." +fi + +echo "" +echo "───────────────────────────────────────────────────────────" +echo " Summary: $P0 P0 blocker(s), $P1 P1 warning(s)" +echo " Lane today: Developer ID + notarization (outside App Store)." +echo " Full plan + Apple citations: docs/APP_STORE_READINESS.md" +echo "───────────────────────────────────────────────────────────" + +if [ "$P0" -gt 0 ]; then + exit 1 +fi +exit 0 From dff0a20339a7ba31d09433b57934724f7d4b2120 Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Sat, 27 Jun 2026 12:48:58 -0700 Subject: [PATCH 022/385] [claude/vc-workflow] docs(release): verify App Store readiness vs live Apple sources + privacy manifest draft ERi re-run: re-checked every Mac App Store constraint against live Apple/official sources (sandbox requirement, MAS vs Developer ID signing+upload path, App Privacy Details, privacy-manifest 2024-05-01 enforcement, FileTimestamp reason code). All core claims held; added the precise FileTimestamp reason code C617.1 mapped to the app's std::fs metadata().modified() usage in history.rs/hf_cache.rs/attachment.rs. Adds scripts/PrivacyInfo.xcprivacy.template as a clearly-marked DRAFT guardrail for the future sandbox-clean Basic SKU (not wired into any build, zero blast radius). Does not enable the sandbox, change entitlements/signing, touch the Makefile, or modify PR36 work. Authored-By: claude session_id: 9c078cd2-268d-4d36-bf2f-26e89bc1ca80 date: 2026-06-27T12:47:25 PDT runtime: terminal --- docs/APP_STORE_READINESS.md | 32 ++++++++++- scripts/PrivacyInfo.xcprivacy.template | 75 ++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 scripts/PrivacyInfo.xcprivacy.template diff --git a/docs/APP_STORE_READINESS.md b/docs/APP_STORE_READINESS.md index a1c9cf2b..be5c5382 100644 --- a/docs/APP_STORE_READINESS.md +++ b/docs/APP_STORE_READINESS.md @@ -52,7 +52,7 @@ is a question of **which product** goes to the store. |---------|--------------------------|-----------------|-----| | **Sandbox** | `scripts/entitlements.plist` explicitly **disables** App Sandbox (documented as "outside Mac App Store") | `app-sandbox = true` mandatory | **P0** | | **Entitlements** | `disable-library-validation`, `allow-unsigned-executable-memory`, `allow-dyld-environment-variables` — all required by embedded Whisper/MiniLM dylibs | Sandboxed apps must team-sign nested code; `disable-library-validation` conflicts | **P0/uncertain** | -| **Privacy manifest** | none (`PrivacyInfo.xcprivacy` absent); app uses `std::fs::metadata().modified()` → **FileTimestamp** required-reason category | `PrivacyInfo.xcprivacy` with declared reasons | **P0** | +| **Privacy manifest** | none (`PrivacyInfo.xcprivacy` absent); app reads file mtimes via `std::fs` `metadata().modified()` in `core/state/history.rs`, `core/hf_cache.rs`, `core/attachment.rs` → **FileTimestamp** required-reason category, reason code **C617.1** (metadata of files in the app's own containers) | `PrivacyInfo.xcprivacy` declaring `NSPrivacyAccessedAPICategoryFileTimestamp` / `C617.1` | **P0** (draft template: `scripts/PrivacyInfo.xcprivacy.template`) | | **App Privacy Details** | only a written `docs/guide/privacy.md`; no App Store Connect record | Nutrition-label questionnaire completed | **P1** (process, blocked on having an app record) | | **Purpose strings** | Mic, Accessibility, Input Monitoring, Screen Capture, Apple Events — generated in `Makefile` bundle target (lines 127–131) | Mic + Input Monitoring OK; Accessibility/Apple Events review-risky | **P1** | | **Basic vs Agentic** | Onboarding has Basic (safe default) + Agentic lanes; Agentic probes MCP readiness | Agentic capabilities are sandbox-incompatible | **architecture** | @@ -115,9 +115,37 @@ the store) and off (for the agent). Do not attempt one binary for both. --- +## Research verification — live Apple sources (2026-06-27) + +The constraints above were re-checked against live Apple/official sources during +an ERi (Examine → Research → Implement) pass. Each core claim held; precision was +added where the first draft was coarse. + +| Claim | Verdict (live) | Source | +|-------|----------------|--------| +| App Sandbox (`com.apple.security.app-sandbox = true`) is required for every Mac App Store app; builds without it are rejected at submission | **Confirmed** | [App Sandbox Entitlement](https://developer.apple.com/documentation/bundleresources/entitlements/com.apple.security.app-sandbox); rejection text on [Apple Developer Forums 41400](https://developer.apple.com/forums/thread/41400) | +| MAS lane needs an **Apple Distribution** signing identity + a **3rd Party Mac Developer Installer** `.pkg` via `productbuild`, uploaded with **Transporter** to App Store Connect — distinct from Developer ID + `notarytool` (outside-store lane; `altool` retired 2023-11-01) | **Confirmed** | [Notarizing macOS software](https://developer.apple.com/documentation/security/notarizing-macos-software-before-distribution), [Distributing software on macOS](https://developer.apple.com/macos/distribution/), [Uploading macOS Builds to App Store Connect (Xojo, 2025)](https://blog.xojo.com/2025/01/14/uploading-macos-builds-to-app-store-connect/) | +| **App Privacy Details** ("nutrition labels") are mandatory to submit new apps/updates, apply to macOS, and require declaring **Audio Data** with purpose + linkage + tracking answers | **Confirmed** | [App Privacy Details](https://developer.apple.com/app-store/app-privacy-details/) | +| **Privacy manifest** (`PrivacyInfo.xcprivacy`) with **required-reason API** declarations is enforced since **2024-05-01**; apps without it are rejected | **Confirmed** | [Privacy updates for App Store submissions](https://developer.apple.com/news/?id=3d8a9yyh), [Reminder: starts May 1](https://developer.apple.com/news/?id=pvszzano) | +| CodeScribe's `metadata().modified()` usage maps to **FileTimestamp** category, reason code **C617.1** (metadata of files in the app's own containers); `DDA9.1` is the alternate (show timestamps to the user, no off-device send) | **Confirmed + made precise** | [NSPrivacyAccessedAPIType](https://developer.apple.com/documentation/bundleresources/app-privacy-configuration/nsprivacyaccessedapitypes/nsprivacyaccessedapitype) | + +**Honest nuance:** the hardest-edged 2024-05-01 gate is scoped most strictly to +*third-party SDKs on Apple's commonly-used list*, but the required-reason +*declaration* obligation also covers an app's own first-party usage of those +APIs. CodeScribe uses a FileTimestamp API directly, so the manifest is required +regardless of SDKs. + +**Still uncertain (needs a real `productbuild` + App Store Connect upload to +settle):** whether `disable-library-validation` (required today by the embedded +Whisper/MiniLM dylibs) can coexist with a sandboxed MAS build, and whether +Input-Monitoring listen-only hotkeys survive App Review in practice. Do not +assert either way from documentation alone. + ## What this repo change does and does NOT do -- **Adds** this document and `scripts/appstore-preflight.sh` (read-only check). +- **Adds** this document, `scripts/appstore-preflight.sh` (read-only check), and + `scripts/PrivacyInfo.xcprivacy.template` (a clearly-marked DRAFT manifest, not + wired into any build — a guardrail/starting point for the future Basic SKU). - **Does not** enable the sandbox, change entitlements, alter signing, touch the `Makefile`, or modify any PR36 work. Those are deliberate follow-ups, gated on the operator's decision to actually stand up a second SKU. diff --git a/scripts/PrivacyInfo.xcprivacy.template b/scripts/PrivacyInfo.xcprivacy.template new file mode 100644 index 00000000..e15abec0 --- /dev/null +++ b/scripts/PrivacyInfo.xcprivacy.template @@ -0,0 +1,75 @@ + + + + + + + NSPrivacyTracking + + + NSPrivacyTrackingDomains + + + + NSPrivacyCollectedDataTypes + + + + NSPrivacyAccessedAPITypes + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategoryFileTimestamp + NSPrivacyAccessedAPITypeReasons + + + C617.1 + + + + + From 1dc5ceeb86332a97a605ad81def2b5c51461798b Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Sat, 27 Jun 2026 13:47:02 -0700 Subject: [PATCH 023/385] =?UTF-8?q?[claude/vc-workflow]=20docs(release):?= =?UTF-8?q?=20App=20Store=20first-choice=20lane=20=E2=80=94=20ERi=20re-fir?= =?UTF-8?q?e=20+=20draft=20Basic-SKU=20entitlements?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Board decision: Mac App Store is now the first-choice distribution lane. Second ERi (Examine -> Research -> Implement) pass re-verified the two points the first readiness draft left "uncertain"; both now resolve toward feasibility: - Embedded Whisper/MiniLM dylibs CAN ship sandboxed: disable-library-validation is only needed because nested dylibs are ad-hoc/foreign-signed. Re-signing them under the app's own team id satisfies library validation AND the sandbox nested-code rule, so the relaxation drops. (TN2206.) - Global hotkeys survive the sandbox: listen-only CGEventTap via Input Monitoring is available to MAS apps; CodeScribe already uses CGEventTap, not NSEvent. Additive, low-blast-radius, no build/sandbox/signing change, PR36 dirty work (Makefile, env_validation.rs) deliberately left untouched: - scripts/entitlements.appstore-basic.plist: DRAFT sandbox-clean entitlement set for the future Basic SKU (sandbox=true, audio-input, no disable-library-validation, no Accessibility/AppleEvents/broad-files). Not wired into any build. - docs/APP_STORE_READINESS.md: ERi re-fire section (research resolutions, exact productbuild/upload command path, true blockers) + FileTimestamp precision fix (stream_postprocess.rs is an additional runtime C617.1 site). - scripts/appstore-preflight.sh: informational check that draft Basic-SKU scaffolding exists (no exit-code change; still 3 P0 / 6 P1). Authored-By: claude session_id: 8113edfc-7bf8-46b3-b5ba-8240bf287956 date: 2026-06-27T13:47:02 PDT runtime: terminal --- docs/APP_STORE_READINESS.md | 115 +++++++++++++++++++++- scripts/appstore-preflight.sh | 14 +++ scripts/entitlements.appstore-basic.plist | 91 +++++++++++++++++ 3 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 scripts/entitlements.appstore-basic.plist diff --git a/docs/APP_STORE_READINESS.md b/docs/APP_STORE_READINESS.md index be5c5382..d09ec673 100644 --- a/docs/APP_STORE_READINESS.md +++ b/docs/APP_STORE_READINESS.md @@ -52,7 +52,7 @@ is a question of **which product** goes to the store. |---------|--------------------------|-----------------|-----| | **Sandbox** | `scripts/entitlements.plist` explicitly **disables** App Sandbox (documented as "outside Mac App Store") | `app-sandbox = true` mandatory | **P0** | | **Entitlements** | `disable-library-validation`, `allow-unsigned-executable-memory`, `allow-dyld-environment-variables` — all required by embedded Whisper/MiniLM dylibs | Sandboxed apps must team-sign nested code; `disable-library-validation` conflicts | **P0/uncertain** | -| **Privacy manifest** | none (`PrivacyInfo.xcprivacy` absent); app reads file mtimes via `std::fs` `metadata().modified()` in `core/state/history.rs`, `core/hf_cache.rs`, `core/attachment.rs` → **FileTimestamp** required-reason category, reason code **C617.1** (metadata of files in the app's own containers) | `PrivacyInfo.xcprivacy` declaring `NSPrivacyAccessedAPICategoryFileTimestamp` / `C617.1` | **P0** (draft template: `scripts/PrivacyInfo.xcprivacy.template`) | +| **Privacy manifest** | none (`PrivacyInfo.xcprivacy` absent); app reads file mtimes via `std::fs` `metadata().modified()` in `core/state/history.rs`, `core/hf_cache.rs`, `core/attachment.rs`, **and runtime `core/pipeline/stream_postprocess.rs`** (lexicon mtime in config dir) → **FileTimestamp** required-reason category, reason code **C617.1** (metadata of files in the app's own containers) | `PrivacyInfo.xcprivacy` declaring `NSPrivacyAccessedAPICategoryFileTimestamp` / `C617.1` | **P0** (draft template: `scripts/PrivacyInfo.xcprivacy.template`) | | **App Privacy Details** | only a written `docs/guide/privacy.md`; no App Store Connect record | Nutrition-label questionnaire completed | **P1** (process, blocked on having an app record) | | **Purpose strings** | Mic, Accessibility, Input Monitoring, Screen Capture, Apple Events — generated in `Makefile` bundle target (lines 127–131) | Mic + Input Monitoring OK; Accessibility/Apple Events review-risky | **P1** | | **Basic vs Agentic** | Onboarding has Basic (safe default) + Agentic lanes; Agentic probes MCP readiness | Agentic capabilities are sandbox-incompatible | **architecture** | @@ -149,3 +149,116 @@ assert either way from documentation alone. - **Does not** enable the sandbox, change entitlements, alter signing, touch the `Makefile`, or modify any PR36 work. Those are deliberate follow-ups, gated on the operator's decision to actually stand up a second SKU. + +--- + +## ERi re-fire — board decision: App Store is the first-choice lane (2026-06-27) + +The operator has made a **board-level product decision**: the Mac App Store is +now the **first-choice distribution lane** for CodeScribe. That does not reverse +the technical verdict above — a single binary still cannot be both sandboxed (for +the store) and un-sandboxed (for the agent). What it changes is the *action*: stop +treating the Basic MAS SKU as a "maybe later" and treat it as the **primary lane +to stand up**, with Developer ID/Agentic as the secondary power-user lane. + +A second ERi pass re-verified the two points the first draft left explicitly +"uncertain". Both are now **resolved in favour of feasibility** — the Basic SKU is +technically viable, not merely conceivable. + +### Resolved uncertainty 1 — embedded ML dylibs CAN ship sandboxed + +`disable-library-validation` is set today **only because the embedded Whisper / +MiniLM dylibs are signed ad-hoc / under a foreign team**. Apple's library-validation +policy lets a binary link any library signed with **the same team identifier** (or +an Apple system library). So the fix is not an entitlement — it is **re-signing +every nested dylib/framework under the app's own team ID at bundle time**. Do that +and library validation is satisfied *and* the sandbox's "nested code must be +team-signed" rule is met, with `disable-library-validation` removed entirely. + +- Library validation / nested-code signing: [TN2206: macOS Code Signing In Depth](https://developer.apple.com/library/archive/technotes/tn2206/_index.html) +- `allow-unsigned-executable-memory` is MAS-compatible (keep only if a launch test shows the ML runtime needs JIT): [allow-unsigned-executable-memory](https://developer.apple.com/documentation/bundleresources/entitlements/com.apple.security.cs.allow-unsigned-executable-memory) + +**Still requires a real `productbuild` + App Store Connect upload to fully settle** — +documentation establishes the rule; only a live upload proves the embedded ML stack +loads under the sandbox. + +### Resolved uncertainty 2 — global hotkeys survive the sandbox + +Listen-only **Input Monitoring** (`CGEventTap` via `CGPreflightListenEventAccess` / +`CGRequestListenEventAccess`) **is available to sandboxed Mac App Store apps**. It +uses the runtime *Input Monitoring* privilege, not the Accessibility privilege that +the sandbox blocks. CodeScribe already detects hotkeys with `CGEventTap` (not the +Accessibility-bound `NSEvent` monitor), so **hotkey detection carries into the +Basic SKU unchanged**. What does *not* carry is *controlling other apps* (paste, +focus-restore) — that needs Accessibility / Apple Events, which stay in the +Developer ID SKU. + +- Source: Apple confirms `CGEventTap` + Input Monitoring works in sandbox where `NSEvent`/Accessibility does not — see [Apple Developer Forums: Accessibility permission in sandboxed app](https://developer.apple.com/forums/thread/707680) and the [App Sandbox Information upload reference](https://developer.apple.com/help/app-store-connect/reference/app-uploads/app-sandbox-information). + +### Required-reason API scope is widening (re-confirmed) + +The 2024-05-01 gate began with third-party SDKs, but Apple has stated the +required-reason obligation **will expand to the entire app binary**, and since +**2025-02-12** a new privacy-impacting SDK must ship a manifest. CodeScribe uses a +FileTimestamp API directly in first-party code, so `PrivacyInfo.xcprivacy` is +required regardless of SDKs — do not defer it. + +- [Privacy updates for App Store submissions](https://developer.apple.com/news/?id=3d8a9yyh) · [List of APIs that require declared reasons](https://developer.apple.com/news/?id=z6fu1dcu) · [TN3183: required reason API entries](https://developer.apple.com/documentation/technotes/tn3183-adding-required-reason-api-entries-to-your-privacy-manifest) + +### What this re-fire adds to the repo + +- **`scripts/entitlements.appstore-basic.plist`** — a clearly-marked **DRAFT** + sandbox-clean entitlement set for the Basic SKU: `app-sandbox = true`, + `device.audio-input`, **no** `disable-library-validation`, **no** + Accessibility / Apple Events / broad file access. Not wired into any build — + the starting point for the MAS build profile. Sibling to the shipping + `scripts/entitlements.plist` (Developer ID). +- This section + the FileTimestamp precision fix (`stream_postprocess.rs` is an + additional runtime FileTimestamp site the first draft missed). +- **No** Makefile / signing / sandbox changes; PR36's dirty work is untouched. + +### Concrete path to the Basic MAS SKU (commands the operator will run later) + +These are the *documented target commands*, not run by this pass (they need an +Apple Distribution identity and a sandboxed build profile that do not exist yet): + +```bash +# 0. Pre-req certs (operator, in Apple Developer portal — NOT done here): +# - "Apple Distribution" certificate +# - "Mac Installer Distribution" (3rd Party Mac Developer Installer) certificate +# - App record + bundle id in App Store Connect (pick ONE canonical id) + +# 1. Build the Basic profile (future Makefile target, sandbox-clean feature set). +# Sign the app AND re-sign each nested ML dylib under the SAME team id: +codesign --force --options runtime \ + --entitlements scripts/entitlements.appstore-basic.plist \ + --sign "Apple Distribution: ()" \ + CodeScribe.app/Contents/MacOS/ # then the .app last + +# 2. Build the installer package for the store: +productbuild --component CodeScribe.app /Applications \ + --sign "3rd Party Mac Developer Installer: ()" \ + CodeScribe.pkg + +# 3. Validate + upload to App Store Connect (Transporter app or notarytool's +# successor for store delivery; altool was retired 2023-11-01): +xcrun altool --validate-app -f CodeScribe.pkg -t macos ... # or Transporter.app + +# 4. In App Store Connect: complete App Privacy Details ("nutrition labels"): +# Audio Data -> purpose "App Functionality", NOT linked to identity, NOT tracking. + +# Read-only gap check any time (this DOES run today): +./scripts/appstore-preflight.sh +``` + +### Remaining true blockers (unchanged by this pass, ordered) + +- **P0** Stand up the sandboxed Basic build profile (new Makefile target + + `entitlements.appstore-basic.plist` wired in + nested-dylib re-signing). +- **P0** Acquire Apple Distribution + Mac Installer Distribution certificates and + add a `productbuild` + App Store Connect upload step (none exist in the repo). +- **P0** Ship `PrivacyInfo.xcprivacy` inside the bundle (template ready). +- **P1** Resolve the bundle-id split to one canonical id (touches the dirty + `Makefile` — out of scope on this branch; see follow-up prompts). +- **P1** Confirm the embedded ML stack actually loads sandboxed via a real + `productbuild` + upload (documentation-confident, not upload-proven). diff --git a/scripts/appstore-preflight.sh b/scripts/appstore-preflight.sh index 0087afe4..197cd240 100755 --- a/scripts/appstore-preflight.sh +++ b/scripts/appstore-preflight.sh @@ -115,6 +115,20 @@ else blocker "No App Store distribution path (productbuild/pkg, Apple Distribution cert, App Store Connect upload). Only Developer ID + notarization exists." fi +# 7. Basic-SKU draft scaffolding — informational (does not affect exit code). +echo "" +echo "▶ Basic SKU draft scaffolding (informational)" +if [ -f "scripts/entitlements.appstore-basic.plist" ]; then + ok "Draft sandbox entitlements present (scripts/entitlements.appstore-basic.plist)." +else + yellow " [info] No draft sandbox entitlements (scripts/entitlements.appstore-basic.plist)." +fi +if [ -f "scripts/PrivacyInfo.xcprivacy.template" ]; then + ok "Draft privacy manifest present (scripts/PrivacyInfo.xcprivacy.template)." +else + yellow " [info] No draft privacy manifest template." +fi + echo "" echo "───────────────────────────────────────────────────────────" echo " Summary: $P0 P0 blocker(s), $P1 P1 warning(s)" diff --git a/scripts/entitlements.appstore-basic.plist b/scripts/entitlements.appstore-basic.plist new file mode 100644 index 00000000..bc6d2efa --- /dev/null +++ b/scripts/entitlements.appstore-basic.plist @@ -0,0 +1,91 @@ + + + + + + + com.apple.security.app-sandbox + + + + com.apple.security.device.audio-input + + + + com.apple.security.cs.allow-unsigned-executable-memory + + + + + From 670022861b30e2f57ac43c108753543686835f89 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 27 Jun 2026 19:20:12 -0700 Subject: [PATCH 024/385] fix(vad): share Silero ONNX session process-wide to stop per-recording leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The embedded Silero VAD session was rebuilt on every recording (`SileroVad::new_embedded` → `Session::commit_from_memory`). Over a long run this fired hundreds of times (378× in one 15h session, with zero unloads), and ORT arena allocations do not return to the OS — driving phys_footprint to ~5GB (peak 10GB) while RSS stayed at ~560MB. The Silero graph is stateless between calls: the recurrent `state` and the 64-sample `context` live in `SileroVad` and are passed in/out as tensors. So we now compile the embedded graph once into a process-wide `OnceLock>>` and hand every instance a clone. The Mutex is required because `ort::Session::run` takes `&mut self`; serializing the short VAD inferences is harmless and correct (per-stream state is untouched). The path-based `new` loader (dev/test override) keeps its own session. Embedder and Whisper already use this singleton pattern; this brings Silero in line. Tests: embedded session is shared across instances (Arc::ptr_eq); shared session keeps independent per-instance state across predict + reset. Co-Authored-By: Claude Opus 4.8 (1M context) --- core/vad/silero_ort.rs | 156 ++++++++++++++++++++++++++++++++--------- 1 file changed, 121 insertions(+), 35 deletions(-) diff --git a/core/vad/silero_ort.rs b/core/vad/silero_ort.rs index 5e404d90..45b82019 100644 --- a/core/vad/silero_ort.rs +++ b/core/vad/silero_ort.rs @@ -4,6 +4,7 @@ //! Model: silero_vad.onnx v6 from https://github.com/snakers4/silero-vad use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; use anyhow::{Context, Result}; use ndarray::ArrayD; @@ -18,6 +19,42 @@ mod embedded { include!(concat!(env!("OUT_DIR"), "/embedded_vad_data.rs")); } +/// Process-wide shared Silero ONNX session for the embedded model. +/// +/// The Silero graph is **stateless** between calls — the recurrent state +/// (`state`) and the 64-sample `context` live in each [`SileroVad`] instance +/// and are passed in/out as tensors per inference. Only the compiled graph is +/// heavy (ORT arena + graph optimizer scratch), and rebuilding it per recording +/// leaked native memory (the session is created hundreds of times over a long +/// run and ORT arenas do not return to the OS). We therefore build it **once** +/// and share it behind a `Mutex` (ORT's `Session::run` takes `&mut self`). +static EMBEDDED_SESSION: OnceLock>> = OnceLock::new(); + +/// Build (once) and return a handle to the shared embedded Silero session. +/// +/// First call compiles the graph from the embedded bytes and caches it; every +/// subsequent call clones the `Arc` — no reload, no extra native allocation. +fn embedded_session() -> Result>> { + // Fast path: already initialized. + if let Some(existing) = EMBEDDED_SESSION.get() { + return Ok(Arc::clone(existing)); + } + // Build outside the OnceLock so a failure isn't cached as a poisoned slot. + info!( + "Loading Silero VAD model from embedded bytes ({} bytes) — shared once, reused", + embedded::MODEL.len() + ); + let session = Session::builder()? + .with_intra_threads(1)? + .commit_from_memory(embedded::MODEL) + .context("Failed to load embedded Silero VAD ONNX model")?; + debug!("Silero VAD model loaded successfully (embedded, shared)"); + // Race-safe: if another thread won the init, drop ours and use theirs. + let handle = Arc::new(Mutex::new(session)); + let stored = EMBEDDED_SESSION.get_or_init(|| handle); + Ok(Arc::clone(stored)) +} + /// Silero VAD sample rate (always 16kHz) pub const VAD_SAMPLE_RATE: u32 = 16000; @@ -89,7 +126,11 @@ impl Resampler { /// - Output names: `output`, `stateN` (v4: positional) /// - Context window: 64 samples prepended to each 512-sample chunk pub struct SileroVad { - session: Session, + /// Shared, compiled Silero graph. The embedded production path hands out a + /// clone of one process-wide session (see [`embedded_session`]); the legacy + /// path-based loader builds its own. The graph is stateless across calls, so + /// sharing it behind a `Mutex` is safe — per-stream state lives below. + session: Arc>, state: ArrayD, context: Vec, config: VadConfig, @@ -110,7 +151,9 @@ impl SileroVad { debug!("Silero VAD model loaded successfully"); Ok(Self { - session, + // Path-based loads are dev/test overrides with a caller-chosen file, + // so they are not cached — each gets its own session. + session: Arc::new(Mutex::new(session)), state: ArrayD::zeros(STATE_SHAPE.as_slice()), context: vec![0.0; CONTEXT_SIZE], config, @@ -119,17 +162,13 @@ impl SileroVad { } /// Load Silero VAD model from embedded bytes (production path, zero I/O). + /// + /// The compiled ONNX graph is shared process-wide (built once, cloned + /// thereafter) — only the lightweight per-stream state is allocated here. + /// This is the hot path: it runs once per recording, so rebuilding the graph + /// every time previously leaked native ORT memory over long sessions. pub(crate) fn new_embedded(config: VadConfig) -> Result { - info!( - "Loading Silero VAD model from embedded bytes ({} bytes)", - embedded::MODEL.len() - ); - let session = Session::builder()? - .with_intra_threads(1)? - .commit_from_memory(embedded::MODEL) - .context("Failed to load embedded Silero VAD ONNX model")?; - - debug!("Silero VAD model loaded successfully (embedded)"); + let session = embedded_session()?; Ok(Self { session, @@ -200,31 +239,45 @@ impl SileroVad { let state_value = Value::from_array(state)?; let sr_value = Value::from_array(sr)?; - let outputs = self.session.run([ - (&input_value).into(), - (&state_value).into(), - (&sr_value).into(), - ])?; - - // Read probability from "output" (safe access — no panic on missing key) - let prob = { - let output = outputs - .get("output") - .context("Silero model missing 'output' tensor")?; - let (_shape, data) = output.try_extract_tensor::()?; - data.first().copied().unwrap_or(0.0) + // Lock the shared session only for the inference call. The graph is + // stateless across runs, so serializing concurrent streams here is + // correct — each keeps its own `state`/`context`. Extract everything we + // need from `outputs` (which borrows the guard) before the guard drops. + let (prob, next_state) = { + let mut session = self + .session + .lock() + .map_err(|e| anyhow::anyhow!("Silero session lock poisoned: {e}"))?; + let outputs = session.run([ + (&input_value).into(), + (&state_value).into(), + (&sr_value).into(), + ])?; + + // Read probability from "output" (safe access — no panic on missing key) + let prob = { + let output = outputs + .get("output") + .context("Silero model missing 'output' tensor")?; + let (_shape, data) = output.try_extract_tensor::()?; + data.first().copied().unwrap_or(0.0) + }; + + // Read updated state from "stateN" (safe access — no panic on missing key) + let next_state = { + let state_output = outputs + .get("stateN") + .context("Silero model missing 'stateN' tensor")?; + let (shape, data) = state_output.try_extract_tensor::()?; + let shape_usize: Vec = shape.as_ref().iter().map(|&d| d as usize).collect(); + ArrayD::from_shape_vec(shape_usize.as_slice(), data.to_vec()).ok() + }; + + (prob, next_state) }; - // Read updated state from "stateN" (safe access — no panic on missing key) - { - let state_output = outputs - .get("stateN") - .context("Silero model missing 'stateN' tensor")?; - let (shape, data) = state_output.try_extract_tensor::()?; - let shape_usize: Vec = shape.as_ref().iter().map(|&d| d as usize).collect(); - if let Ok(arr) = ArrayD::from_shape_vec(shape_usize.as_slice(), data.to_vec()) { - self.state = arr; - } + if let Some(arr) = next_state { + self.state = arr; } Ok(prob) @@ -452,4 +505,37 @@ mod tests { let vad = AccumulatingVad::new(16000); assert!(vad.is_ok(), "embedded VAD must load: {:?}", vad.err()); } + + #[test] + fn embedded_session_is_shared_across_instances() { + // The core of the memory fix: two embedded VAD instances must point at + // ONE underlying ONNX session (graph built once, cloned thereafter), + // instead of each rebuilding it and leaking native ORT memory. + let a = SileroVad::new_embedded(VadConfig::default()).expect("first embedded VAD"); + let b = SileroVad::new_embedded(VadConfig::default()).expect("second embedded VAD"); + assert!( + Arc::ptr_eq(&a.session, &b.session), + "embedded Silero session must be shared, not rebuilt per instance" + ); + } + + #[test] + fn shared_session_keeps_per_instance_state() { + // Sharing the session must not couple per-stream recurrent state: each + // instance owns its `state`/`context`, so both predict correctly and a + // reset on one does not disturb the other. + let mut a = SileroVad::new_embedded(VadConfig::default()).expect("VAD a"); + let mut b = SileroVad::new_embedded(VadConfig::default()).expect("VAD b"); + + let chunk = vec![0.0f32; CHUNK_SIZE]; + let pa = a.predict(&chunk).expect("predict a"); + let pb = b.predict(&chunk).expect("predict b"); + assert!((0.0..=1.0).contains(&pa), "prob a in range: {pa}"); + assert!((0.0..=1.0).contains(&pb), "prob b in range: {pb}"); + + // Reset one stream; the other must keep working on the shared session. + a.reset(); + let pb2 = b.predict(&chunk).expect("predict b after a.reset"); + assert!((0.0..=1.0).contains(&pb2), "prob b2 in range: {pb2}"); + } } From 5001549b524bf389f2f65850d44f8cb6a88d94ec Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 27 Jun 2026 20:48:39 -0700 Subject: [PATCH 025/385] fix(memory): return freed heap pages to the OS after each recording MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live measurement showed that after a few recordings ~700MB of MALLOC_LARGE (plus reclaimable MALLOC_SMALL) was freed-but-retained — only ~37MB was actually live. The system allocator caches freed dirty pages for reuse instead of returning them to the kernel, so phys_footprint creeps up across a long session even though nothing leaks. Add `codescribe_core::memory::release_freed_heap()`, a thin macOS FFI to `malloc_zone_pressure_relief(NULL, 0)`, and call it once per recording at the quiescent "State reset to IDLE" point in the controller. No-op off macOS. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/controller/mod.rs | 6 +++++ core/lib.rs | 1 + core/memory.rs | 51 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 core/memory.rs diff --git a/app/controller/mod.rs b/app/controller/mod.rs index 72fd3d07..1d8abf38 100644 --- a/app/controller/mod.rs +++ b/app/controller/mod.rs @@ -1632,6 +1632,12 @@ impl RecordingController { crate::ui::voice_chat::update_voice_chat_status(final_status); info!("Processing finished successfully. State reset to IDLE."); + // The transcription just freed large transient buffers (audio, + // mel, model scratch). Hand those freed-but-retained pages back + // to the OS now, while idle, instead of letting phys_footprint + // creep up across a long session. + codescribe_core::memory::release_freed_heap(); + if let Some(reason) = outcome.no_speech_reason.as_deref() { info!("NoSpeech outcome in finish_recording: reason={reason}"); if !assistive { diff --git a/core/lib.rs b/core/lib.rs index 89c5559f..83b2a61f 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -48,6 +48,7 @@ mod hf_cache; pub mod ipc; pub mod llm; pub mod mcp; +pub mod memory; pub mod pipeline; pub mod quality; pub mod state; diff --git a/core/memory.rs b/core/memory.rs new file mode 100644 index 00000000..b4e9b1a9 --- /dev/null +++ b/core/memory.rs @@ -0,0 +1,51 @@ +//! Process memory hygiene helpers. +//! +//! After a recording's transcription completes, large transient buffers (audio, +//! mel spectrograms, candle/ORT scratch) are freed — but the system allocator +//! keeps the dirty pages cached for reuse instead of returning them to the OS. +//! Measured on a fresh run: ~700 MB of freed-but-retained `MALLOC_LARGE` after +//! only three recordings (37 MB actually live), inflating `phys_footprint` +//! toward the multi-GB figures users see in Activity Monitor. This module asks +//! the allocator to give that memory back at natural quiescent points. + +/// Ask the system allocator to return freed-but-retained pages to the OS. +/// +/// On macOS this calls `malloc_zone_pressure_relief(NULL, 0)`, which madvises +/// free pages in every malloc zone back to the kernel. It is safe to call at +/// any time; the cost is a scan of free regions, so call it at natural +/// quiescent points (e.g. once after each recording finishes), never in a hot +/// loop. No-op on non-macOS targets. +pub fn release_freed_heap() { + #[cfg(target_os = "macos")] + { + // SAFETY: FFI to a stable libmalloc entry point. A NULL zone means + // "all zones" and a goal of 0 means "release as much as possible". + // Returns the number of bytes handed back to the OS. + let released = unsafe { malloc_zone_pressure_relief(std::ptr::null_mut(), 0) }; + tracing::debug!("release_freed_heap: allocator returned {released} bytes to the OS"); + } +} + +#[cfg(target_os = "macos")] +unsafe extern "C" { + /// `size_t malloc_zone_pressure_relief(malloc_zone_t *zone, size_t goal);` + /// from ``. Not exposed by the `libc` crate as a function, + /// so we declare it directly. + fn malloc_zone_pressure_relief(zone: *mut core::ffi::c_void, goal: usize) -> usize; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn release_freed_heap_is_callable_and_safe() { + // Allocate and drop a large buffer so there is something to reclaim, + // then ensure the call does not panic (and is a no-op off macOS). + let big = vec![0u8; 32 * 1024 * 1024]; + let len = big.len(); + drop(big); + assert_eq!(len, 32 * 1024 * 1024); + release_freed_heap(); + } +} From 2b8bb1fbe5ca560b1da81a981e6e56711f19a302 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 27 Jun 2026 20:54:25 -0700 Subject: [PATCH 026/385] feat(whisper): unload idle Whisper engine to free ~3GB GPU memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whisper large-v3-turbo lives on the Metal GPU and is the single largest memory consumer — measured ~3GB resident (IOAccelerator) the moment it loads, held for the whole process lifetime even when idle. Make the engine unloadable: replace the `OnceLock>` singleton with a resettable `Mutex<{ engine: Option, last_used: Instant }>`. A background reaper drops the engine after a configurable idle period (CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS, default 300s, 0 disables), returning GPU/host memory to the system; the next transcription transparently reloads it (~6s). All engine access now goes through `with_engine`/`try_with_engine` closures that lazily (re)load, refresh the idle clock, and serialize on the single mutex exactly as before — so the reaper can never drop an engine mid-transcription. External callers (stt::mod candle_* helpers, offline batch) move to the new singleton functions; the public transcribe/init/is_initialized API is preserved (is_initialized now reflects current load state). Co-Authored-By: Claude Opus 4.8 (1M context) --- core/pipeline/streaming/offline.rs | 14 +- core/stt/mod.rs | 21 +-- core/stt/whisper/singleton.rs | 235 ++++++++++++++++++++++------- 3 files changed, 195 insertions(+), 75 deletions(-) diff --git a/core/pipeline/streaming/offline.rs b/core/pipeline/streaming/offline.rs index dc519873..b1d0b172 100644 --- a/core/pipeline/streaming/offline.rs +++ b/core/pipeline/streaming/offline.rs @@ -2,12 +2,12 @@ //! session path; compiled only for tests and the `offline_eval` feature //! (the module is cfg-gated in `mod.rs`). -use anyhow::{Result, anyhow}; +use anyhow::Result; use tracing::{debug, info}; use crate::pipeline::dedup::dedup_chunk_overlap; use crate::pipeline::stream_postprocess::StreamPostProcessor; -use crate::stt::whisper::singleton::engine as get_engine; +use crate::stt::whisper::singleton::transcribe_chunk; use super::tuning::{env_bool_default, env_f32}; @@ -53,11 +53,9 @@ pub fn transcribe_streaming_samples( effective_audio_sec ); - let engine_mutex = get_engine()?; - let mut engine = engine_mutex - .lock() - .map_err(|e| anyhow!("Lock error: {}", e))?; - + // Per-chunk engine acquisition via the singleton: chunks are independent + // (overlap dedup happens on the text below), and routing through the + // singleton keeps the idle-unload/reload bookkeeping consistent. let mut out = String::new(); let mut offset = 0usize; let mut chunks_processed = 0usize; @@ -68,7 +66,7 @@ pub fn transcribe_streaming_samples( let chunk = &samples[offset..end]; let chunk_sec = chunk.len() as f32 / sample_rate as f32; let t_chunk = std::time::Instant::now(); - let text = engine.transcribe_with_language(chunk, sample_rate, language)?; + let text = transcribe_chunk(chunk, sample_rate, language)?; let chunk_ms = t_chunk.elapsed().as_millis(); chunks_processed += 1; diff --git a/core/stt/mod.rs b/core/stt/mod.rs index a26385f2..38068214 100644 --- a/core/stt/mod.rs +++ b/core/stt/mod.rs @@ -108,11 +108,9 @@ fn candle_transcribe_chunk( sample_rate: u32, language: Option<&str>, ) -> anyhow::Result { - let engine = whisper::singleton::engine()?; - let mut guard = engine - .lock() - .map_err(|e| anyhow::anyhow!("Candle lock error: {}", e))?; - guard.transcribe_with_language(audio, sample_rate, language) + // Engine acquisition + idle-clock refresh + lazy (re)load live in the + // singleton now, so it can unload Whisper when idle and reload on demand. + whisper::singleton::transcribe_chunk(audio, sample_rate, language) } fn candle_transcribe_long_with_segments( @@ -120,11 +118,7 @@ fn candle_transcribe_long_with_segments( sample_rate: u32, language: Option<&str>, ) -> anyhow::Result { - let engine = whisper::singleton::engine()?; - let mut guard = engine - .lock() - .map_err(|e| anyhow::anyhow!("Candle lock error: {}", e))?; - guard.transcribe_long_with_language_segments(audio, sample_rate, language) + whisper::singleton::transcribe_with_segments(audio, sample_rate, language) } #[allow(dead_code)] @@ -133,11 +127,8 @@ fn candle_try_transcribe_long_with_segments( sample_rate: u32, language: Option<&str>, ) -> anyhow::Result { - let engine = whisper::singleton::engine()?; - let mut guard = engine - .try_lock() - .map_err(|_| anyhow::anyhow!("Whisper engine busy, skipping correction"))?; - guard.transcribe_long_with_language_segments(audio, sample_rate, language) + // Non-blocking acquisition: skip the correction pass if the engine is busy. + whisper::singleton::try_transcribe_with_segments(audio, sample_rate, language) } /// Initialize whichever STT engine is active by env. diff --git a/core/stt/whisper/singleton.rs b/core/stt/whisper/singleton.rs index 71ade56f..4d0a7ef3 100644 --- a/core/stt/whisper/singleton.rs +++ b/core/stt/whisper/singleton.rs @@ -3,14 +3,24 @@ //! The canonical product path is an embedded Whisper payload built into the //! binary. Runtime lookup remains as a fallback for explicit no-embed builds, //! developer overrides, and recovery when the payload is unavailable. +//! +//! ## Idle unload +//! +//! The Whisper model lives on the GPU (Metal) and is by far the largest single +//! memory consumer (~3 GB resident). Keeping it loaded across long idle periods +//! wastes that memory, so the engine is held in a *resettable* slot: after a +//! configurable idle period with no transcription a background reaper drops it, +//! returning the GPU/host memory to the system, and the next call transparently +//! reloads it. Set `CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS=0` to disable. // This entire module is a public API for library consumers use std::path::PathBuf; use std::sync::{Mutex, OnceLock}; +use std::time::{Duration, Instant}; use anyhow::{Context, Result, anyhow}; -use tracing::info; +use tracing::{info, warn}; use crate::config::Config; use crate::config::models::resolve_runtime_whisper_model_path; @@ -22,12 +32,47 @@ use super::params::DecodingParams; /// Default model name (for dev/fallback mode) pub use crate::config::models::DEFAULT_MODEL; -/// Global singleton engine -static ENGINE: OnceLock> = OnceLock::new(); +/// Default idle period after which the Whisper engine is unloaded to free GPU +/// memory. Overridable via `CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS` (0 disables). +const DEFAULT_IDLE_UNLOAD_SECS: u64 = 300; + +/// How often the reaper wakes to check for idleness. +const REAPER_TICK: Duration = Duration::from_secs(30); + +/// Resettable engine slot: `None` when unloaded, plus the last-use timestamp the +/// reaper consults. A single `Mutex` serializes loads, transcriptions, and +/// unloads — exactly as the previous `Mutex` did. +struct WhisperSlot { + engine: Option, + last_used: Instant, +} + +static SLOT: OnceLock> = OnceLock::new(); /// Runtime model path used only when embedded provisioning is unavailable. static MODEL_PATH: OnceLock = OnceLock::new(); +/// Guard so the idle reaper thread is spawned at most once. +static REAPER_STARTED: OnceLock<()> = OnceLock::new(); + +fn slot() -> &'static Mutex { + SLOT.get_or_init(|| { + Mutex::new(WhisperSlot { + engine: None, + last_used: Instant::now(), + }) + }) +} + +/// Resolve the configured idle-unload period, or `None` when disabled (0). +fn idle_unload_after() -> Option { + let secs = std::env::var("CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS") + .ok() + .and_then(|s| s.trim().parse::().ok()) + .unwrap_or(DEFAULT_IDLE_UNLOAD_SECS); + (secs > 0).then(|| Duration::from_secs(secs)) +} + /// Resolve the model path for runtime Whisper fallback loading. fn resolve_model_path_fallback() -> Result { let config = Config::load(); @@ -53,50 +98,116 @@ pub fn get_model_path() -> Result<&'static PathBuf> { .ok_or_else(|| anyhow!("Failed to store model path")) } -/// Initialize the global engine (call once at startup). -/// -/// Embedded Whisper is the product-default truth. Runtime path resolution is a -/// deliberate fallback for no-embed builds and local recovery. -pub fn init() -> Result<()> { +/// Build a fresh engine, embedded-first with a runtime-path fallback. +fn load_engine() -> Result { // 1. Primary shipped path: embedded Whisper payload. if let Some(embedded) = super::embedded::get_embedded_data() { let engine = LocalWhisperEngine::from_embedded(&embedded) .context("Failed to initialize from embedded model")?; - - ENGINE - .set(Mutex::new(engine)) - .map_err(|_| anyhow!("Engine already initialized"))?; - - info!("Whisper engine initialized from embedded model (zero I/O)"); - return Ok(()); + info!("Whisper engine loaded from embedded model (zero I/O)"); + return Ok(engine); } // 2. Fallback path: resolve Whisper model at runtime. let path = get_model_path()?; let engine = LocalWhisperEngine::new_with_params(path, DecodingParams::default()) .context("Failed to initialize Whisper engine from path")?; + info!("Whisper engine loaded from path: {}", path.display()); + Ok(engine) +} - ENGINE - .set(Mutex::new(engine)) - .map_err(|_| anyhow!("Engine already initialized"))?; +/// Spawn the idle reaper once (only when idle-unload is enabled). +fn ensure_reaper() { + if idle_unload_after().is_none() { + return; + } + REAPER_STARTED.get_or_init(|| { + let spawned = std::thread::Builder::new() + .name("whisper-idle-reaper".into()) + .spawn(reaper_loop); + if let Err(e) = spawned { + warn!("Failed to spawn Whisper idle reaper: {e}"); + } + }); +} - info!("Whisper engine initialized from path: {}", path.display()); - Ok(()) +/// Background loop: drops the engine after it has been idle long enough. +fn reaper_loop() { + loop { + std::thread::sleep(REAPER_TICK); + let Some(threshold) = idle_unload_after() else { + continue; + }; + let mut guard = match slot().lock() { + Ok(g) => g, + Err(_) => continue, + }; + if guard.engine.is_some() && guard.last_used.elapsed() >= threshold { + // Drop the engine (and its Metal device) while holding the lock so + // no transcription can be mid-flight, then return freed pages. + guard.engine = None; + drop(guard); + info!( + "Whisper engine unloaded after {}s idle; releasing GPU/host memory", + threshold.as_secs() + ); + crate::memory::release_freed_heap(); + } + } } -/// Check if engine is initialized -pub fn is_initialized() -> bool { - ENGINE.get().is_some() +/// Run `f` with the engine, loading it on demand and refreshing the idle clock. +fn with_engine(f: impl FnOnce(&mut LocalWhisperEngine) -> Result) -> Result { + let mut guard = slot() + .lock() + .map_err(|e| anyhow!("Failed to lock engine: {}", e))?; + if guard.engine.is_none() { + guard.engine = Some(load_engine()?); + ensure_reaper(); + } + guard.last_used = Instant::now(); + let engine = guard + .engine + .as_mut() + .ok_or_else(|| anyhow!("Engine not initialized"))?; + f(engine) } -/// Get the global engine (initializes on first call if needed) -pub fn engine() -> Result<&'static Mutex> { - if !is_initialized() { - init()?; +/// Like [`with_engine`] but never blocks: if the engine is busy, return an error +/// instead of waiting. Used by best-effort correction passes. +fn try_with_engine(f: impl FnOnce(&mut LocalWhisperEngine) -> Result) -> Result { + let mut guard = slot() + .try_lock() + .map_err(|_| anyhow!("Whisper engine busy, skipping correction"))?; + if guard.engine.is_none() { + guard.engine = Some(load_engine()?); + ensure_reaper(); } - ENGINE - .get() - .ok_or_else(|| anyhow!("Engine not initialized")) + guard.last_used = Instant::now(); + let engine = guard + .engine + .as_mut() + .ok_or_else(|| anyhow!("Engine not initialized"))?; + f(engine) +} + +/// Initialize the global engine (call once at startup). +/// +/// Embedded Whisper is the product-default truth. Runtime path resolution is a +/// deliberate fallback for no-embed builds and local recovery. Idempotent: a +/// no-op if the engine is already loaded. +pub fn init() -> Result<()> { + with_engine(|_| Ok(())) +} + +/// Check if the engine is currently loaded. +/// +/// Note: with idle-unload enabled this can become `false` again after a period +/// of inactivity; the next transcription call reloads transparently. +pub fn is_initialized() -> bool { + SLOT.get() + .and_then(|m| m.lock().ok().map(|g| g.engine.is_some())) + .unwrap_or(false) } /// Transcribe audio samples using the global engine @@ -110,12 +221,9 @@ pub fn transcribe_with_segments( sample_rate: u32, language: Option<&str>, ) -> Result { - let engine_mutex = engine()?; - let mut engine = engine_mutex - .lock() - .map_err(|e| anyhow!("Failed to lock engine: {}", e))?; - - engine.transcribe_long_with_language_segments(samples, sample_rate, language) + with_engine(|engine| { + engine.transcribe_long_with_language_segments(samples, sample_rate, language) + }) } /// Transcribe with streaming callback @@ -125,12 +233,7 @@ pub fn transcribe_streaming<'a>( language: Option<&str>, callback: Option>, ) -> Result { - let engine_mutex = engine()?; - let mut engine = engine_mutex - .lock() - .map_err(|e| anyhow!("Failed to lock engine: {}", e))?; - - engine.transcribe_long_streaming(samples, sample_rate, language, callback) + with_engine(|engine| engine.transcribe_long_streaming(samples, sample_rate, language, callback)) } /// Transcribe a file with full structured verdict (VAD stats, confidence, provenance). @@ -139,21 +242,35 @@ pub fn transcribe_file_verdict( language: Option<&str>, options: FileTranscriptionOptions, ) -> Result { - let engine_mutex = engine()?; - let mut engine = engine_mutex - .lock() - .map_err(|e| anyhow!("Failed to lock engine: {}", e))?; - engine.transcribe_file_with_language(path, language, options) + with_engine(|engine| engine.transcribe_file_with_language(path, language, options)) } /// Detect language from audio samples pub fn detect_language(samples: &[f32], sample_rate: u32) -> Result { - let engine_mutex = engine()?; - let mut engine = engine_mutex - .lock() - .map_err(|e| anyhow!("Failed to lock engine: {}", e))?; + with_engine(|engine| engine.detect_language(samples, sample_rate)) +} + +/// Transcribe with a non-blocking engine acquisition (best-effort correction). +/// +/// Returns an error instead of waiting if the engine is busy with another +/// transcription. +pub fn try_transcribe_with_segments( + samples: &[f32], + sample_rate: u32, + language: Option<&str>, +) -> Result { + try_with_engine(|engine| { + engine.transcribe_long_with_language_segments(samples, sample_rate, language) + }) +} - engine.detect_language(samples, sample_rate) +/// Transcribe a single (already-windowed) chunk, blocking on the engine. +pub fn transcribe_chunk( + samples: &[f32], + sample_rate: u32, + language: Option<&str>, +) -> Result { + with_engine(|engine| engine.transcribe_with_language(samples, sample_rate, language)) } #[cfg(test)] @@ -161,6 +278,20 @@ mod tests { use super::*; use serial_test::serial; + #[test] + fn idle_unload_disabled_when_zero() { + // SAFETY: single-threaded test mutating a process env var it owns. + unsafe { std::env::set_var("CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS", "0") }; + assert!(idle_unload_after().is_none()); + unsafe { std::env::set_var("CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS", "120") }; + assert_eq!(idle_unload_after(), Some(Duration::from_secs(120))); + unsafe { std::env::remove_var("CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS") }; + assert_eq!( + idle_unload_after(), + Some(Duration::from_secs(DEFAULT_IDLE_UNLOAD_SECS)) + ); + } + #[test] #[serial] fn test_model_path_resolution_and_real_whisper_noop_load() { From a0e4cb7da09f31de918407a5325dcb522ea5a4e0 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 27 Jun 2026 22:59:42 -0700 Subject: [PATCH 027/385] feat(embedder): unload idle MiniLM embedder to free ~470MB The MiniLM embedder is the other always-resident ML model: a candle BertModel on the Metal GPU (~230MB) plus its multilingual tokenizer (~250K vocab, the dominant ~240MB host-side small-allocation bucket). Like Whisper it was loaded on first use and held for the whole process lifetime, and it accounts for the ~230MB Metal residual that remained even after the Whisper idle-unload. Mirror the Whisper idle-unload: the singleton becomes a resettable `Mutex<{ engine: Option, last_used }>`, a background reaper drops it after `CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS` (default 300s, 0 disables), and the next embed transparently reloads it. All access routes through a `with_embedder` closure so the reaper can never drop an engine mid-embed. The (re)load config is captured once via `CONFIG`. Public API (init/embed*/similarity/dimension/is_initialized) is preserved. Co-Authored-By: Claude Opus 4.8 (1M context) --- core/embedder/singleton.rs | 233 ++++++++++++++++++++++++------------- 1 file changed, 149 insertions(+), 84 deletions(-) diff --git a/core/embedder/singleton.rs b/core/embedder/singleton.rs index ff2e0b51..0cd3f3ca 100644 --- a/core/embedder/singleton.rs +++ b/core/embedder/singleton.rs @@ -1,105 +1,174 @@ //! Singleton pattern for embedder - easy global access. //! -//! Provides a global embedder instance that's initialized once and reused. -//! Thread-safe via OnceLock + Mutex pattern. +//! Provides a global embedder instance that is loaded on demand and reused. +//! Thread-safe via a single `Mutex` guarding a resettable slot. +//! +//! ## Idle unload +//! +//! Like Whisper, the MiniLM embedder lives on the GPU (Metal, candle BertModel) +//! and its multilingual tokenizer is a large host-side structure — together a +//! few hundred MB held for the whole process. The engine is therefore held in a +//! *resettable* slot: after a configurable idle period with no embedding a +//! background reaper drops it, returning GPU/host memory to the system, and the +//! next call transparently reloads it. Set +//! `CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS=0` to disable. use std::sync::{Mutex, OnceLock}; +use std::time::{Duration, Instant}; use anyhow::Result; -use tracing::info; +use tracing::{info, warn}; use super::engine::{EmbedderConfig, EmbedderEngine}; -/// Global embedder instance -static EMBEDDER_INSTANCE: OnceLock> = OnceLock::new(); +/// Default idle period after which the embedder is unloaded to free GPU memory. +/// Overridable via `CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS` (0 disables). +const DEFAULT_IDLE_UNLOAD_SECS: u64 = 300; -/// Initialize the embedder with default config -pub fn init() -> Result<()> { - init_with_config(EmbedderConfig::default()) +/// How often the reaper wakes to check for idleness. +const REAPER_TICK: Duration = Duration::from_secs(30); + +/// Resettable engine slot: `None` when unloaded, plus the last-use timestamp. +struct EmbedderSlot { + engine: Option, + last_used: Instant, } -/// Initialize with custom configuration -pub fn init_with_config(config: EmbedderConfig) -> Result<()> { - if EMBEDDER_INSTANCE.get().is_some() { - info!("Embedder already initialized"); - return Ok(()); - } +static SLOT: OnceLock> = OnceLock::new(); - let engine = EmbedderEngine::with_config(config)?; +/// Config used to (re)load the engine. First value wins (default unless +/// `init_with_config` set one before the first load). +static CONFIG: OnceLock = OnceLock::new(); - EMBEDDER_INSTANCE - .set(Mutex::new(engine)) - .map_err(|_| anyhow::anyhow!("Embedder already initialized"))?; +/// Guard so the idle reaper thread is spawned at most once. +static REAPER_STARTED: OnceLock<()> = OnceLock::new(); - info!("Embedder singleton initialized"); - Ok(()) +fn slot() -> &'static Mutex { + SLOT.get_or_init(|| { + Mutex::new(EmbedderSlot { + engine: None, + last_used: Instant::now(), + }) + }) } -/// Check if embedder is initialized -pub fn is_initialized() -> bool { - EMBEDDER_INSTANCE.get().is_some() +fn config() -> EmbedderConfig { + CONFIG.get_or_init(EmbedderConfig::default).clone() } -/// Embed a single text (query) -/// -/// Auto-initializes with default config if not already done. -pub fn embed(text: &str) -> Result> { - ensure_initialized()?; +/// Resolve the configured idle-unload period, or `None` when disabled (0). +fn idle_unload_after() -> Option { + let secs = std::env::var("CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS") + .ok() + .and_then(|s| s.trim().parse::().ok()) + .unwrap_or(DEFAULT_IDLE_UNLOAD_SECS); + (secs > 0).then(|| Duration::from_secs(secs)) +} - let embedder = EMBEDDER_INSTANCE - .get() - .ok_or_else(|| anyhow::anyhow!("Embedder not initialized"))?; +fn load_engine() -> Result { + let engine = EmbedderEngine::with_config(config())?; + info!("Embedder engine loaded"); + Ok(engine) +} + +/// Spawn the idle reaper once (only when idle-unload is enabled). +fn ensure_reaper() { + if idle_unload_after().is_none() { + return; + } + REAPER_STARTED.get_or_init(|| { + let spawned = std::thread::Builder::new() + .name("embedder-idle-reaper".into()) + .spawn(reaper_loop); + if let Err(e) = spawned { + warn!("Failed to spawn embedder idle reaper: {e}"); + } + }); +} - let mut guard = embedder +/// Background loop: drops the engine after it has been idle long enough. +fn reaper_loop() { + loop { + std::thread::sleep(REAPER_TICK); + let Some(threshold) = idle_unload_after() else { + continue; + }; + let mut guard = match slot().lock() { + Ok(g) => g, + Err(_) => continue, + }; + if guard.engine.is_some() && guard.last_used.elapsed() >= threshold { + guard.engine = None; + drop(guard); + info!( + "Embedder engine unloaded after {}s idle; releasing GPU/host memory", + threshold.as_secs() + ); + crate::memory::release_freed_heap(); + } + } +} + +/// Run `f` with the engine, loading it on demand and refreshing the idle clock. +fn with_embedder(f: impl FnOnce(&mut EmbedderEngine) -> Result) -> Result { + let mut guard = slot() .lock() .map_err(|e| anyhow::anyhow!("Embedder lock poisoned: {}", e))?; + if guard.engine.is_none() { + guard.engine = Some(load_engine()?); + ensure_reaper(); + } + guard.last_used = Instant::now(); + let engine = guard + .engine + .as_mut() + .ok_or_else(|| anyhow::anyhow!("Embedder not initialized"))?; + f(engine) +} - guard.embed(text) +/// Initialize the embedder with default config. +pub fn init() -> Result<()> { + with_embedder(|_| Ok(())) } -/// Embed a passage (document) for indexing -pub fn embed_passage(text: &str) -> Result> { - ensure_initialized()?; +/// Initialize with custom configuration. +/// +/// The config is captured for (re)loads; the first config wins. Idempotent. +pub fn init_with_config(config: EmbedderConfig) -> Result<()> { + let _ = CONFIG.set(config); + with_embedder(|_| Ok(())) +} - let embedder = EMBEDDER_INSTANCE - .get() - .ok_or_else(|| anyhow::anyhow!("Embedder not initialized"))?; +/// Check if the embedder is currently loaded. +/// +/// Note: with idle-unload enabled this can become `false` again after a period +/// of inactivity; the next call reloads transparently. +pub fn is_initialized() -> bool { + SLOT.get() + .and_then(|m| m.lock().ok().map(|g| g.engine.is_some())) + .unwrap_or(false) +} - let mut guard = embedder - .lock() - .map_err(|e| anyhow::anyhow!("Embedder lock poisoned: {}", e))?; +/// Embed a single text (query) +/// +/// Auto-initializes with default config if not already done. +pub fn embed(text: &str) -> Result> { + with_embedder(|engine| engine.embed(text)) +} - guard.embed_passage(text) +/// Embed a passage (document) for indexing +pub fn embed_passage(text: &str) -> Result> { + with_embedder(|engine| engine.embed_passage(text)) } /// Embed multiple texts at once pub fn embed_batch(texts: &[&str]) -> Result>> { - ensure_initialized()?; - - let embedder = EMBEDDER_INSTANCE - .get() - .ok_or_else(|| anyhow::anyhow!("Embedder not initialized"))?; - - let mut guard = embedder - .lock() - .map_err(|e| anyhow::anyhow!("Embedder lock poisoned: {}", e))?; - - guard.embed_batch(texts) + with_embedder(|engine| engine.embed_batch(texts)) } /// Embed multiple passages at once pub fn embed_passages(texts: &[&str]) -> Result>> { - ensure_initialized()?; - - let embedder = EMBEDDER_INSTANCE - .get() - .ok_or_else(|| anyhow::anyhow!("Embedder not initialized"))?; - - let mut guard = embedder - .lock() - .map_err(|e| anyhow::anyhow!("Embedder lock poisoned: {}", e))?; - - guard.embed_passages(texts) + with_embedder(|engine| engine.embed_passages(texts)) } /// Calculate cosine similarity between two embeddings @@ -109,25 +178,7 @@ pub fn similarity(a: &[f32], b: &[f32]) -> f32 { /// Get embedding dimension for the current model pub fn dimension() -> Result { - ensure_initialized()?; - - let embedder = EMBEDDER_INSTANCE - .get() - .ok_or_else(|| anyhow::anyhow!("Embedder not initialized"))?; - - let guard = embedder - .lock() - .map_err(|e| anyhow::anyhow!("Embedder lock poisoned: {}", e))?; - - Ok(guard.dimension()) -} - -/// Ensure embedder is initialized (auto-init with defaults if not) -fn ensure_initialized() -> Result<()> { - if !is_initialized() { - init()?; - } - Ok(()) + with_embedder(|engine| Ok(engine.dimension())) } #[cfg(test)] @@ -142,5 +193,19 @@ mod tests { assert!((sim - 1.0).abs() < 0.001); } + #[test] + fn idle_unload_disabled_when_zero() { + // SAFETY: single-threaded test mutating a process env var it owns. + unsafe { std::env::set_var("CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS", "0") }; + assert!(idle_unload_after().is_none()); + unsafe { std::env::set_var("CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS", "90") }; + assert_eq!(idle_unload_after(), Some(Duration::from_secs(90))); + unsafe { std::env::remove_var("CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS") }; + assert_eq!( + idle_unload_after(), + Some(Duration::from_secs(DEFAULT_IDLE_UNLOAD_SECS)) + ); + } + // Note: Full embedding tests require model download and are in integration tests } From 2bb1d13825665c4d0bb1ccafb578ed9ed4cda46f Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Sat, 27 Jun 2026 23:54:43 -0700 Subject: [PATCH 028/385] [codex/vc-ownership] fix(release): settle stop target and legacy env test --- Makefile | 2 +- core/tests/env_validation.rs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9e1dc362..3bdd160d 100644 --- a/Makefile +++ b/Makefile @@ -163,7 +163,7 @@ start: @echo "CodeScribe started (logs: /tmp/codescribe.log)" stop: - @pkill -f "codescribe$$" 2>/dev/null || true + @pkill -x codescribe 2>/dev/null || true @rm -f ~/.codescribe/codescribe.pid 2>/dev/null || true @echo "Stopped" diff --git a/core/tests/env_validation.rs b/core/tests/env_validation.rs index ad598a9c..013d6755 100644 --- a/core/tests/env_validation.rs +++ b/core/tests/env_validation.rs @@ -110,11 +110,13 @@ fn env_ignores_legacy_llm_host() { let _g2 = EnvGuard::set("OLLAMA_HOST", "http://ollama-host"); let _g3 = EnvGuard::unset("LLM_ENDPOINT"); - let mut cfg = Config::default(); // Isolate the env loader contract: the runtime default endpoint is covered // by Config::load()/Config::default() tests, while legacy host envs should // not populate an otherwise-empty llm_endpoint. - cfg.llm_endpoint = None; + let mut cfg = Config { + llm_endpoint: None, + ..Default::default() + }; cfg.load_from_env(); assert!(cfg.llm_endpoint.is_none()); From 15e6decd70ff100f798501330a333553587a10e0 Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Sun, 28 Jun 2026 00:31:39 -0700 Subject: [PATCH 029/385] [codex/vc-ownership] test(env): register idle unload controls --- docs/ENV_REGISTRY.toml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/ENV_REGISTRY.toml b/docs/ENV_REGISTRY.toml index 9376e9eb..79478f37 100644 --- a/docs/ENV_REGISTRY.toml +++ b/docs/ENV_REGISTRY.toml @@ -225,6 +225,13 @@ reload = "restart" category = "stt" description = "Initial prompt hint for Whisper decoding (ignored by ONNX adapter)" +[vars.CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS] +default = "300" +type = "u64" +reload = "hot" +category = "stt" +description = "Unload idle Whisper engine after this many seconds (0 disables)" + [vars.WHISPER_INITIAL_PROMPT] default = "" type = "string" @@ -427,6 +434,13 @@ reload = "rebuild" category = "embedder" description = "HF repo for E5 embedder (used for embedding at build/runtime)" +[vars.CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS] +default = "300" +type = "u64" +reload = "hot" +category = "embedder" +description = "Unload idle embedder engine after this many seconds (0 disables)" + # ═══════════════════════════════════════════════════════════════════════════════ # LLM / AI Formatting # ═══════════════════════════════════════════════════════════════════════════════ From 350d97c3469e4634fbb1d732ce1ab961f3a25971 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sun, 28 Jun 2026 01:16:55 -0700 Subject: [PATCH 030/385] docs(env): document idle-unload env vars + memory changelog The two idle-unload knobs were registered in ENV_REGISTRY.toml when #37 merged, but the human-facing docs lagged. Add CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS and CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS to .env.example and docs/env.md (matching the registry's hot-reload/300s contract), and record the memory footprint work (model idle-unload, Silero session sharing, allocator relief) under CHANGELOG [Unreleased]. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 2 ++ CHANGELOG.md | 6 ++++++ docs/env.md | 2 ++ 3 files changed, 10 insertions(+) diff --git a/.env.example b/.env.example index 072010b3..33c74981 100644 --- a/.env.example +++ b/.env.example @@ -23,6 +23,7 @@ # CODESCRIBE_TOGGLE_FINAL_PASS=1 # 0 = restore preview-only toggle stop path # CODESCRIBE_WHISPER_INITIAL_PROMPT=CodeScribe, LibraxisAI, Docker, GitHub # CODESCRIBE_MODEL_PATH=/path/to/whisper/model # runtime fallback / dev override +# CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS=300 # unload Whisper from GPU after N s idle (0 = never) # ============================================================================= # CLOUD STT (OPTIONAL) @@ -145,6 +146,7 @@ # CODESCRIBE_STREAM_NOVELTY=0.20 # CODESCRIBE_STREAM_DISABLE_EMBEDDINGS=1 # CODESCRIBE_STREAM_FORCE_EMBEDDINGS=1 +# CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS=300 # unload MiniLM embedder from GPU after N s idle (0 = never) # CODESCRIBE_STREAM_LEXICON=1 # CODESCRIBE_STREAM_LOG=1 # CODESCRIBE_STREAM_LOG_PATH=/tmp/codescribe_stream.log diff --git a/CHANGELOG.md b/CHANGELOG.md index f84d7bac..2419b31e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Public release hygiene** — release packaging, repository metadata, and public-facing docs are being aligned for a current `v0.12.x` public release. - **Dual DMG release variants** — release automation now builds a standard notarized DMG with embedded Silero + embedder and runtime Whisper cache/download, plus a `_full` notarized DMG with Whisper embedded. +- **Memory footprint** — idle RAM cut from ~5 GB (peak 10 GB) to ~0.8 GB. The Whisper and MiniLM embedder models now unload from the GPU after a period of inactivity and reload transparently on next use (`CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS`, `CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS`, default 300s, 0 disables). + +### Fixed + +- **Silero VAD reload leak** — the Silero ONNX session is now compiled once and shared process-wide instead of being rebuilt per recording (which leaked native ORT memory over long sessions). +- **Allocator retention** — freed transient buffers are returned to the OS after each recording (`malloc_zone_pressure_relief` on macOS) instead of inflating the resident footprint across a session. ## [0.12.2] - 2026-06-22 diff --git a/docs/env.md b/docs/env.md index aa31d3c3..76239a02 100644 --- a/docs/env.md +++ b/docs/env.md @@ -163,6 +163,7 @@ i runtime nie może znaleźć Whispera przez cache / config: - `CODESCRIBE_WHISPER_INITIAL_PROMPT` (RESTART NEEDED; alias legacy: `WHISPER_INITIAL_PROMPT`; ignorowane przez ONNX) - `STT_ENDPOINT`, `STT_API_KEY` (RESTART NEEDED) - `CODESCRIBE_MODEL_PATH`, `CODESCRIBE_MODELS_DIR` (RESTART NEEDED) +- `CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS` (HOT RELOADED; default `300`; `0` wyłącza) — po N s bezczynności silnik Whisper jest zwalniany z GPU i ładowany ponownie przy następnym użyciu ### Streaming / VAD / buffer @@ -178,6 +179,7 @@ i runtime nie może znaleźć Whispera przez cache / config: - `CODESCRIBE_STREAM_SIMILARITY` (HOT RELOADED) - `CODESCRIBE_STREAM_NOVELTY` (HOT RELOADED) - `CODESCRIBE_STREAM_DISABLE_EMBEDDINGS` (HOT RELOADED) +- `CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS` (HOT RELOADED; default `300`; `0` wyłącza) — po N s bezczynności embedder MiniLM jest zwalniany z GPU i ładowany ponownie przy następnym użyciu ### LLM (formatting/assistive) From 7c26af6f2ace09814969cf8dfe959a2c8dfd800c Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sun, 28 Jun 2026 11:44:43 -0700 Subject: [PATCH 031/385] =?UTF-8?q?docs(env):=20clarify=20idle-unload=20re?= =?UTF-8?q?claims=20GPU/host=20memory=20+=200=E2=86=92N=20caveat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address PR review (non-blocking P3s): - "from GPU" → "from GPU/host memory" across env.md, .env.example, CHANGELOG to match the runtime log ("releasing GPU/host memory") and the actual RAM win. - Note that the hot-reload tag applies to the threshold value; enabling from 0 (disabled) requires a restart, since the reaper is only spawned on (re)load. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 4 ++-- CHANGELOG.md | 2 +- docs/env.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index 33c74981..9e9a0e1d 100644 --- a/.env.example +++ b/.env.example @@ -23,7 +23,7 @@ # CODESCRIBE_TOGGLE_FINAL_PASS=1 # 0 = restore preview-only toggle stop path # CODESCRIBE_WHISPER_INITIAL_PROMPT=CodeScribe, LibraxisAI, Docker, GitHub # CODESCRIBE_MODEL_PATH=/path/to/whisper/model # runtime fallback / dev override -# CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS=300 # unload Whisper from GPU after N s idle (0 = never) +# CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS=300 # unload Whisper from GPU/host memory after N s idle (0 = never) # ============================================================================= # CLOUD STT (OPTIONAL) @@ -146,7 +146,7 @@ # CODESCRIBE_STREAM_NOVELTY=0.20 # CODESCRIBE_STREAM_DISABLE_EMBEDDINGS=1 # CODESCRIBE_STREAM_FORCE_EMBEDDINGS=1 -# CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS=300 # unload MiniLM embedder from GPU after N s idle (0 = never) +# CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS=300 # unload MiniLM embedder from GPU/host memory after N s idle (0 = never) # CODESCRIBE_STREAM_LEXICON=1 # CODESCRIBE_STREAM_LOG=1 # CODESCRIBE_STREAM_LOG_PATH=/tmp/codescribe_stream.log diff --git a/CHANGELOG.md b/CHANGELOG.md index 2419b31e..c0a27ff6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Public release hygiene** — release packaging, repository metadata, and public-facing docs are being aligned for a current `v0.12.x` public release. - **Dual DMG release variants** — release automation now builds a standard notarized DMG with embedded Silero + embedder and runtime Whisper cache/download, plus a `_full` notarized DMG with Whisper embedded. -- **Memory footprint** — idle RAM cut from ~5 GB (peak 10 GB) to ~0.8 GB. The Whisper and MiniLM embedder models now unload from the GPU after a period of inactivity and reload transparently on next use (`CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS`, `CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS`, default 300s, 0 disables). +- **Memory footprint** — idle RAM cut from ~5 GB (peak ~10 GB) to ~0.8 GB. The Whisper and MiniLM embedder models now unload from GPU/host memory after a period of inactivity and reload transparently on next use (`CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS`, `CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS`, default 300s, 0 disables). ### Fixed diff --git a/docs/env.md b/docs/env.md index 76239a02..ceb91f87 100644 --- a/docs/env.md +++ b/docs/env.md @@ -163,7 +163,7 @@ i runtime nie może znaleźć Whispera przez cache / config: - `CODESCRIBE_WHISPER_INITIAL_PROMPT` (RESTART NEEDED; alias legacy: `WHISPER_INITIAL_PROMPT`; ignorowane przez ONNX) - `STT_ENDPOINT`, `STT_API_KEY` (RESTART NEEDED) - `CODESCRIBE_MODEL_PATH`, `CODESCRIBE_MODELS_DIR` (RESTART NEEDED) -- `CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS` (HOT RELOADED; default `300`; `0` wyłącza) — po N s bezczynności silnik Whisper jest zwalniany z GPU i ładowany ponownie przy następnym użyciu +- `CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS` (HOT RELOADED dla wartości progu; default `300`; `0` wyłącza — włączenie z `0` wymaga restartu) — po N s bezczynności silnik Whisper jest zwalniany z pamięci (GPU/host) i ładowany ponownie przy następnym użyciu ### Streaming / VAD / buffer @@ -179,7 +179,7 @@ i runtime nie może znaleźć Whispera przez cache / config: - `CODESCRIBE_STREAM_SIMILARITY` (HOT RELOADED) - `CODESCRIBE_STREAM_NOVELTY` (HOT RELOADED) - `CODESCRIBE_STREAM_DISABLE_EMBEDDINGS` (HOT RELOADED) -- `CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS` (HOT RELOADED; default `300`; `0` wyłącza) — po N s bezczynności embedder MiniLM jest zwalniany z GPU i ładowany ponownie przy następnym użyciu +- `CODESCRIBE_EMBEDDER_IDLE_UNLOAD_SECS` (HOT RELOADED dla wartości progu; default `300`; `0` wyłącza — włączenie z `0` wymaga restartu) — po N s bezczynności embedder MiniLM jest zwalniany z pamięci (GPU/host) i ładowany ponownie przy następnym użyciu ### LLM (formatting/assistive) From eb46b962224a540558c5c576ef4b8dcd1adae370 Mon Sep 17 00:00:00 2001 From: Szowesgad Date: Sun, 28 Jun 2026 12:18:06 -0700 Subject: [PATCH 032/385] [codex/vc-ownership] release: embed models by default Drive embed-by-default from the delivery layer (Makefile recipes + build-dmg.sh defaults) rather than flipping the build.rs default, so `make build` / raw `cargo build` stay lean for dev iteration while release/install/DMG user-delivery lanes always embed Silero + MiniLM + Whisper. release-codescribe-embedded and release-full collapse to compatibility aliases; --embed-whisper becomes a documented no-op and --no-embed remains the dev/recovery path (CODESCRIBE_NO_EMBED still wins in build.rs, so the recovery lane needs no loader rewrite). Authored-By: claude session_id: eb1ce15e-5876-4262-a6dd-edc747d81617 date: 2026-06-28T12:17:47 PDT runtime: claude --- Makefile | 37 +++++++++++++++++++------------------ README.md | 2 +- scripts/build-dmg.sh | 34 +++++++++++++++++++--------------- 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index 3bdd160d..97d587f6 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # CodeScribe - Pure Rust Build System # Speech-to-text tray app for macOS -.PHONY: all build release release-codescribe release-qube install install-no-embed config bundle install-app \ +.PHONY: all build release release-codescribe release-codescribe-embedded release-qube install install-no-embed config bundle install-app \ start stop restart status logs logs-follow \ bump bump-patch bump-minor bump-major version \ lint format test test-quick test-e2e test-e2e-real test-sse test-formatting test-all \ @@ -60,14 +60,14 @@ build: @echo "Building (debug)..." @cargo build -release-codescribe: - @echo "Building codescribe (release, embedded support assets; Whisper from cache/download)..." - @cargo build --release --bin codescribe - -release-codescribe-embedded: - @echo "Building codescribe (release, EMBEDDED Whisper — distribution)..." +release-codescribe: ensure-models + @echo "Building codescribe (release, embedded models: Silero + MiniLM + Whisper)..." @CODESCRIBE_EMBED_WHISPER=1 cargo build --release --bin codescribe +# Compatibility alias — embedding Whisper is now the default for release-codescribe. +# Kept so existing scripts / muscle memory keep working; NOT a separate public lane. +release-codescribe-embedded: release-codescribe + release-qube: @echo "Building qube-* (release, runtime model resolve from HF cache)..." @CODESCRIBE_NO_EMBED=1 cargo build --release --target-dir target-noembed --bin qube-daemon --bin qube-report @@ -75,16 +75,16 @@ release-qube: release: release-codescribe release-qube install: - @echo "Installing CodeScribe (embedded support assets; Whisper from cache/download)..." + @echo "Installing CodeScribe (embedded models: Silero + MiniLM + Whisper)..." @./scripts/ensure-models.sh - @cargo install --path . --force + @CODESCRIBE_EMBED_WHISPER=1 cargo install --path . --force @mkdir -p ~/.codescribe @pwd > ~/.codescribe/repo_path @$(MAKE) hooks @echo "Installed: codescribe $$(grep '^version' $(VERSION_FILE) | head -1 | sed 's/.*\"\(.*\)\"/v\1/')" install-no-embed: - @echo "Installing CodeScribe (runtime Whisper fallback + no optional embedded support assets)..." + @echo "Installing CodeScribe (DEV/RECOVERY: runtime Whisper fallback + no optional embedded support assets)..." @./scripts/ensure-models.sh @CODESCRIBE_NO_EMBED=1 cargo install --path . --force @mkdir -p ~/.codescribe @@ -376,8 +376,8 @@ help: @printf '\n' @printf ' $(HELP_C_YELLOW)%s$(HELP_C_RESET)\n' 'BUILD & INSTALL' @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'build' 'Build debug binary' - @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'release' 'Build release binary with embedded support assets; Whisper from cache/download' - @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'install' 'Install CLI with embedded support assets' + @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'release' 'Build release binary with embedded models (Silero + MiniLM + Whisper)' + @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'install' 'Install CLI with embedded models (Silero + MiniLM + Whisper)' @printf '%s\n' ' make install-no-embed Install without optional embedded assets (needs CODESCRIBE_MODEL_PATH)' @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'config' 'Edit ~/.codescribe/.env' @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'bundle' 'Create CodeScribe.app bundle' @@ -386,9 +386,9 @@ help: @printf ' $(HELP_C_YELLOW)%s$(HELP_C_RESET)\n' 'RELEASE & DISTRIBUTION' @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'dmg' 'Build DMG (ad-hoc signed)' @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'dmg-signed' 'Build DMG (Developer ID signed)' - @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'release-standard' 'Build + sign + notarize standard DMG' - @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'release-full' 'Build + sign + notarize full DMG with embedded Whisper' - @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'release-dmgs' 'Build both notarized release DMGs' + @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'release-standard' 'Build + sign + notarize release DMG (embedded models)' + @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'release-full' 'Compatibility alias for release-standard (embedded by default)' + @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'release-dmgs' 'Build the notarized release DMG' @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'notarize' 'Notarize DMG with Apple' @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'download-model' 'Download Whisper model from HF' @printf ' $(HELP_C_GREEN)%-18s$(HELP_C_RESET) %s\n' 'download-e5' 'Download E5 embedder model from HF' @@ -437,10 +437,11 @@ dmg-signed: release-standard: @./scripts/build-dmg.sh --sign --notarize -release-full: - @./scripts/build-dmg.sh --sign --notarize --embed-whisper --dmg-suffix _full +# Compatibility alias — the standard DMG now embeds Whisper by default, so it IS +# the real user artifact. `_full` no longer denotes a separate "real" build. +release-full: release-standard -release-dmgs: release-standard release-full +release-dmgs: release-standard notarize: @if ls CodeScribe_*.dmg 1> /dev/null 2>&1; then \ diff --git a/README.md b/README.md index 358b5dea..0c5eb80f 100644 --- a/README.md +++ b/README.md @@ -351,7 +351,7 @@ CodeScribe uses **whisper-large-v3-turbo-mlx-q8**: ### Runtime Whisper (Current) -The standard build embeds Silero VAD and MiniLM semantic support assets, then resolves Whisper at runtime from the locations below. The full release DMG embeds the same support assets plus Whisper by building with `CODESCRIBE_EMBED_WHISPER=1`. +User-delivery builds (`make release`, `make install`, the release DMG) embed everything by default: Silero VAD, the MiniLM semantic embedder, and Whisper (`CODESCRIBE_EMBED_WHISPER=1`). The fast developer build (`make build` / raw `cargo build`) stays lean and resolves Whisper at runtime from the locations below; that same resolution order is the fallback whenever a build is not embedded. 1. `CODESCRIBE_MODEL_PATH` environment variable 2. `~/.codescribe/models/whisper-large-v3-turbo-mlx-q8/` diff --git a/scripts/build-dmg.sh b/scripts/build-dmg.sh index 9522fb97..65a60b11 100755 --- a/scripts/build-dmg.sh +++ b/scripts/build-dmg.sh @@ -19,21 +19,27 @@ APP_PATH="$ROOT_DIR/bundle/${APP_NAME}.app" SIGN=0 NOTARIZE=0 NO_EMBED=0 -EMBED_WHISPER=0 +# Embedding everything (Silero + MiniLM + Whisper) is the default for user +# delivery. --no-embed flips this off for dev/recovery builds only. +EMBED_WHISPER=1 +EMBED_WHISPER_EXPLICIT=0 DMG_SUFFIX="" usage() { cat < Override codesign identity --entitlements

Entitlements plist path (default: $ENTITLEMENTS) - --embed-whisper Embed the Whisper model in the app bundle + --embed-whisper Compatibility no-op: Whisper is embedded by default --dmg-suffix Append suffix before .dmg (for example: _full) - --no-embed Disable all model embedding (CODESCRIBE_NO_EMBED=1) + --no-embed DEV/RECOVERY only: disable all model embedding + (CODESCRIBE_NO_EMBED=1) — not the public product path EOF } @@ -43,7 +49,7 @@ while [[ $# -gt 0 ]]; do --notarize) NOTARIZE=1; shift 1;; --identity) IDENTITY="$2"; shift 2;; --entitlements) ENTITLEMENTS="$2"; shift 2;; - --embed-whisper) EMBED_WHISPER=1; shift 1;; + --embed-whisper) EMBED_WHISPER=1; EMBED_WHISPER_EXPLICIT=1; shift 1;; --dmg-suffix) DMG_SUFFIX="$2"; shift 2;; --no-embed) NO_EMBED=1; shift 1;; -h|--help) usage; exit 0;; @@ -51,10 +57,14 @@ while [[ $# -gt 0 ]]; do esac done -if [[ "$NO_EMBED" -eq 1 && "$EMBED_WHISPER" -eq 1 ]]; then +if [[ "$NO_EMBED" -eq 1 && "$EMBED_WHISPER_EXPLICIT" -eq 1 ]]; then echo "ERROR: --no-embed and --embed-whisper cannot be used together" >&2 exit 1 fi +# --no-embed (dev/recovery) wins over the embed-by-default policy. +if [[ "$NO_EMBED" -eq 1 ]]; then + EMBED_WHISPER=0 +fi DMG_NAME="CodeScribe_${VERSION}${DMG_SUFFIX}.dmg" DMG_PATH="$ROOT_DIR/$DMG_NAME" @@ -67,12 +77,8 @@ BUILD_ENV+=(-u CODESCRIBE_EMBED_TTS) if [[ "$NO_EMBED" -eq 1 ]]; then BUILD_ENV+=(-u CODESCRIBE_EMBED_WHISPER CODESCRIBE_NO_EMBED=1) else - BUILD_ENV+=(-u CODESCRIBE_NO_EMBED) - if [[ "$EMBED_WHISPER" -eq 1 ]]; then - BUILD_ENV+=(CODESCRIBE_EMBED_WHISPER=1) - else - BUILD_ENV+=(-u CODESCRIBE_EMBED_WHISPER) - fi + # Default user-delivery path: embed Whisper alongside Silero + MiniLM. + BUILD_ENV+=(-u CODESCRIBE_NO_EMBED CODESCRIBE_EMBED_WHISPER=1) fi echo "=== Build DMG ===" @@ -80,11 +86,9 @@ echo "App: $APP_NAME" echo "Bundle ID: $BUNDLE_ID" echo "Version: $VERSION" if [[ "$NO_EMBED" -eq 1 ]]; then - echo "Models: runtime assets only (CODESCRIBE_NO_EMBED=1)" -elif [[ "$EMBED_WHISPER" -eq 1 ]]; then - echo "Models: embedded Silero + embedder + Whisper" + echo "Models: runtime assets only (CODESCRIBE_NO_EMBED=1) — dev/recovery build, not the public artifact" else - echo "Models: embedded Silero + embedder; Whisper resolves from cache/download" + echo "Models: embedded Silero + MiniLM + Whisper" fi echo "DMG: $DMG_PATH" From ecbc5a3421994791c8c17e1e5cee96b9011805ae Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sun, 28 Jun 2026 12:56:46 -0700 Subject: [PATCH 033/385] chore(docs): remove stale and internal docs for public repo Drop docs that are stale, internal, or leak private setup: - docs/BACKLOG.md (stale working backlog) - docs/audits/ (internal audit, contained an absolute home path) - docs/ADR/2026-01-*/2026-02-* (22 dated snapshots duplicating live docs) - docs/future/ + docs/quality-loop-vision.md (vision docs naming an internal dev workstation) - docs/reports/ (internal AI-agent forensics artifact) Keep the two ADRs that are linked as authoritative (layered pipeline + its correction). Fix dangling references in README and ARCHITECTURE. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 6 +- docs/ADR/2026-01-16-LICENSE_NOTES.md | 23 - docs/ADR/2026-01-18-WHISPER_LIVE.md | 152 ----- docs/ADR/2026-01-19-INSTALLATION.md | 202 ------- docs/ADR/2026-01-19-TEAM_SETUP.md | 186 ------ docs/ADR/2026-01-25-ARCHITECTURE.md | 259 -------- docs/ADR/2026-01-25-ARCHITECTURE_VISION.md | 186 ------ docs/ADR/2026-01-25-BACKLOG.md | 119 ---- docs/ADR/2026-01-25-FEASIBILITY_ANALYSIS.md | 117 ---- docs/ADR/2026-01-25-README.md | 100 --- docs/ADR/2026-01-25-chat-overlay.md | 207 ------- docs/ADR/2026-01-25-installation.md | 140 ----- docs/ADR/2026-01-25-modes.md | 199 ------ docs/ADR/2026-01-25-quality-loop-vision.md | 569 ------------------ docs/ADR/2026-01-26-OVERLAY_STREAMING.md | 177 ------ docs/ADR/2026-01-26-QUICKSTART.md | 145 ----- docs/ADR/2026-01-26-SPEECH_TO_SPEECH_AGENT.md | 556 ----------------- docs/ADR/2026-01-26-VOICE_STACK_WBS.md | 311 ---------- docs/ADR/2026-01-26-env.md | 32 - docs/ADR/2026-01-26-privacy.md | 291 --------- docs/ADR/2026-01-26-settings.md | 340 ----------- docs/ADR/2026-01-26-troubleshooting.md | 321 ---------- .../2026-02-15-CODESCRIBE_RUNTIME_IN_VISTA.md | 168 ------ docs/ARCHITECTURE.md | 7 +- docs/BACKLOG.md | 106 ---- .../DIAGNOSIS_top_shots.md | 115 ---- .../2026-05-28-faza1-handsoff/audit_report.md | 179 ------ .../audit_requirements_matrix.jsonl | 9 - docs/future/ARCHITECTURE_VISION.md | 186 ------ docs/future/FEASIBILITY_ANALYSIS.md | 117 ---- docs/future/SPEECH_TO_SPEECH_AGENT.md | 556 ----------------- docs/quality-loop-vision.md | 569 ------------------ docs/reports/20260502_chat_shell_forensics.md | 227 ------- 33 files changed, 4 insertions(+), 6873 deletions(-) delete mode 100644 docs/ADR/2026-01-16-LICENSE_NOTES.md delete mode 100644 docs/ADR/2026-01-18-WHISPER_LIVE.md delete mode 100644 docs/ADR/2026-01-19-INSTALLATION.md delete mode 100644 docs/ADR/2026-01-19-TEAM_SETUP.md delete mode 100644 docs/ADR/2026-01-25-ARCHITECTURE.md delete mode 100644 docs/ADR/2026-01-25-ARCHITECTURE_VISION.md delete mode 100644 docs/ADR/2026-01-25-BACKLOG.md delete mode 100644 docs/ADR/2026-01-25-FEASIBILITY_ANALYSIS.md delete mode 100644 docs/ADR/2026-01-25-README.md delete mode 100644 docs/ADR/2026-01-25-chat-overlay.md delete mode 100644 docs/ADR/2026-01-25-installation.md delete mode 100644 docs/ADR/2026-01-25-modes.md delete mode 100644 docs/ADR/2026-01-25-quality-loop-vision.md delete mode 100644 docs/ADR/2026-01-26-OVERLAY_STREAMING.md delete mode 100644 docs/ADR/2026-01-26-QUICKSTART.md delete mode 100644 docs/ADR/2026-01-26-SPEECH_TO_SPEECH_AGENT.md delete mode 100644 docs/ADR/2026-01-26-VOICE_STACK_WBS.md delete mode 100644 docs/ADR/2026-01-26-env.md delete mode 100644 docs/ADR/2026-01-26-privacy.md delete mode 100644 docs/ADR/2026-01-26-settings.md delete mode 100644 docs/ADR/2026-01-26-troubleshooting.md delete mode 100644 docs/ADR/2026-02-15-CODESCRIBE_RUNTIME_IN_VISTA.md delete mode 100644 docs/BACKLOG.md delete mode 100644 docs/audits/2026-05-28-faza1-handsoff/DIAGNOSIS_top_shots.md delete mode 100644 docs/audits/2026-05-28-faza1-handsoff/audit_report.md delete mode 100644 docs/audits/2026-05-28-faza1-handsoff/audit_requirements_matrix.jsonl delete mode 100644 docs/future/ARCHITECTURE_VISION.md delete mode 100644 docs/future/FEASIBILITY_ANALYSIS.md delete mode 100644 docs/future/SPEECH_TO_SPEECH_AGENT.md delete mode 100644 docs/quality-loop-vision.md delete mode 100644 docs/reports/20260502_chat_shell_forensics.md diff --git a/README.md b/README.md index 358b5dea..f20432fc 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ flowchart TB > **Status:** current source version is `0.12.2` (see `Cargo.toml`) and ships as a native macOS tray/settings/overlay app with local live preview, tiered settings (`settings.json` + Keychain + optional `.env`), and quality-loop tooling. -See: [`docs/WHISPER_LIVE.md`](docs/WHISPER_LIVE.md) | [`docs/BACKLOG.md`](docs/BACKLOG.md) | [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) +See: [`docs/WHISPER_LIVE.md`](docs/WHISPER_LIVE.md) | [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) ## OpenAI Provider @@ -254,7 +254,7 @@ pipeline, while overlays/chat bubbles still receive only backspace-encoded | **Formatting** | Double‑tap `Left Option` | AI formatting pass, auto‑paste | | **Assistive (Agent)** | Double‑tap `Right Option` | Agent chat with optional selection context | -See [`docs/BACKLOG.md`](docs/BACKLOG.md) for detailed mode descriptions and future enhancements (VAD, Overlay). +See [`docs/guide/modes.md`](docs/guide/modes.md) for detailed mode descriptions. ## Configuration @@ -448,7 +448,7 @@ Grant permissions in System Settings > Privacy & Security when prompted. - Preserve the explicit split between onboarding, settings, dictation overlay, and assistive overlay. - Ship the macOS distribution path cleanly: bundle, sign, and notarize the DMG story. -See [`docs/BACKLOG.md`](docs/BACKLOG.md) for the working backlog, [`docs/PUBLIC_RELEASE_CHECKLIST.md`](docs/PUBLIC_RELEASE_CHECKLIST.md) for the public launch gate, and [`docs/ARCHITECTURE_VISION.md`](docs/ARCHITECTURE_VISION.md) for longer-range ideas that are not part of the current shipped surface. +See [`docs/PUBLIC_RELEASE_CHECKLIST.md`](docs/PUBLIC_RELEASE_CHECKLIST.md) for the public launch gate. ## License diff --git a/docs/ADR/2026-01-16-LICENSE_NOTES.md b/docs/ADR/2026-01-16-LICENSE_NOTES.md deleted file mode 100644 index a79b7c6c..00000000 --- a/docs/ADR/2026-01-16-LICENSE_NOTES.md +++ /dev/null @@ -1,23 +0,0 @@ -# CodeScribe License Notes - -CodeScribe is distributed under the **BSD 4-Clause License (Original BSD)**. The clause that -usually raises questions is the _advertising clause_: - -> "All advertising materials mentioning features or use of this software must display the -> acknowledgement: This product includes software developed by Loctree." - -What this means in practice: - -| Scenario | Requirement | -| -------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -| README / documentation inside this repo | already satisfied (see README §License). | -| Public marketing site, blog post, product page mentioning CodeScribe | include the acknowledgement sentence verbatim. | -| Binary redistributions (DMG, PKG) | include the acknowledgement in the splash / "About" dialog or accompanying README. | -| Internal deployments (no external advertising) | no additional action beyond keeping the LICENSE file intact. | - -If you fork CodeScribe and redistribute it, you must keep the LICENSE file and the acknowledgement -sentence wherever you describe the fork. If you embed CodeScribe inside another product, place the -acknowledgement next to other third‑party credits. - -For questions about alternative licensing please reach out Loctree team -at [contact@loctree.io](mailto:contact@loctree.io). diff --git a/docs/ADR/2026-01-18-WHISPER_LIVE.md b/docs/ADR/2026-01-18-WHISPER_LIVE.md deleted file mode 100644 index ed35563e..00000000 --- a/docs/ADR/2026-01-18-WHISPER_LIVE.md +++ /dev/null @@ -1,152 +0,0 @@ -# WHISPER LIVE (Embedded + Streaming Transcription) - -> **Status:** DONE ✅ (2026-01-16) -> -> **Tagline:** Whisper is welded into the binary, and transcription happens _during recording_. - -## TL;DR - -CodeScribe’s core power-up is: - -1. **Embedded Whisper model** (`whisper-large-v3-turbo-mlx-q8`) in the release binary - - **Zero disk I/O** for local STT - - Model loads once into GPU/Metal and stays in-process -2. **Live (streaming) transcription** while the user is recording - - Audio is chunked and transcribed in the background - - On `stop()` we only “close” the last fragment → **near-instant time-to-paste** - -## What we shipped - -### 1) Embedded Whisper (Strict Policy) - -- **ALWAYS Embedded:** The model (`whisper-large-v3-turbo-mlx-q8`) is welded into the release binary. - - **Zero Exceptions:** We never bundle the `models/` folder in the `.app`. - - **Zero Disk I/O:** Model loads directly from memory to Metal GPU. - - **Native Power:** Minimal abstraction layers (approx. 4) compared to typical 32+ in heavy JS/Python bridges. -- **Global Singleton:** A process-wide engine instance loads once and stays resident. - -Key behavior: - -- **Release:** Strict embedded mode. If the model isn't found at build time, the build fails. -- **Development:** Local debug builds can still resolve external paths for rapid iteration, but the release pipeline enforces embedding. - -### 2) Streaming transcription (during recording) - -We removed the old bottleneck: - -```text -Audio callback → buffer → stop() → WAV write/read → transcribe entire audio → LLM -``` - -And replaced it with: - -```text -Audio callback → non-blocking channel → chunking worker → spawn_blocking(Whisper) → transcript buffer - ↓ - overlap dedup - -stop() → transcribe last pending samples → return final transcript → LLM/paste -``` - -Practical win: - -- **~35s recording:** `stop()` is ~0.5s (last chunk only) instead of ~4s (whole audio) - -## What’s new around Whisper Live - -- **Stream postprocess** (`src/stream_postprocess.rs`) — semantic gating and cleanup of chunk output - before final paste/LLM, reducing low‑quality fragments in live mode. -- **IPC server** (`src/ipc/`) — stable runtime interface for GUI/clients; Whisper Live can be - consumed and extended outside the tray flow. -- **Quality loop/report** (`src/quality_loop.rs`, `src/quality_report.rs`) — automated scoring and - batch diagnostics for streaming accuracy and regressions. - -## How it works (high level) - -```mermaid -flowchart TD - A[CPAL input callback (audio thread)] -->|try_send f32 samples| B[mpsc channel] - B --> C[StreamingRecorder worker (tokio task)] - C -->|accumulate| D[chunk buffer] - D -->|every ~15s with ~2s overlap| E[spawn_blocking] - E --> F[Whisper singleton engine (Metal)] - F --> G[chunk text] - G --> H[append_with_overlap_dedup] - H --> I[transcript_buffer] - I --> J[controller stop(): finalize + paste / LLM] -``` - -## Where in the code - -### Embedded + singleton engine - -- `src/whisper/embedded.rs` — embedded model bytes and accessors -- `src/whisper/singleton.rs` — global engine singleton (loads embedded model and exposes `transcribe*()`) -- `src/whisper/engine.rs` — Candle/Whisper inference, chunking, overlap dedup (`append_with_overlap_dedup`) - -### Live streaming recorder - -- `src/audio/recorder.rs` - - CPAL input stream at **native device rate** (often `48000Hz`) - - callback hook for raw `f32` samples - - exposes `Recorder::actual_sample_rate()` -- `src/audio/streaming_recorder.rs` - - connects recorder callback → `mpsc::channel` (non-blocking) - - chunking (default: `15s` chunks + `2s` overlap) - - background transcription via `tokio::spawn_blocking` - - dedup between chunks via `append_with_overlap_dedup` -- `src/controller.rs` - - uses `StreamingRecorder` and prefers the streaming transcript on `stop()` - - can still save the WAV for logs and/or cloud fallback - -## Build & distribution - -### Install from source (embedded model) - -```bash -make download-model # ensures models/whisper-large-v3-turbo-mlx-q8 exists for embedding -make install # builds + installs an ~888MB binary with embedded model -``` - -### Bundle / DMG (Whisper + Silero only) - -```bash -make bundle -make dmg-signed -``` - -Notes: - -- DMG ships the `.app` with **the embedded model only** (no `Resources/models/*` duplication) -- A “too small” release binary is treated as a build error (guardrail in `scripts/build-release.sh`) - -## Troubleshooting / FAQ - -### “My binary is small — is the model embedded?” - -If the release binary is far below ~500MB, embedding likely didn’t happen. - -Checklist: - -- ensure `models/whisper-large-v3-turbo-mlx-q8/` exists (download step) -- ensure `CODESCRIBE_NO_EMBED` is not set -- rebuild with `cargo build --release` - -### “Why does streaming care about actual sample rate?” - -Microphones usually run at `48kHz`. We record at the device’s native rate for compatibility, -and Whisper internally resamples to `16kHz`. - -**Important:** streaming must pass the **real** `sample_rate` to the engine — otherwise you -get hallucinations and low confidence (classic “gibberish” pattern). - -## Benchmarks (rule of thumb) - -- Model load: ~7s (first time after app start, embedded → GPU) -- Live transcription: overlaps with recording -- After `stop()`: usually just final chunk, typically well below 1s - ---- - -**Made with (งಠ_ಠ)ง by the ⌜ CodeScribe ⌟ 𝖙𝖊𝖆𝖒 (c) 2024-2026 -Maciej & Monika + Klaudiusz (AI) + Junie (AI)** diff --git a/docs/ADR/2026-01-19-INSTALLATION.md b/docs/ADR/2026-01-19-INSTALLATION.md deleted file mode 100644 index bfd900e7..00000000 --- a/docs/ADR/2026-01-19-INSTALLATION.md +++ /dev/null @@ -1,202 +0,0 @@ -# CodeScribe Installation and Launch Guide - -This document describes the installation methods, configuration paths, and how the application locates its resources. - -## Installation Methods - -### Method 1: CLI Install (Recommended for Development) - -```bash -make download-model # Download Whisper model (required for embedding) -make install # Build and install to ~/.cargo/bin/ -``` - -**Result**: Binary `codescribe` installed to `~/.cargo/bin/` (~888MB with embedded model). - -**How it runs**: Direct execution from terminal or as background daemon. - -### Method 2: App Bundle (For Distribution) - -```bash -make bundle # Creates bundle/CodeScribe.app -make install-app # Copies to /Applications/CodeScribe.app -``` - -**Result**: Standard macOS .app bundle in `/Applications/`. - -**How it runs**: Double-click or launch from Spotlight. - -### Method 3: DMG Distribution (For End Users) - -```bash -make dmg-signed # Build signed DMG -make notarize # Notarize with Apple (requires Developer ID) -# or one-shot: -# make release-full # Build + sign + notarize -``` - -**Result**: `CodeScribe_X.Y.Z.dmg` ready for distribution. - -## Configuration - -### Config Directory - -All configuration is stored in: - -``` -~/.codescribe/ -├── .env # Main configuration file -├── prompts/ # Custom AI prompts -│ ├── formatting.txt -│ └── assistive.txt -├── history/ # Transcription history -├── reports/ # Quality reports -└── repo_path # Path to source repo (set during install) -``` - -### Environment Variables (.env) - -The application loads configuration from `~/.codescribe/.env` using these priorities: - -1. **Environment variables** (highest priority) -2. **~/.codescribe/.env** (main config file) -3. **Default values** (fallback) - -```mermaid -flowchart TD - A[Application Start] --> B{Check ENV vars} - B -->|Set| C[Use ENV value] - B -->|Not set| D{Check ~/.codescribe/.env} - D -->|Exists| E[Load with dotenvy] - D -->|Missing| F[Create from template] - F --> G{Find template} - G -->|Bundle| H[../Resources/.env.example] - G -->|Repo| I[.env.example in repo root] - G -->|None| J[Generate minimal .env] - E --> K[Apply defaults for missing keys] - H --> K - I --> K - J --> K - C --> L[Config Ready] - K --> L -``` - -### Key Configuration Variables - -```env -# Speech-to-Text -WHISPER_LANGUAGE=pl # pl | en | de | fr -USE_LOCAL_STT=1 # 1 = embedded Whisper - -# Hotkeys -HOLD_MODS=ctrl # ctrl | ctrl_alt | ctrl_shift -TOGGLE_TRIGGER=double_option # double_option | none - -# AI Formatting -AI_FORMATTING_ENABLED=1 -LLM_ENDPOINT=https://api.openai.com/v1/responses -LLM_MODEL=gpt-4.1-mini -LLM_API_KEY=sk-xxx - -# Optional: Separate providers for modes -LLM_FORMATTING_{ENDPOINT,MODEL,API_KEY}=... -LLM_ASSISTIVE_{ENDPOINT,MODEL,API_KEY}=... -``` - -## Bundle Structure - -``` -CodeScribe.app/ -└── Contents/ - ├── Info.plist # Bundle metadata (icon, identifier, version) - ├── MacOS/ - │ └── codescribe # Main executable (~888MB with embedded model) - └── Resources/ - └── AppIcon.icns # Application icon -``` - -### Info.plist Keys - -| Key | Value | Purpose | -| ---------------------------- | --------------------- | ---------------------------- | -| CFBundleIdentifier | io.loctree.codescribe | Unique app identifier | -| CFBundleIconFile | AppIcon | Points to AppIcon.icns | -| CFBundleExecutable | codescribe | Main binary name | -| LSMinimumSystemVersion | 14.0 | Requires macOS Sonoma+ | -| NSMicrophoneUsageDescription | ... | Microphone permission prompt | - -## Icons - -### Tray Icon - -- **Source**: `assets/icon.png` (embedded via `include_bytes!`) -- **Location in code**: `src/tray/icons.rs` -- **Size**: 44x44 pixels (Retina), 22x22 logical - -### Dock Icon - -- **For CLI**: Programmatically set via `set_dock_icon()` in `src/ui.rs` -- **For Bundle**: Uses `CFBundleIconFile` from Info.plist pointing to `AppIcon.icns` -- **Source**: `assets/AppIcon.icns` - -### Icon Loading Flow - -```mermaid -flowchart LR - subgraph CLI["CLI Mode (codescribe)"] - A1[Start] --> A2[set_dock_icon] - A2 --> A3[NSImage from include_bytes] - A3 --> A4[setApplicationIconImage] - end - - subgraph Bundle["Bundle Mode (.app)"] - B1[Start] --> B2[macOS reads Info.plist] - B2 --> B3[CFBundleIconFile = AppIcon] - B3 --> B4[Load AppIcon.icns from Resources] - end - - subgraph Tray["Tray Icon (both modes)"] - C1[Tray init] --> C2[load_custom_icon] - C2 --> C3[include_bytes icon.png] - C3 --> C4[tray_icon::Icon] - end -``` - -## Permissions Required - -Grant in **System Settings > Privacy & Security**: - -| Permission | Purpose | When Prompted | -| ---------------- | ---------------------- | ----------------------- | -| Microphone | Audio recording | First recording attempt | -| Accessibility | Global hotkeys, paste | First hotkey press | -| Input Monitoring | Keyboard event capture | First hotkey press | - -## Troubleshooting - -### Empty Dock Icon - -- **CLI mode**: `set_dock_icon()` should set it programmatically -- **Bundle mode**: Check that `Info.plist` exists and has `CFBundleIconFile` -- **Verify**: `plutil -lint /Applications/CodeScribe.app/Contents/Info.plist` - -### Empty Tray Icon - -- Check that `assets/icon.png` exists and is valid PNG -- Rebuild with `cargo build --release` - -### Config Not Loading - -- Check `~/.codescribe/.env` exists -- Verify syntax: `cat ~/.codescribe/.env` -- Check logs: `codescribe -v` for verbose output - -### Hotkeys Not Working - -- Grant Accessibility permission -- Grant Input Monitoring permission -- Restart the application after granting - ---- - -_Copyright © 2024–2026 VetCoders_ diff --git a/docs/ADR/2026-01-19-TEAM_SETUP.md b/docs/ADR/2026-01-19-TEAM_SETUP.md deleted file mode 100644 index e6a398e9..00000000 --- a/docs/ADR/2026-01-19-TEAM_SETUP.md +++ /dev/null @@ -1,186 +0,0 @@ -# CodeScribe - Team Setup (Pure Rust Era) - -> **Historical snapshot (2026-01-19).** CLI names here (`codescribe-quality`, -> `codescribe-loop`) were renamed in 0.9.0 to `qube-report` and `qube-daemon`. -> For the current team setup, see [`docs/TEAM_SETUP.md`](../TEAM_SETUP.md). - -## Quick Start - -### 1. Prerequisites - -- macOS 14+ (Apple Silicon ARM64 only) -- Rust 1.83+ - -### 2. Build & Run (CLI) - -```bash -# Clone -git clone git@github.com:VetCoders/CodeScribe.git -cd CodeScribe - -# Build and run CLI -cargo build --release -p codescribe -./target/release/codescribe -``` - -### 3. Development Mode - -```bash -# Run debug binary -cargo run -``` - -## Permissions Required - -Grant in: System Settings > Privacy & Security - -1. **Microphone** - for audio recording -2. **Accessibility** - for global hotkeys -3. **Input Monitoring** - for hotkey capture - -## Hotkeys - -| Key | Action | AI Mode | -| -------------------------- | --------------------------------------- | ------------------ | -| Hold **Ctrl** | Record → paste raw transcript | ALWAYS RAW (no AI) | -| Hold **Ctrl+Shift** | Record → AI assistant response | ALWAYS Assistive | -| Double-tap **Option** | Toggle recording (hands-free) | Respects AI toggle | -| Triple-tap **Option** | Toggle AI Formatting on/off | Shows toast | -| **Shift** during Ctrl hold | Upgrade to Assistive mode mid-recording | — | - -### Mode Behavior - -- **RAW mode (Ctrl)**: Fast dictation. Transcript is pasted as-is (only local repetition cleanup). - Ignores AI_FORMATTING_ENABLED setting. -- **Toggle mode (Double Option)**: Respects the AI Formatting toggle. If enabled, sends to AI - for formatting. If disabled, pastes raw. -- **Assistive mode (Ctrl+Shift)**: Full AI assistant. Model can answer questions, expand ideas, - or pass through dictation based on detected intent (KURIER/ASYSTENT system). - -## Model - -**Strictly Embedded (Release Policy)**: `whisper-large-v3-turbo-mlx-q8` (~888MB) - -- **Zero Exceptions:** Release binaries ALWAYS contain the model. -- **No external files:** We never bundle `Resources/models/*`. -- **Zero I/O:** Model loads from memory directly to Metal. - -**Developer note (Build Time):** -You still need the model files locally to _build_ the app (because they are `include_bytes!`-ed into the binary). - -```bash -make download-model # Required for build -``` - -Location (build-time only): `models/whisper-large-v3-turbo-mlx-q8/` - -## CLI Usage - -```bash -# Transcribe audio file -codescribe transcribe audio.wav - -# With AI formatting -codescribe transcribe audio.wav --format - -# Specify language -codescribe transcribe audio.wav --language pl -``` - -## Quality & Tools - -New CLI tools for batch processing and automation: - -```bash -# Batch quality report -codescribe-quality --help - -# Self-improving quality loop -codescribe-loop --help -``` - -## Configuration - -File: `~/.codescribe/.env` - -```env -USE_LOCAL_STT=1 - -# Whisper -WHISPER_LANGUAGE=pl - -# AI formatting (optional) - separate providers for formatting vs assistive -AI_FORMATTING_ENABLED=1 - -# Formatting mode (fast, cheap) - for Ctrl Hold with AI toggle -LLM_FORMATTING_ENDPOINT=https://api.libraxis.cloud/v1/responses -LLM_FORMATTING_MODEL=gpt-5-mini -LLM_FORMATTING_API_KEY=sk-xxx - -# Assistive mode (smart) - for Ctrl+Shift Hold -LLM_ASSISTIVE_ENDPOINT=https://api.libraxis.cloud/v1/responses -LLM_ASSISTIVE_MODEL=gpt-5.2 -LLM_ASSISTIVE_API_KEY=sk-xxx - -# Shared fallback (if mode-specific not set) -LLM_ENDPOINT=https://api.openai.com/v1/responses -LLM_MODEL=gpt-4.1-mini -LLM_API_KEY=sk-proj-xxx -``` - -### Custom Prompts - -Prompts are loaded from `~/.codescribe/prompts/` at each request (no restart needed): - -- `formatting.txt` - System prompt for formatting mode (punctuation, structure) -- `assistive.txt` - System prompt for assistive mode (KURIER/ASYSTENT logic) - -Edit these files to customize AI behavior. Changes take effect immediately. - -## Quality Assurance - -### Local (recommended) - -```bash -# Install pre-commit hooks (runs check/fmt on commit, clippy/semgrep on push) -make hooks - -# Manual quality gate -make check # fmt + clippy + unit tests - -# E2E tests with real API -make test-sse # SSE streaming tests (requires ~/.codescribe/.env) -``` - -### CI (GitHub Actions) - -**Note:** Full build requires macOS + Swift 6.0 (CoreML, Metal). GitHub runners have Swift 5.10, so CI only runs: - -- **Format check** (`cargo fmt --check`) on Linux -- **Semgrep** security scan on Linux - -Clippy and tests run **locally** via pre-commit hooks or `make check`. - -For full CI, configure a self-hosted macOS runner (Dragon recommended). - -## Troubleshooting - -### App doesn't start - -- Check Console.app for crash logs -- If building locally: ensure the model exists in `models/` (for embedding at build time) - -### Hotkeys don't work - -- Grant Accessibility permission -- Grant Input Monitoring permission -- Restart app after granting - -### No transcription - -- Check `USE_LOCAL_STT=1` in config -- If using local STT: confirm the app is using the embedded engine (default in release builds) - ---- - -_Copyright © 2024–2026 VetCoders_ diff --git a/docs/ADR/2026-01-25-ARCHITECTURE.md b/docs/ADR/2026-01-25-ARCHITECTURE.md deleted file mode 100644 index 60770a5d..00000000 --- a/docs/ADR/2026-01-25-ARCHITECTURE.md +++ /dev/null @@ -1,259 +0,0 @@ -# CodeScribe Architecture - -> Created by M&K (c)2026 VetCoders - -> **Historical snapshot (2026-01-25).** Paths and CLI names in this document -> reflect the pre-refactor layout. Current truth: -> - `src/voice_chat_ui.rs` → `app/ui/voice_chat/mod.rs` -> - `src/transcription_overlay.rs` → `app/ui/overlay/mod.rs` -> - `codescribe-quality` → `qube-report` (renamed in 0.9.0) -> - `codescribe-loop` → `qube-daemon` (renamed in 0.9.0) -> -> For the current architecture, see [`docs/ARCHITECTURE.md`](../ARCHITECTURE.md) -> and the repo-level [README](../../README.md). - -## System Overview - -```mermaid -flowchart TB - %% High-level packaging / layers - - subgraph APP[codescribe crate - bin/daemon] - direction LR - HK[hotkeys.rs] - CTRL[controller/] - IPC_SERVER[ipc/server.rs] - TRAY[tray/] - OVERLAY[voice_chat_ui/] - - subgraph CORE[codescribe-core crate] - direction LR - WH[whisper/] - CO[config/] - AU[audio/] - IPC_CORE[ipc types] - end - - APP --> CORE - end - - WH --> MODEL[Whisper Model\nlarge-v3-turbo\nmlx-q8 ~888MB\nembedded in bin] - - subgraph TOOLS[Quality & CLI Tools] - CLI[codescribe-quality] - LOOP[codescribe-loop] - end - - APP -.-> TOOLS -``` - -## Module Architecture - -### Recording Flow - -``` -┌─────────────┐ ┌────────────┐ ┌───────────────┐ ┌──────────────┐ -│ CGEventTap │───►│ hotkeys.rs │───►│ controller/ │───►│ whisper/ │ -│ (macOS API) │ │ │ │ mod.rs │ │ engine.rs │ -└─────────────┘ └────────────┘ └───────────────┘ └──────────────┘ - │ │ │ - │ ▼ ▼ - │ ┌──────────────┐ ┌──────────────┐ - │ │ voice_chat │ │ transcription│ - │ │ _ui/ │ │ _overlay.rs │ - │ └──────────────┘ └──────────────┘ - │ - Ctrl hold → Raw mode (no AI) - Ctrl+Shift hold → Assistive mode (AI) - Double Option → Toggle mode (respects AI setting) -``` - -### Voice Chat UI (Mission Control) - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ Status Header [Collapse] │ -├─────────────────────────────────────┬───────────────────────────┤ -│ LEFT PANEL (60%) │ RIGHT PANEL (40%) │ -│ │ │ -│ Chat bubbles (NSStackView) │ [Transcriptions][Settings]│ -│ ┌─────────────────────────────┐ │ │ -│ │ User message (blue, right) │ │ Draft files list │ -│ └─────────────────────────────┘ │ [Format] [Copy] [Augment] │ -│ ┌─────────────────────────┐ │ │ -│ │ AI response (gray,left) │ │ Settings toggles │ -│ └─────────────────────────┘ │ [Edit Config] [Edit Prompt]│ -│ │ │ -│ [Auto] [📎] [Input...] [Send] │ │ -└─────────────────────────────────────┴───────────────────────────┘ -``` - -## File Structure - -``` -CodeScribe/ -├── codescribe-core/ # Core library (portable, no macOS deps) -│ └── src/ -│ ├── whisper/ # Embedded + singleton Whisper engine -│ │ ├── engine.rs # Transcription logic -│ │ ├── singleton.rs # Global instance (lazy init) -│ │ └── embedded.rs # Model bytes (include_bytes!) -│ ├── audio/ # Recorder + StreamingRecorder -│ │ ├── recorder.rs # cpal audio capture -│ │ └── streaming_recorder.rs # Live transcription -│ ├── config/ # Configuration management -│ ├── stream_postprocess.rs # Semantic gating for live chunks -│ ├── quality_loop.rs # Self-improvement loop -│ ├── quality_report.rs # Batch quality reports -│ └── ipc/ # IPC types -│ -├── src/ # codescribe crate (macOS-specific) -│ ├── main.rs # CLI entry (daemon/transcribe) -│ ├── lib.rs # Library exports -│ │ -│ ├── controller/ # Recording state machine -│ │ ├── mod.rs # RecordingController impl -│ │ ├── types.rs # State, HotkeyInput, etc. -│ │ ├── helpers.rs # Session state, callbacks -│ │ └── tests.rs # Controller tests -│ │ -│ ├── voice_chat_ui/ # Voice Chat Overlay (Mission Control) -│ │ ├── mod.rs # UI creation (AppKit) -│ │ ├── state.rs # VoiceChatOverlayState -│ │ ├── handlers.rs # Objective-C callbacks -│ │ └── api.rs # Public API functions -│ │ -│ ├── tray/ # System tray menu -│ │ ├── mod.rs # Tray setup -│ │ ├── menu.rs # Menu creation -│ │ ├── handlers.rs # Menu actions -│ │ ├── icons.rs # Icon generation -│ │ └── types.rs # MenuIds, TrayMenuEvent -│ │ -│ ├── hotkeys.rs # CGEventTap handler -│ ├── transcription_overlay.rs # Simple text overlay -│ ├── ui.rs # Badge, Dock icon -│ ├── ui_helpers.rs # AppKit utilities -│ ├── clipboard.rs # Paste to active app -│ ├── permissions.rs # macOS permission checks -│ └── ipc/ # IPC server (Unix socket) -│ -├── src/bin/ # CLI tools -│ ├── codescribe_quality.rs # Batch quality reports -│ └── codescribe_loop.rs # Self-improving loop -│ -├── docs/ -│ ├── guide/ # User documentation -│ │ ├── README.md # Quick start -│ │ ├── installation.md -│ │ ├── modes.md -│ │ ├── chat-overlay.md -│ │ ├── settings.md -│ │ ├── troubleshooting.md -│ │ └── privacy.md -│ ├── ARCHITECTURE.md # This file -│ ├── WHISPER_LIVE.md # Streaming transcription -│ ├── TEAM_SETUP.md # Developer setup -│ └── future/ # Aspirational docs -│ ├── ARCHITECTURE_VISION.md -│ └── FEASIBILITY_ANALYSIS.md -│ -└── tests/ # Integration tests -``` - -## Key Components - -### Controller State Machine - -```rust -// src/controller/types.rs -pub enum State { - Idle, // Ready for input - RecHold, // Recording (hold mode) - RecToggle, // Recording (toggle mode) - Busy, // Processing transcription -} -``` - -State transitions: - -- `Idle` + Ctrl down → (800ms delay) → `RecHold` -- `Idle` + Double Option → `RecToggle` -- `RecHold` + Ctrl up → `Busy` → `Idle` -- `RecToggle` + Double Option → `Busy` → `Idle` -- `RecToggle` + 5s silence (VAD) → `Busy` → `Idle` - -### Mode Determination - -```rust -// src/controller/mod.rs - handle_hotkey_event() -match (hotkey, flags) { - (Hold, no_shift) => force_raw = true, // Ctrl: always raw - (Hold, shift) => assistive = true, // Ctrl+Shift: always AI - (Toggle, force_ai)=> force_ai = true, // Left Option x2: force AI - (Toggle, _) => /* respects AI_FORMATTING_ENABLED */ -} -``` - -### Voice Chat UI Components - -| Module | LOC | Purpose | -| ------------- | --- | -------------------------------- | -| `mod.rs` | 632 | UI creation with AppKit | -| `api.rs` | 589 | Public API (update_status, etc.) | -| `handlers.rs` | 450 | Objective-C action handlers | -| `state.rs` | 148 | VoiceChatOverlayState struct | - -### Whisper Engine - -- **Singleton pattern**: One global instance, lazy initialized -- **Metal acceleration**: Uses Apple GPU via candle-core -- **Streaming**: Chunks processed during recording -- **Embedded**: Model bytes in binary (~888MB) - -## Implementation Status - -| Feature | Status | -| -------------------------------------------- | ------ | -| Local Whisper STT (Metal GPU) | ✅ | -| Embedded model (~888MB binary) | ✅ | -| Global hotkeys (CGEventTap) | ✅ | -| Three recording modes (Raw/Assistive/Toggle) | ✅ | -| Voice Chat UI (split panel) | ✅ | -| Chat bubbles (NSStackView) | ✅ | -| Drafts panel with tabs | ✅ | -| Settings in overlay | ✅ | -| AI formatting (Responses API) | ✅ | -| Streaming AI responses | ✅ | -| Tray app with submenus | ✅ | -| History with slug filenames | ✅ | -| IPC server (runtime interface) | ✅ | -| Stream postprocess (semantic gating) | ✅ | -| Quality loop + report | ✅ | -| CodeScribe Core separation | ✅ | -| VAD (utterance boundary on silence) | ✅ | -| Transcription overlay | ✅ | -| Tauri GUI (future) | 📋 | - -## Model Location - -**Release Builds**: Model embedded via `include_bytes!` (~888MB total). -Zero disk I/O, model bytes loaded directly into GPU memory. - -**Development**: External model from: - -1. `CODESCRIBE_MODEL_PATH` environment variable -2. `~/.codescribe/models/whisper-large-v3-turbo-mlx-q8/` -3. `./models/whisper-large-v3-turbo-mlx-q8/` in repo - -## Related Documentation - -- [`guide/README.md`](guide/README.md) — User documentation -- [`WHISPER_LIVE.md`](WHISPER_LIVE.md) — Embedded + streaming transcription -- [`TEAM_SETUP.md`](TEAM_SETUP.md) — Developer setup guide -- [`BACKLOG.md`](BACKLOG.md) — Feature backlog -- [`future/ARCHITECTURE_VISION.md`](future/ARCHITECTURE_VISION.md) — Libraxis Qube Protocol vision - ---- - -**Made with ⌜ CodeScribe ⌟ by Maciej & Monika + Klaudiusz (AI) (c) 2024-2026** diff --git a/docs/ADR/2026-01-25-ARCHITECTURE_VISION.md b/docs/ADR/2026-01-25-ARCHITECTURE_VISION.md deleted file mode 100644 index cf2eb815..00000000 --- a/docs/ADR/2026-01-25-ARCHITECTURE_VISION.md +++ /dev/null @@ -1,186 +0,0 @@ -# Architecture Vision: The Libraxis Qube Protocol - -> **Status:** Draft / Conceptual -> **Date:** 2026-01-19 -> **Author:** Junie (AI) for VetCoders - -## 1. Core Concept: The Libraxis Qube - -We are renaming the central orchestration node (formerly "CodeScribe Core") to **The Libraxis Qube**. - -The **Libraxis Qube** is a central, location-agnostic **Stream Router & Orchestrator**. It is not just a backend for an app; -it is the "infinite cube" that holds the state of the conversation and manages all flows of information (Audio, Text, -Artifacts). - -### Key Philosophy: Deployment Neutrality - -The architecture removes the distinction between "Local" and "Cloud". - -- **Distance is irrelevant.** -- The Libraxis Qube can run on `localhost` (your laptop) or on `Dragon` (remote workstation). -- The **Client** (Microphone/Speaker) connects to the Libraxis Qube via a **WebSocket**. -- Whether that WebSocket connects to `ws://127.0.0.1:8000` or `wss://dragon.lan:8000` changes nothing in the system - logic. It only changes the latency of the wire. - -## 2. System Topology - -```mermaid -graph TD - subgraph Client [User / Client Side] - Mic[Microphone] - Speaker[Speaker] - UI[Visual Overlay/Logs] - end - - subgraph Libraxis QubeNode [The Libraxis Qube Node] - Router[Libraxis Qube Orchestrator
(State, Routing, Demux)] - - subgraph Modules [Attached Modules] - ASR[Candle Transformers
(Whisper v3 Turbo - Streaming)] - Agent[LLM Agent
(Reasoning & Tools)] - TTS[TTS Engine
(Voice Synthesis)] - Artifacts[Artifact Generator
(PDF, Files, JSON)] - end - end - - %% Connections - Mic ==>|"WebSocket Stream (Audio Bytes)"| Router - Router ==>|"WebSocket Stream (Audio/Events)"| Speaker - Router -.->|"Updates (SSE/WS)"| UI - - %% Internal Flows - Router <==>|"Raw Audio / Tokens"| ASR - Router <==>|"Context / Intention"| Agent - Router ==>|"Text to Speak"| TTS - TTS ==>|"Audio Bytes"| Router - Router ==>|"Content"| Artifacts -``` - -## 3. The "Single Stream" Protocol - -To maintain synchronization and simplicity, the Agent/Model does not open multiple TCP connections. Instead, it emits a -**single continuous stream** of tokens. The Libraxis Qube is responsible for **demuxing** (splitting) this stream based on -**Tags**. - -### Stream Structure - -The stream contains interspersed content types, delimited by XML-like tags or special markers. - -**Example Stream Flow (Server to Client):** - -```text -[STREAM_START] -... (thinking tokens, internal log) ... -Here is the summary of the patient's condition. -[ROUTER DETECTS TAG -> Routes content to TTS -> Sends Audio Bytes to Client] -... - - { "patient": "Fretka Ziggy", "diagnosis": "Healthy" } - -[ROUTER DETECTS TAG -> Routes content to PDF Generator -> Saves File -> Notifies Client] -... -Displaying chart... -[ROUTER DETECTS TAG -> Routes to Overlay] -... -[STREAM_END] -``` - -### Routing Logic (The Demuxer) - -1. **Default Channel**: Tokens flow to the internal buffer/context. -2. **Audio Channel (`` / `

CodeScribe Quality Report

Generated: {}

Metrics reference: {}

Raw semantics: text_committed={} • quality_gate_dropped={} • no_speech_detected={}

", + "

Codescribe Quality Report

Generated: {}

Metrics reference: {}

Raw semantics: text_committed={} • quality_gate_dropped={} • no_speech_detected={}

", html_escape(&report.generated_at), html_escape(&report.environment.metrics_reference), report.summary.raw_text_committed, @@ -952,7 +952,7 @@ fn render_html(report: &QualityReport, config: &QualityReportConfig) -> String { -CodeScribe Quality Report +Codescribe Quality Report