diff --git a/docs/ENGINE.md b/docs/ENGINE.md index 95d9340e..a81a03cb 100644 --- a/docs/ENGINE.md +++ b/docs/ENGINE.md @@ -746,12 +746,16 @@ provisioned. Mutating routes require the `assistant` token capability. See accepts an optional `utterance` (the raw recognised text before the repair pass) so a repaired turn logs both forms. - `POST /api/assistant/reset` -> `OkResponse` -- `POST /api/assistant/stt` (multipart audio, field `file`) -> `{"text": string}` - — server-side transcription. Proxies to `/audio/transcriptions` on an - ordered, independently-configured base-URL list (voicemode semantics: local - first, cloud fallback). **404** when unconfigured or every entry fails, so the - client falls back. Scope-gated on `assistant` (a POST under - `/api/assistant`). Audio is proxied, never stored. +- `POST /api/assistant/stt` (multipart audio, field `file`, optional text + field `prompt`) -> `{"text": string}` — server-side transcription. Proxies to + `/audio/transcriptions` on an ordered, independently-configured base-URL list + (voicemode semantics: local first, cloud fallback). A `prompt` field is + forwarded to the backend as its bias prompt (trimmed and bounded to 2 KiB) — + the mobile client sends the deployment's project and session names there, so + a transcriber that has never heard `komodo` is told to expect it; a backend + that ignores the field is no worse off. **404** when unconfigured or every + entry fails, so the client falls back. Scope-gated on `assistant` (a POST + under `/api/assistant`). Audio and prompt are proxied, never stored. - `POST /api/assistant/tts` `{"text": "..."}` -> an audio stream (`audio/*`) — server-side synthesis. Proxies `{model, input, voice}` to `/audio/speech` on the same kind of ordered list. **404** when unconfigured/all-failed. Audio is diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index d38ef15b..4cbd09d9 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -479,6 +479,13 @@ imperfect echo cancellation can cut a reply short by mistake). Turn timing is tunable per device through the `vogt.assistant.voice.*` browser settings (silence window, idle timeout, barge-in); the defaults suit a phone. +When the deployment offers server-side transcription, Settings shows +**Transcribe voice on the server**. On by default a phone uses its own +recognizer, which is fast but has never heard your project names; turning this +on sends captured audio to the server transcriber instead, handed those names +as a hint, so "check komodo on Node B" is far likelier to come through as the +words you said. It takes effect the next time the app launches. + An approved write is audited to **your** actor, using the core token paired with the token that pressed approve. There is no shared "assistant" actor to fall back to; an unpaired approver is refused by name. diff --git a/engine/server/src/assistant_speech.rs b/engine/server/src/assistant_speech.rs index d263a20c..2973b21a 100644 --- a/engine/server/src/assistant_speech.rs +++ b/engine/server/src/assistant_speech.rs @@ -178,11 +178,25 @@ pub async fn stt( // Take the first file-bearing field. A voice client sends one audio blob; // we do not care what it named the field, only that it carried bytes. let mut audio: Option<(Vec, String, String)> = None; + // An optional `prompt` text field: the domain vocabulary the client + // biases the transcriber with (project slugs, session names, the words a + // recognizer mangles). Whisper-family backends take it as `prompt`; a + // backend that ignores the field is no worse off. Not read as audio, and + // not required — a client that sends none transcribes as before. + let mut prompt: Option = None; while let Some(field) = multipart .next_field() .await .map_err(|e| ApiError::BadRequest(format!("reading upload: {e}")))? { + if field.name() == Some("prompt") { + let text = field + .text() + .await + .map_err(|e| ApiError::BadRequest(format!("reading prompt: {e}")))?; + prompt = clamp_stt_prompt(&text); + continue; + } let file_name = field .file_name() .map(str::to_owned) @@ -195,9 +209,10 @@ pub async fn stt( .bytes() .await .map_err(|e| ApiError::BadRequest(format!("reading upload bytes: {e}")))?; - if !bytes.is_empty() { + // First file-bearing field wins; keep scanning so a `prompt` sent + // after the audio is still read rather than dropped on an early break. + if audio.is_none() && !bytes.is_empty() { audio = Some((bytes.to_vec(), file_name, content_type)); - break; } } let (bytes, file_name, content_type) = @@ -216,9 +231,12 @@ pub async fn stt( .mime_str("application/octet-stream") .expect("octet-stream is a valid mime") }); - let form = reqwest::multipart::Form::new() + let mut form = reqwest::multipart::Form::new() .text("model", backend.model.clone()) .part("file", part); + if let Some(prompt) = &prompt { + form = form.text("prompt", prompt.clone()); + } let url = format!("{}/audio/transcriptions", base_url.trim_end_matches('/')); let mut request = speech @@ -328,6 +346,21 @@ pub async fn tts(State(state): State>, Json(req): Json) -> Err(ApiError::NotFound) } +/// The largest vocabulary prompt forwarded to a transcription backend. A +/// prompt is a bias, not a document; a long one crowds the audio's own tokens +/// and some backends reject an oversized one outright. +const MAX_STT_PROMPT_BYTES: usize = 2048; + +/// A client-supplied STT bias prompt, trimmed and bounded, or nothing when it +/// is empty — an empty `prompt` field is the same as sending none. +fn clamp_stt_prompt(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + Some(truncate(trimmed, MAX_STT_PROMPT_BYTES)) +} + fn truncate(s: &str, max: usize) -> String { if s.len() <= max { s.to_string() @@ -340,3 +373,29 @@ fn truncate(s: &str, max: usize) -> String { format!("{}…", &s[..end]) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_prompt_is_trimmed_and_empty_becomes_none() { + assert_eq!(clamp_stt_prompt(" "), None); + assert_eq!(clamp_stt_prompt(""), None); + assert_eq!( + clamp_stt_prompt(" projects: komodo, vogt ").as_deref(), + Some("projects: komodo, vogt") + ); + } + + #[test] + fn a_long_prompt_is_bounded() { + let long = "komodo ".repeat(1000); + let clamped = clamp_stt_prompt(&long).expect("non-empty"); + // `truncate` keeps up to the byte budget then marks the cut with a + // single-char ellipsis, so the bound is the budget plus that marker. + assert!(clamped.len() <= MAX_STT_PROMPT_BYTES + "…".len()); + assert!(clamped.len() < long.len()); + assert!(clamped.ends_with('…')); + } +} diff --git a/web/src/Assistant.tsx b/web/src/Assistant.tsx index cf6d1ca8..f30b578e 100644 --- a/web/src/Assistant.tsx +++ b/web/src/Assistant.tsx @@ -37,6 +37,7 @@ import { writeClipboardText } from "./clipboard"; import { describeRepairs, repairUtterance } from "./voiceRepair"; import { readToolDraft, writeToolDraft } from "./toolDrafts"; import { pendingAction, setPendingAction } from "./pendingAction"; +import { getPreferServerStt, sttVocabularyPrompt } from "./sttPref"; import { isConnected, sessionsError, sessionsStore } from "./store"; import { deferAssistantHydration, @@ -633,6 +634,14 @@ export default function Assistant(props: AssistantProps) { const configureSpeechInput = async () => { if (sttBackend || sttAvailable()) return; + // A device whose on-device recognizer keeps mangling the vocabulary can + // opt to transcribe on the server instead, where the names bias the + // decode. Honoured only when the deployment actually offers server STT. + if (getPreferServerStt() && serverSttEnabled() && mediaRecorderAvailable()) { + sttBackend = "server"; + setSttAvailable(true); + return; + } if (Capacitor.isPluginAvailable("SpeechRecognition")) { try { const { SpeechRecognition } = await import( @@ -1133,7 +1142,11 @@ export default function Assistant(props: AssistantProps) { const controller = new AbortController(); transcriptionController = controller; try { - const { text: heard } = await api.assistantStt(blob, controller.signal); + const { text: heard } = await api.assistantStt( + blob, + controller.signal, + sttVocabularyPrompt(slugs()), + ); if (controller.signal.aborted || !heard.trim()) return; setSpeechStatus(""); const { text: repairedText, repairs } = repairUtterance(heard, slugs()); diff --git a/web/src/Settings.tsx b/web/src/Settings.tsx index 1680c6b5..546a5b62 100644 --- a/web/src/Settings.tsx +++ b/web/src/Settings.tsx @@ -16,6 +16,7 @@ import { type PushSubscriptionEntry, } from "./api"; import { getLayoutMode, setLayoutMode, type LayoutMode } from "./layout"; +import { getPreferServerStt, setPreferServerStt } from "./sttPref"; import TemplateEditor from "./TemplateEditor"; import Dialog from "./Dialog"; import { THEMES, getThemeName, setThemeName } from "./terminalThemes"; @@ -167,6 +168,7 @@ const Settings: Component = (props) => { >("idle"); const [authCheckMsg, setAuthCheckMsg] = createSignal(null); const [layoutMode, setL] = createSignal(getLayoutMode()); + const [preferServerStt, setPreferServerSttSig] = createSignal(getPreferServerStt()); const [pushOn, setPushOn] = createSignal(false); const [pushPerm, setPushPerm] = createSignal("default"); const [pushBusy, setPushBusy] = createSignal(false); @@ -408,6 +410,7 @@ const Settings: Component = (props) => { setAuthCheck("idle"); setAuthCheckMsg(null); setL(getLayoutMode()); + setPreferServerSttSig(getPreferServerStt()); setAppThemeSel(getAppThemeSelection()); setTerminalTheme(getThemeName()); setStoragePrefsState(getStoragePrefs()); @@ -907,6 +910,24 @@ const Settings: Component = (props) => { spellcheck={false} /> + + +