feat(voice): server-STT vocabulary bias + a device preference for server transcription

docs/ENGINE.md

@@ -746,12 +746,16 @@ provisioned. Mutating routes require the `assistant` token capability. See
   accepts an optional `utterance` (the raw recognised text before the repair
   pass) so a repaired turn logs both forms.
 - `POST /api/assistant/reset` -> `OkResponse`
-- `POST /api/assistant/stt` (multipart audio, field `file`) -> `{"text": string}`
-  — server-side transcription. Proxies to `/audio/transcriptions` on an
-  ordered, independently-configured base-URL list (voicemode semantics: local
-  first, cloud fallback). **404** when unconfigured or every entry fails, so the
-  client falls back. Scope-gated on `assistant` (a POST under
-  `/api/assistant`). Audio is proxied, never stored.
+- `POST /api/assistant/stt` (multipart audio, field `file`, optional text
+  field `prompt`) -> `{"text": string}` — server-side transcription. Proxies to
+  `/audio/transcriptions` on an ordered, independently-configured base-URL list
+  (voicemode semantics: local first, cloud fallback). A `prompt` field is
+  forwarded to the backend as its bias prompt (trimmed and bounded to 2 KiB) —
+  the mobile client sends the deployment's project and session names there, so
+  a transcriber that has never heard `komodo` is told to expect it; a backend
+  that ignores the field is no worse off. **404** when unconfigured or every
+  entry fails, so the client falls back. Scope-gated on `assistant` (a POST
+  under `/api/assistant`). Audio and prompt are proxied, never stored.
 - `POST /api/assistant/tts` `{"text": "..."}` -> an audio stream (`audio/*`) —
   server-side synthesis. Proxies `{model, input, voice}` to `/audio/speech` on
   the same kind of ordered list. **404** when unconfigured/all-failed. Audio is

docs/USER_GUIDE.md

@@ -479,6 +479,13 @@ imperfect echo cancellation can cut a reply short by mistake). Turn timing is
 tunable per device through the `vogt.assistant.voice.*` browser settings (silence
 window, idle timeout, barge-in); the defaults suit a phone.
 
+When the deployment offers server-side transcription, Settings shows
+**Transcribe voice on the server**. On by default a phone uses its own
+recognizer, which is fast but has never heard your project names; turning this
+on sends captured audio to the server transcriber instead, handed those names
+as a hint, so "check komodo on Node B" is far likelier to come through as the
+words you said. It takes effect the next time the app launches.
+
 An approved write is audited to **your** actor, using the core token paired with
 the token that pressed approve. There is no shared "assistant" actor to fall
 back to; an unpaired approver is refused by name.

engine/server/src/assistant_speech.rs

@@ -178,11 +178,25 @@ pub async fn stt(
     // Take the first file-bearing field. A voice client sends one audio blob;
     // we do not care what it named the field, only that it carried bytes.
     let mut audio: Option<(Vec<u8>, String, String)> = None;
+    // An optional `prompt` text field: the domain vocabulary the client
+    // biases the transcriber with (project slugs, session names, the words a
+    // recognizer mangles). Whisper-family backends take it as `prompt`; a
+    // backend that ignores the field is no worse off. Not read as audio, and
+    // not required — a client that sends none transcribes as before.
+    let mut prompt: Option<String> = None;
     while let Some(field) = multipart
         .next_field()
         .await
         .map_err(|e| ApiError::BadRequest(format!("reading upload: {e}")))?
     {
+        if field.name() == Some("prompt") {
+            let text = field
+                .text()
+                .await
+                .map_err(|e| ApiError::BadRequest(format!("reading prompt: {e}")))?;
+            prompt = clamp_stt_prompt(&text);
+            continue;
+        }
         let file_name = field
             .file_name()
             .map(str::to_owned)
@@ -195,9 +209,10 @@ pub async fn stt(
             .bytes()
             .await
             .map_err(|e| ApiError::BadRequest(format!("reading upload bytes: {e}")))?;
-        if !bytes.is_empty() {
+        // First file-bearing field wins; keep scanning so a `prompt` sent
+        // after the audio is still read rather than dropped on an early break.
+        if audio.is_none() && !bytes.is_empty() {
             audio = Some((bytes.to_vec(), file_name, content_type));
-            break;
         }
     }
     let (bytes, file_name, content_type) =
@@ -216,9 +231,12 @@ pub async fn stt(
                     .mime_str("application/octet-stream")
                     .expect("octet-stream is a valid mime")
             });
-        let form = reqwest::multipart::Form::new()
+        let mut form = reqwest::multipart::Form::new()
             .text("model", backend.model.clone())
             .part("file", part);
+        if let Some(prompt) = &prompt {
+            form = form.text("prompt", prompt.clone());
+        }
 
         let url = format!("{}/audio/transcriptions", base_url.trim_end_matches('/'));
         let mut request = speech
@@ -328,6 +346,21 @@ pub async fn tts(State(state): State<Arc<AppState>>, Json(req): Json<TtsReq>) ->
     Err(ApiError::NotFound)
 }
 
+/// The largest vocabulary prompt forwarded to a transcription backend. A
+/// prompt is a bias, not a document; a long one crowds the audio's own tokens
+/// and some backends reject an oversized one outright.
+const MAX_STT_PROMPT_BYTES: usize = 2048;
+
+/// A client-supplied STT bias prompt, trimmed and bounded, or nothing when it
+/// is empty — an empty `prompt` field is the same as sending none.
+fn clamp_stt_prompt(raw: &str) -> Option<String> {
+    let trimmed = raw.trim();
+    if trimmed.is_empty() {
+        return None;
+    }
+    Some(truncate(trimmed, MAX_STT_PROMPT_BYTES))
+}
+
 fn truncate(s: &str, max: usize) -> String {
     if s.len() <= max {
         s.to_string()
@@ -340,3 +373,29 @@ fn truncate(s: &str, max: usize) -> String {
         format!("{}…", &s[..end])
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_prompt_is_trimmed_and_empty_becomes_none() {
+        assert_eq!(clamp_stt_prompt("   "), None);
+        assert_eq!(clamp_stt_prompt(""), None);
+        assert_eq!(
+            clamp_stt_prompt("  projects: komodo, vogt  ").as_deref(),
+            Some("projects: komodo, vogt")
+        );
+    }
+
+    #[test]
+    fn a_long_prompt_is_bounded() {
+        let long = "komodo ".repeat(1000);
+        let clamped = clamp_stt_prompt(&long).expect("non-empty");
+        // `truncate` keeps up to the byte budget then marks the cut with a
+        // single-char ellipsis, so the bound is the budget plus that marker.
+        assert!(clamped.len() <= MAX_STT_PROMPT_BYTES + "…".len());
+        assert!(clamped.len() < long.len());
+        assert!(clamped.ends_with('…'));
+    }
+}

web/src/Assistant.tsx

@@ -37,6 +37,7 @@ import { writeClipboardText } from "./clipboard";
 import { describeRepairs, repairUtterance } from "./voiceRepair";
 import { readToolDraft, writeToolDraft } from "./toolDrafts";
 import { pendingAction, setPendingAction } from "./pendingAction";
+import { getPreferServerStt, sttVocabularyPrompt } from "./sttPref";
 import { isConnected, sessionsError, sessionsStore } from "./store";
 import {
   deferAssistantHydration,
@@ -633,6 +634,14 @@ export default function Assistant(props: AssistantProps) {
 
   const configureSpeechInput = async () => {
     if (sttBackend || sttAvailable()) return;
+    // A device whose on-device recognizer keeps mangling the vocabulary can
+    // opt to transcribe on the server instead, where the names bias the
+    // decode. Honoured only when the deployment actually offers server STT.
+    if (getPreferServerStt() && serverSttEnabled() && mediaRecorderAvailable()) {
+      sttBackend = "server";
+      setSttAvailable(true);
+      return;
+    }
     if (Capacitor.isPluginAvailable("SpeechRecognition")) {
       try {
         const { SpeechRecognition } = await import(
@@ -1133,7 +1142,11 @@ export default function Assistant(props: AssistantProps) {
     const controller = new AbortController();
     transcriptionController = controller;
     try {
-      const { text: heard } = await api.assistantStt(blob, controller.signal);
+      const { text: heard } = await api.assistantStt(
+        blob,
+        controller.signal,
+        sttVocabularyPrompt(slugs()),
+      );
       if (controller.signal.aborted || !heard.trim()) return;
       setSpeechStatus("");
       const { text: repairedText, repairs } = repairUtterance(heard, slugs());

web/src/Settings.tsx

@@ -16,6 +16,7 @@ import {
   type PushSubscriptionEntry,
 } from "./api";
 import { getLayoutMode, setLayoutMode, type LayoutMode } from "./layout";
+import { getPreferServerStt, setPreferServerStt } from "./sttPref";
 import TemplateEditor from "./TemplateEditor";
 import Dialog from "./Dialog";
 import { THEMES, getThemeName, setThemeName } from "./terminalThemes";
@@ -167,6 +168,7 @@ const Settings: Component<Props> = (props) => {
   >("idle");
   const [authCheckMsg, setAuthCheckMsg] = createSignal<string | null>(null);
   const [layoutMode, setL] = createSignal<LayoutMode>(getLayoutMode());
+  const [preferServerStt, setPreferServerSttSig] = createSignal(getPreferServerStt());
   const [pushOn, setPushOn] = createSignal(false);
   const [pushPerm, setPushPerm] = createSignal<PushPermissionState>("default");
   const [pushBusy, setPushBusy] = createSignal(false);
@@ -408,6 +410,7 @@ const Settings: Component<Props> = (props) => {
     setAuthCheck("idle");
     setAuthCheckMsg(null);
     setL(getLayoutMode());
+    setPreferServerSttSig(getPreferServerStt());
     setAppThemeSel(getAppThemeSelection());
     setTerminalTheme(getThemeName());
     setStoragePrefsState(getStoragePrefs());
@@ -907,6 +910,24 @@ const Settings: Component<Props> = (props) => {
               spellcheck={false}
             />
           </label>
+          <Show when={props.publicConfig?.assistant_stt_enabled}>
+            <label
+              class="settings-show-token"
+              title="Send captured audio to the server transcriber, which is given this deployment's project and session names as a hint. Helps when the on-device recognizer mishears them."
+            >
+              <input
+                type="checkbox"
+                checked={preferServerStt()}
+                onChange={(e) => {
+                  const on = e.currentTarget.checked;
+                  setPreferServerStt(on);
+                  setPreferServerSttSig(on);
+                }}
+              />
+              Transcribe voice on the server (better with project names; takes
+              effect next launch)
+            </label>
+          </Show>
           <div style={{ display: "flex", "flex-direction": "column", gap: "6px" }}>
             <button
               type="button"

web/src/__tests__/sttPref.test.ts

@@ -0,0 +1,32 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import {
+  getPreferServerStt,
+  setPreferServerStt,
+  sttVocabularyPrompt,
+} from "../sttPref";
+
+beforeEach(() => localStorage.clear());
+afterEach(() => localStorage.clear());
+
+describe("sttPref", () => {
+  it("prefer-server defaults off and round-trips", () => {
+    expect(getPreferServerStt()).toBe(false);
+    setPreferServerStt(true);
+    expect(getPreferServerStt()).toBe(true);
+    setPreferServerStt(false);
+    expect(getPreferServerStt()).toBe(false);
+  });
+
+  it("builds a vocabulary prompt from project slugs plus fixed terms", () => {
+    const prompt = sttVocabularyPrompt(["komodo", "vogt", " ", "rustnzbd"]);
+    expect(prompt).toContain("Project names: komodo, vogt, rustnzbd.");
+    expect(prompt).toContain("shell");
+    expect(prompt).not.toContain(",  ,"); // blank slug dropped
+  });
+
+  it("omits the project line when there are no slugs", () => {
+    const prompt = sttVocabularyPrompt([]);
+    expect(prompt).not.toContain("Project names");
+    expect(prompt).toContain("Terms:");
+  });
+});

web/src/api.ts

@@ -1058,11 +1058,19 @@ export const api = {
    * when the route is unconfigured, which the caller reads as "fall back"
    * — audio is proxied, never stored.
    */
-  assistantStt: async (audio: Blob, signal?: AbortSignal): Promise<{ text: string }> => {
+  assistantStt: async (
+    audio: Blob,
+    signal?: AbortSignal,
+    prompt?: string,
+  ): Promise<{ text: string }> => {
     const form = new FormData();
     // The engine forwards the first file-bearing field regardless of name; the
     // filename's extension hints the provider at the container.
     form.append("file", audio, "take.webm");
+    // An optional vocabulary bias — project and session names the transcriber
+    // has never heard — forwarded to the backend as `prompt`. Omitted when
+    // empty so the request is byte-for-byte the old one.
+    if (prompt && prompt.trim()) form.append("prompt", prompt.trim());
     const res = await runtimeTransport().request(`${getBase()}/api/assistant/stt`, {
       method: "POST",
       headers: authHeaders(),

web/src/sttPref.ts

@@ -0,0 +1,59 @@
+// Device-local speech-to-text preferences and the vocabulary bias a server
+// transcriber is given.
+//
+// On a phone the on-device recognizer is chosen first, and it mishears every
+// name it has never been taught — project slugs, session names, "shell" as
+// "show". A server transcriber (Whisper) can be handed those names as a bias
+// prompt, so this exists for the person whose device recognizer keeps
+// mangling the vocabulary: turn it on and captured audio goes to the server
+// with the names attached instead.
+
+const PREFER_SERVER_STT_KEY = "vogt.assistant.stt.prefer_server";
+
+/** Whether this device should transcribe via the server ahead of any on-device recognizer. */
+export function getPreferServerStt(): boolean {
+  try {
+    return localStorage.getItem(PREFER_SERVER_STT_KEY) === "1";
+  } catch {
+    return false;
+  }
+}
+
+export function setPreferServerStt(on: boolean): void {
+  try {
+    localStorage.setItem(PREFER_SERVER_STT_KEY, on ? "1" : "0");
+  } catch {
+    // localStorage unavailable — the preference simply does not persist.
+  }
+}
+
+/** Words worth teaching the server transcriber, beyond the project names: the
+ *  session vocabulary a supervisor request is built from. Kept short — a bias
+ *  prompt is a hint, not a document. */
+const DOMAIN_TERMS = [
+  "shell",
+  "session",
+  "terminal",
+  "Vogt",
+  "backlog",
+  "work item",
+] as const;
+
+/**
+ * The vocabulary bias prompt for a server transcription, built from the
+ * project slugs this client knows plus a few fixed domain terms. Empty when
+ * there are no slugs and nothing to bias toward, so the caller can omit the
+ * field entirely. Bounded so a large registry does not send a paragraph.
+ */
+export function sttVocabularyPrompt(slugs: readonly string[]): string {
+  const projects = slugs
+    .map((slug) => slug.trim())
+    .filter((slug) => slug.length > 0)
+    .slice(0, 64);
+  const parts: string[] = [];
+  if (projects.length > 0) {
+    parts.push(`Project names: ${projects.join(", ")}.`);
+  }
+  parts.push(`Terms: ${DOMAIN_TERMS.join(", ")}.`);
+  return parts.join(" ");
+}