Voice assistant: hands-free barge-in (WI-174 v2)

docs/ENGINE.md

@@ -1802,16 +1802,24 @@ become instructions.
   `idle → arming → listening → endpointing → sending → speaking → listening`,
   plus `paused_for_approval` (a pending write is announced, the mic closed
   until the on-screen approve/deny — voice still never approves) and a `muted`
-  flag that keeps the session alive. v1 is half-duplex (mic closed while a
-  reply plays; barge-in is v2). Turn detection is client-owned on every
-  backend, tuned by localStorage in the OpenAI Realtime vocabulary so a
-  Realtime-shaped backend adopts it unrenamed:
-  `vogt.assistant.voice.silence_duration_ms` (1000),
+  flag that keeps the session alive. Default is half-duplex (mic closed while a
+  reply plays). Turn detection is client-owned on every backend, tuned by
+  localStorage in the OpenAI Realtime vocabulary so a Realtime-shaped backend
+  adopts it unrenamed: `vogt.assistant.voice.silence_duration_ms` (1000),
   `final_result_grace_ms` (300), `max_turn_ms` (30000),
   `idle_timeout_ms` (60000), `max_empty_turns` (3),
-  `interrupt_response` (false in v1). Requires an event-driven recognizer
+  `interrupt_response` (false). Requires an event-driven recognizer
   (native plugin or Web Speech) and a TTS path; the server-STT path is excluded
-  in v1 and the control is disabled with its reason.
+  and the control is disabled with its reason.
+- **Barge-in** (opt-in, `interrupt_response=1`) — the speaker can talk over a
+  playing reply: while a reply plays, an echo-cancelled `getUserMedia` capture
+  runs a leaky-accumulator onset detector (`voiceVad.ts`, `vad_threshold` 0.045,
+  `vad_onset_ms` 500), and a sustained onset halts the reply (`stopSpeaking`) and
+  re-opens the mic to catch the interruption. The onset logic is a pure module,
+  unit-tested off frame energies; the capture is a thin Web Audio shell that a
+  stronger model (e.g. a WASM Silero VAD) can replace behind the same seam. Off
+  by default because the WebView's AEC residual is imperfect and a false trigger
+  cuts a reply short; half-duplex is the safe default.
 
 There is no setting that lets the assistant type without asking. The
 convenience it would buy a trusted single-user setup is outweighed by what it

docs/USER_GUIDE.md

@@ -471,9 +471,13 @@ for approval / Muted / Ended; in a conversation the microphone button mutes and
 unmutes with a tap (or **M** on the desktop), and a pending write still waits for
 an on-screen approval — the announcement says so and never offers a spoken yes.
 The conversation ends when you turn it off, leave the surface, mute-and-forget it
-past the idle timeout, or hear nothing for a few turns. Turn timing is tunable
-per device through `vogt.assistant.voice.*` browser settings (silence window,
-idle timeout); the defaults suit a phone.
+past the idle timeout, or hear nothing for a few turns. By default the microphone
+is closed while a reply is spoken and re-opens when it finishes; setting
+`vogt.assistant.voice.interrupt_response` on lets you **talk over a reply** — the
+device stops speaking and listens the moment you start (off by default, because
+imperfect echo cancellation can cut a reply short by mistake). Turn timing is
+tunable per device through the `vogt.assistant.voice.*` browser settings (silence
+window, idle timeout, barge-in); the defaults suit a phone.
 
 An approved write is audited to **your** actor, using the core token paired with
 the token that pressed approve. There is no shared "assistant" actor to fall

web/src/Assistant.tsx

@@ -56,6 +56,7 @@ import {
   type VoicePorts,
   type VoiceState,
 } from "./voiceTurn";
+import { startBargeInDetection } from "./voiceVad";
 
 const TTS_PREF_KEY = "vogt.assistant.tts";
 
@@ -383,6 +384,11 @@ export default function Assistant(props: AssistantProps) {
   // The hands-free loop, created lazily when Conversation is turned on. Declared
   // here because `applyReply` and `send` feed it their outcomes.
   let conversation: VoiceConversation | null = null;
+  // Whether barge-in (v2) is enabled for this session — read once when the
+  // conversation begins. Gates the echo-cancelled detection capture below; the
+  // machine ignores `speechDetected` when it is off, so this is only to avoid
+  // opening a second mic during playback for nothing.
+  let bargeInEnabled = false;
 
   const haltSpeech = () => {
     speechController?.abort();
@@ -1293,11 +1299,24 @@ export default function Assistant(props: AssistantProps) {
     }
     // Prime the synth inside the user gesture — the Android WebView requires it.
     window.speechSynthesis?.speak(new SpeechSynthesisUtterance(""));
-    conversation = new VoiceConversation(conversationPorts, readVoiceConfig());
+    const cfg = readVoiceConfig();
+    bargeInEnabled = cfg.interrupt_response;
+    conversation = new VoiceConversation(conversationPorts, cfg);
     setConversationOn(true);
     conversation.begin();
   };
 
+  // Barge-in (v2): while a reply is playing and interrupt_response is on, run an
+  // echo-cancelled capture that lets the speaker cut in. An onset halts the
+  // reply and re-opens the mic (the machine's `speechDetected`). The capture is
+  // open only during `speaking`, and torn down the moment that ends — so the
+  // second microphone is never held longer than the reply it listens over.
+  createEffect(() => {
+    if (!conversationOn() || !bargeInEnabled || voiceState() !== "speaking") return;
+    const stop = startBargeInDetection(() => conversation?.speechDetected());
+    onCleanup(stop);
+  });
+
   // Desktop shortcut: `M` mutes/unmutes an active conversation, unless the
   // caret is in a text field (where `m` is just a letter).
   createEffect(() => {

web/src/__tests__/voiceTurn.test.ts

@@ -172,6 +172,46 @@ describe("the hands-free conversation loop", () => {
     expect(vc.getState()).toBe("arming");
   });
 
+  it("ignores barge-in while a reply plays when interrupt_response is off (the default)", () => {
+    const { vc, clock, count } = live({ silence_duration_ms: 100 });
+    vc.partial("hello");
+    clock.advance(100);
+    vc.replied({ text: "a long spoken reply", hasPendingAction: false });
+    expect(vc.getState()).toBe("speaking");
+    const opensBefore = count("openMic");
+    vc.speechDetected();
+    expect(vc.getState()).toBe("speaking"); // half-duplex: no interruption
+    expect(count("openMic")).toBe(opensBefore);
+  });
+
+  it("barges in when interrupt_response is on: halts the reply and re-opens the mic", () => {
+    const { vc, clock, events, count } = live({
+      silence_duration_ms: 100,
+      interrupt_response: true,
+    });
+    vc.partial("hello");
+    clock.advance(100);
+    vc.replied({ text: "a long spoken reply", hasPendingAction: false });
+    expect(vc.getState()).toBe("speaking");
+    const opensBefore = count("openMic");
+    vc.speechDetected();
+    expect(events).toContain("stopSpeaking");
+    expect(vc.getState()).toBe("arming"); // re-opened to capture the interruption
+    expect(count("openMic")).toBe(opensBefore + 1);
+    // The halted reply's own speechFinished now arrives, and is ignored — we
+    // have already left `speaking`.
+    vc.speechFinished();
+    expect(vc.getState()).toBe("arming");
+  });
+
+  it("does not barge in outside playback, even with interrupt_response on", () => {
+    const { vc } = live({ interrupt_response: true });
+    // In `listening`, a stray VAD onset must not disturb the turn.
+    expect(vc.getState()).toBe("listening");
+    vc.speechDetected();
+    expect(vc.getState()).toBe("listening");
+  });
+
   it("keeps the session alive when muted, and captures nothing until unmuted", () => {
     const { clock, vc, events, count } = live({ silence_duration_ms: 1000 });
     vc.toggleMute();

web/src/__tests__/voiceVad.test.ts

@@ -0,0 +1,97 @@
+// The barge-in onset decision, proven off a list of frame energies.
+//
+// The capture shell (getUserMedia + Web Audio) is a thin, jsdom-hostile wrapper
+// and is left to the browser round trip; what is worth pinning here is the
+// decision it feeds — that a sustained onset fires once, a click does not, and
+// natural dips are tolerated — because that is what makes barge-in usable
+// rather than trigger-happy.
+
+import { describe, expect, it } from "vitest";
+
+import {
+  ONSET_DEFAULTS,
+  OnsetDetector,
+  frameRms,
+  readOnsetConfig,
+  type OnsetConfig,
+} from "../voiceVad";
+
+const cfg: OnsetConfig = { vad_threshold: 0.1, vad_onset_ms: 500 };
+
+/** Push `n` frames of a constant level, 50 ms each. */
+function push(det: OnsetDetector, level: number, n: number, dt = 50): void {
+  for (let i = 0; i < n; i += 1) det.push(level, dt);
+}
+
+describe("frameRms", () => {
+  it("is zero for silence and the amplitude for a constant frame", () => {
+    expect(frameRms(new Float32Array([0, 0, 0]))).toBe(0);
+    expect(frameRms(new Float32Array([0.5, 0.5, 0.5]))).toBeCloseTo(0.5, 6);
+    expect(frameRms(new Float32Array(0))).toBe(0);
+  });
+});
+
+describe("OnsetDetector", () => {
+  it("fires once after a sustained loud stretch", () => {
+    let onsets = 0;
+    const det = new OnsetDetector(cfg, () => (onsets += 1));
+    push(det, 0.3, 9); // 450 ms — not yet
+    expect(onsets).toBe(0);
+    push(det, 0.3, 1); // crosses 500 ms
+    expect(onsets).toBe(1);
+    push(det, 0.3, 20); // stays loud — but it only fires once
+    expect(onsets).toBe(1);
+  });
+
+  it("does not fire on a quiet signal or a brief click", () => {
+    let onsets = 0;
+    const det = new OnsetDetector(cfg, () => (onsets += 1));
+    push(det, 0.02, 40); // well below threshold for 2 s
+    push(det, 0.3, 2); // a 100 ms click
+    push(det, 0.02, 40);
+    expect(onsets).toBe(0);
+  });
+
+  it("tolerates the natural dips in speech but resets on real silence", () => {
+    let onsets = 0;
+    const det = new OnsetDetector(cfg, () => (onsets += 1));
+    // Loud with a one-frame dip every third frame: the accumulator still climbs.
+    for (let i = 0; i < 30 && onsets === 0; i += 1) {
+      det.push(i % 3 === 2 ? 0.02 : 0.3, 50);
+    }
+    expect(onsets).toBe(1);
+
+    // A fresh detector that goes quiet before reaching the threshold never fires.
+    const det2 = new OnsetDetector(cfg, () => (onsets += 1));
+    push(det2, 0.3, 8); // 400 ms up
+    push(det2, 0.02, 8); // 400 ms down — back to zero
+    push(det2, 0.3, 8); // 400 ms up again, still short of 500
+    expect(onsets).toBe(1); // unchanged
+  });
+
+  it("re-arms after reset", () => {
+    let onsets = 0;
+    const det = new OnsetDetector(cfg, () => (onsets += 1));
+    push(det, 0.3, 10);
+    expect(onsets).toBe(1);
+    det.reset();
+    push(det, 0.3, 10);
+    expect(onsets).toBe(2);
+  });
+});
+
+describe("readOnsetConfig", () => {
+  it("is the defaults when nothing is stored", () => {
+    expect(readOnsetConfig(() => null)).toEqual(ONSET_DEFAULTS);
+  });
+
+  it("takes overrides and ignores malformed values", () => {
+    const store: Record<string, string> = {
+      "vogt.assistant.voice.vad_threshold": "0.08",
+      "vogt.assistant.voice.vad_onset_ms": "nope",
+    };
+    const c = readOnsetConfig((k) => store[k] ?? null);
+    expect(c.vad_threshold).toBe(0.08);
+    expect(c.vad_onset_ms).toBe(ONSET_DEFAULTS.vad_onset_ms);
+  });
+});

web/src/voiceTurn.ts

@@ -290,6 +290,20 @@ export class VoiceConversation {
     this.resumeListening();
   }
 
+  /** Barge-in (v2): the speaker started talking over a playing reply. Only
+   *  honoured when `interrupt_response` is on and a reply is actually playing —
+   *  the host runs an echo-cancelled VAD during `speaking` and calls this on a
+   *  confident onset. Halts the reply and re-opens the mic at once, so the
+   *  interrupting words are captured as the next turn. A no-op otherwise, so a
+   *  false onset outside playback cannot disturb the loop. The reply's own
+   *  `speechFinished` that follows the halt is ignored (we have left `speaking`). */
+  speechDetected(): void {
+    if (!this.cfg.interrupt_response) return;
+    if (this.state !== "speaking") return;
+    this.ports.stopSpeaking();
+    this.resumeListening();
+  }
+
   /** The turn's send failed. The mode stays (the surface shows the failed
    *  bubble + Retry); the loop re-opens the mic for another turn. */
   sendFailed(): void {