Add dictation: speak into the message box without a spoken reply

docs/guide/voice.md

@@ -57,6 +57,36 @@ an already accepted agent task. Existing transcript history is never read aloud
 on activation. Status text shows listening, speech detection, transcription,
 and playback; errors remain visible in the voice screen.
 
+## Dictation
+
+Voice mode is a conversation. When you only want to get words into the message
+box, use **dictation**: the **microphone icon** left of the waveform icon. Speak,
+and what you say is transcribed and typed in; the agent never speaks back and a
+run in progress is not interrupted.
+
+It listens the same way voice mode does, with the same in-browser speech
+detection and the same recognition service, so it works as soon as Voice is
+enabled. Speech synthesis is not used and nothing is loaded onto the GPU for it.
+While you talk, a line above the box shows whether you are being heard and the
+words recognised so far. Pausing for about a second ends a phrase, and the next
+phrase follows it.
+
+Choose where the words go with the switch on that line. The choice is remembered.
+
+| | What happens |
+| --- | --- |
+| **Edit first** (default) | Each phrase is typed into the box **at the cursor**, spaced like a word, and the cursor moves to the end of it. Click into the text to dictate in the middle of a sentence, correct a word by keyboard, then send with Enter as usual. Dictation keeps listening after you send. |
+| **Send at once** | A message is sent once you have stopped talking and it has been transcribed. If you start speaking again while a phrase is still being transcribed, the two go out together. A draft already in the box is left alone. If sending fails, the words are put back in the box rather than lost. |
+
+Stopping dictation still delivers a sentence you were in the middle of. Starting
+voice mode turns dictation off, since both use the microphone, and leaving the
+session drops anything not yet transcribed instead of sending it elsewhere.
+Whisper's placeholders for silence, such as `[BLANK_AUDIO]`, are not typed.
+
+Messages sent by dictation are ordinary text messages. Unlike voice-mode turns
+they are not marked as audio and the agent is not asked to reply in speakable
+style.
+
 ## Alternative: manually managed Python services
 
 ::: details Show alternative deployment details

package.json

@@ -9,7 +9,7 @@
   ],
   "scripts": {
     "build": "npm run build -w server && npm run build -w web",
-    "test": "npm run build -w server && npm test -w server && node --import tsx --test tests/completed-transcript.test.mts",
+    "test": "npm run build -w server && npm test -w server && node --import tsx --test tests/*.test.mts",
     "dev:server": "npm run dev -w server",
     "dev:web": "npm run dev -w web",
     "start": "node server/dist/index.js",

tests/dictation.test.mts

@@ -0,0 +1,45 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { cleanTranscript, insertAtCaret } from "../web/src/dictation.ts";
+
+test("silence notes from Whisper are not text", () => {
+  for (const note of ["[BLANK_AUDIO]", " (silence) ", "[Music]", "[BLANK_AUDIO] [BLANK_AUDIO]", "(laughs) [applause]"]) {
+    assert.equal(cleanTranscript(note), "", note);
+  }
+});
+
+test("words are kept, brackets inside a sentence included", () => {
+  assert.equal(cleanTranscript("  hello there "), "hello there");
+  assert.equal(cleanTranscript("see (a) below"), "see (a) below");
+  assert.equal(cleanTranscript("[BLANK_AUDIO] and then"), "[BLANK_AUDIO] and then");
+});
+
+test("dictating into an empty box adds no spaces", () => {
+  assert.deepEqual(insertAtCaret("", 0, 0, "hello"), { value: "hello", caret: 5 });
+});
+
+test("a phrase after a word gets a space, and the next one continues after it", () => {
+  const first = insertAtCaret("write a test", 12, 12, "for the parser");
+  assert.equal(first.value, "write a test for the parser");
+  const second = insertAtCaret(first.value, first.caret, first.caret, "please");
+  assert.equal(second.value, "write a test for the parser please");
+});
+
+test("no space is added after a newline or before one", () => {
+  assert.equal(insertAtCaret("line one\n", 9, 9, "line two").value, "line one\nline two");
+  assert.equal(insertAtCaret("a\nb", 1, 1, "x").value, "a x\nb");
+});
+
+test("in the middle of a sentence the caret follows the words and the existing space stays after it", () => {
+  const out = insertAtCaret("fix the bug", 7, 7, "annoying");
+  assert.equal(out.value, "fix the annoying bug");
+  assert.equal(out.caret, "fix the annoying".length);
+});
+
+test("a selection is replaced", () => {
+  assert.equal(insertAtCaret("fix the old bug", 8, 11, "new").value, "fix the new bug");
+});
+
+test("a caret outside the text is pulled back to it", () => {
+  assert.deepEqual(insertAtCaret("ab", 40, 50, "c"), { value: "ab c", caret: 4 });
+});

web/src/components/Chat.tsx

@@ -4,7 +4,10 @@ import { CanvasPanel } from "./CanvasPanel";
 import { displaySpeechText } from "../voice";
 import { latestBrowserActivity, latestTerminalActivity } from "../voice-browser";
 import { VoiceControl } from "./VoiceControl";
-import { useEffect, useMemo, useRef, useState } from "react";
+import { DictationButton, DictationStrip } from "./Dictation";
+import { insertAtCaret } from "../dictation";
+import { useDictation } from "../use-dictation";
+import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
 import { Streamdown, type DiagramPlugin } from "streamdown";
 import { LuGlobe, LuSquareTerminal, LuSquare, LuFileText, LuArrowUp, LuAudioLines } from "react-icons/lu";
 import { api, type PiCommand, type PortalEvent, type Session } from "../api";
@@ -95,6 +98,13 @@ export function Chat({
   onClientCommand: (name: string, args: string) => void | Promise<void>;
 }) {
   const [input, setInput] = useState("");
+  // Where dictated words go. Kept beside the state because several phrases can
+  // arrive before React has drawn the first, and each must land after the last.
+  const box = useRef<HTMLTextAreaElement>(null);
+  const draft = useRef(input);
+  draft.current = input;
+  const caret = useRef<{ start: number; end: number } | null>(null);
+  const caretTo = useRef<number | null>(null);
   const [voiceMode, setVoiceMode] = useState(false);
   const [canvasOpen, setCanvasOpen] = useState(false);
   const [voiceHost, setVoiceHost] = useState<HTMLDivElement | null>(null);
@@ -235,32 +245,79 @@ export function Chat({
     settled.current = true;
   }, [items.length, events.length]);
 
-  const send = async () => {
-    const msg = input.trim();
-    if (!msg || sending) return;
-
+  /** Send `msg` as a message, or run it if it is one of the portal's own commands. */
+  const submit = async (msg: string, fromBox: boolean) => {
     // Some builtins are UI, not prompts: /model opens the picker the pill uses,
     // /settings opens the modal. Sending them to pi would just be a chat line.
     const parsed = /^\/([\w-]+)\s*(.*)$/.exec(msg);
     const client = parsed
       ? commands.find((c) => c.name === parsed[1] && c.where === "client")
       : undefined;
     if (client && parsed) {
-      setInput("");
+      if (fromBox) clearBox();
       if (client.name === "model") setPanelRequest("model");
       else await onClientCommand(client.name, parsed[2]);
       return;
     }
 
     setSending(true);
-    setInput("");
+    if (fromBox) clearBox();
     try {
       await onSend(msg, voiceMode ? { voice: true } : undefined);
     } finally {
       setSending(false);
     }
   };
 
+  const send = async () => {
+    const msg = input.trim();
+    if (!msg || sending) return;
+    await submit(msg, true);
+  };
+
+  const clearBox = () => {
+    caret.current = null;
+    setInput("");
+  };
+
+  /** Dictated words, put in the box where the cursor was and the cursor left after them. */
+  const insertSpoken = (text: string) => {
+    const at = caret.current ?? { start: draft.current.length, end: draft.current.length };
+    const next = insertAtCaret(draft.current, at.start, at.end, text);
+    draft.current = next.value;
+    caret.current = { start: next.caret, end: next.caret };
+    caretTo.current = next.caret;
+    setInput(next.value);
+  };
+  useLayoutEffect(() => {
+    if (caretTo.current === null) return;
+    box.current?.setSelectionRange(caretTo.current, caretTo.current);
+    caretTo.current = null;
+  }, [input]);
+
+  // Phrases sent as they are said go one at a time: a second must not overtake
+  // the first, and one that fails goes back in the box rather than being lost.
+  const spoken = useRef<Promise<unknown>>(Promise.resolve());
+  const sendSpoken = (text: string) => {
+    spoken.current = spoken.current.then(async () => {
+      try {
+        await submit(text, false);
+      } catch {
+        insertSpoken(text);
+      }
+    });
+  };
+  const dictation = useDictation({
+    sessionId: session.id,
+    disabled: voiceMode,
+    onText: insertSpoken,
+    onSend: sendSpoken,
+  });
+  // One microphone: voice mode takes over from dictation.
+  useEffect(() => {
+    if (voiceMode) void dictation.stop();
+  }, [voiceMode, dictation.stop]);
+
   return (
     <div className="session-workspace relative flex h-full min-h-0 flex-col">
       <CanvasPanel showToggle={false} key={session.id} sessionId={session.id} open={canvasOpen} setOpen={setCanvasOpen}/>
@@ -451,6 +508,7 @@ export function Chat({
                 type="button"
                 onMouseDown={(e) => {
                   e.preventDefault();
+                  caret.current = null;
                   setInput(`/${c.name} `);
                 }}
                 className="flex w-full items-baseline gap-2 px-3 py-2 text-left transition hover:bg-fg/5"
@@ -462,17 +520,31 @@ export function Chat({
             ))}
           </div>
         )}
+        <DictationStrip dictation={dictation} />
         <textarea
+          ref={box}
           value={input}
-          onChange={(e) => setInput(e.target.value)}
+          onChange={(e) => {
+            caret.current = { start: e.target.selectionStart, end: e.target.selectionEnd };
+            setInput(e.target.value);
+          }}
+          onSelect={(e) => {
+            caret.current = { start: e.currentTarget.selectionStart, end: e.currentTarget.selectionEnd };
+          }}
           onKeyDown={(e) => {
             if (e.key === "Enter" && !e.shiftKey) {
               e.preventDefault();
               send();
             }
           }}
           rows={2}
-          placeholder={running ? "pi is working — send to queue a follow-up…" : "Describe the task…"}
+          placeholder={
+            dictation.active
+              ? "Speak — your words appear here…"
+              : running
+                ? "pi is working — send to queue a follow-up…"
+                : "Describe the task…"
+          }
           aria-label="Message"
           className="prompt-input"
         />
@@ -483,6 +555,7 @@ export function Chat({
             panelRequest={panelRequest}
             onPanelConsumed={() => setPanelRequest(null)}
             actions={<>
+              <DictationButton dictation={dictation} />
               <VoiceControl canvasOpen={canvasOpen} onCanvasMinimize={()=>setCanvasOpen(false)} onCanvasToggle={()=>setCanvasOpen(value=>!value)} key={session.id} sessionId={session.id} items={items} running={running} onSend={onSend} onAbort={onAbort} stageTarget={voiceHost} onModeChange={setVoiceMode} title={session.title} browserAvailable={browserUp} browserActivity={latestBrowserActivity(events)} terminalActivity={latestTerminalActivity(events)} toolEvents={events} />
               {running && !input.trim() ? <button type="button" aria-label="Stop generation" title="Stop generation" onClick={onAbort} className="prompt-action prompt-stop">
                 <LuSquare aria-hidden className="h-4 w-4" fill="currentColor" />

web/src/components/Dictation.tsx

@@ -0,0 +1,87 @@
+import { LuLoaderCircle, LuMic } from "react-icons/lu";
+import type { Dictation, DictationMode } from "../use-dictation";
+
+/** Turns dictation on and off. Hidden until voice is set up, since that is what transcribes. */
+export function DictationButton({ dictation }: { dictation: Dictation }) {
+  if (!dictation.available) return null;
+  const on = dictation.active || dictation.starting;
+  return (
+    <button
+      type="button"
+      onClick={dictation.toggle}
+      aria-pressed={on}
+      aria-label={on ? "Stop dictating" : "Dictate a message"}
+      title={
+        on
+          ? "Stop dictating"
+          : dictation.mode === "send"
+            ? "Dictate — what you say is sent when you pause"
+            : "Dictate — what you say is typed into the box"
+      }
+      className="prompt-action"
+    >
+      {dictation.starting ? <LuLoaderCircle aria-hidden className="animate-spin" /> : <LuMic aria-hidden />}
+    </button>
+  );
+}
+
+const MODES: { id: DictationMode; label: string; hint: string }[] = [
+  { id: "review", label: "Edit first", hint: "Words are typed into the box, so you can change them before sending" },
+  { id: "send", label: "Send at once", hint: "A message is sent as soon as you pause" },
+];
+
+/**
+ * What dictation is doing, above the message box: whether it hears you, what it
+ * has made of it so far, and where the words will go.
+ */
+export function DictationStrip({ dictation }: { dictation: Dictation }) {
+  if (!dictation.showing) return null;
+  const hearing = dictation.phase === "Hearing you";
+  const working = dictation.active || dictation.starting || dictation.phase === "Transcribing" || dictation.held !== "";
+  if (!working) {
+    // Only a failure is left, e.g. a microphone that was refused.
+    return (
+      <p role="alert" className="mb-1 px-2 pt-0.5 text-xs text-danger">
+        {dictation.error}
+      </p>
+    );
+  }
+  const words = [dictation.held, dictation.partial].filter(Boolean).join(" ");
+  return (
+    <div className="mb-1 flex flex-wrap items-center gap-x-3 gap-y-1 px-2 pt-0.5 text-xs">
+      <span role="status" className="flex shrink-0 items-center gap-1.5 text-fg-muted">
+        <span
+          aria-hidden
+          className={`h-2 w-2 rounded-full ${
+            hearing ? "animate-pulse bg-accent" : dictation.phase === "Transcribing" ? "bg-fg-subtle" : "bg-fg-faint"
+          }`}
+        />
+        {dictation.starting ? "Starting the microphone…" : dictation.active ? dictation.phase : "Finishing…"}
+      </span>
+      <span className="min-w-0 flex-1 basis-40 truncate italic text-fg-subtle" aria-live="off">
+        {words}
+      </span>
+      <div role="group" aria-label="Where dictated words go" className="flex shrink-0 rounded-lg bg-fg/5 p-0.5">
+        {MODES.map((m) => (
+          <button
+            key={m.id}
+            type="button"
+            title={m.hint}
+            aria-pressed={dictation.mode === m.id}
+            onClick={() => dictation.setMode(m.id)}
+            className={`rounded-md px-2 py-0.5 transition ${
+              dictation.mode === m.id ? "bg-surface text-fg shadow-sm" : "text-fg-subtle hover:text-fg"
+            }`}
+          >
+            {m.label}
+          </button>
+        ))}
+      </div>
+      {dictation.error && (
+        <p role="alert" className="basis-full text-danger">
+          {dictation.error}
+        </p>
+      )}
+    </div>
+  );
+}

web/src/components/VoiceControl.tsx

@@ -6,7 +6,7 @@ import { useCallback, useEffect, useRef, useState } from "react";
 import { voiceCue, type VoiceCue } from "../voice-cues";
 import { createPortal } from "react-dom";
 import { VoiceStage, type VoiceLevels } from "./VoiceStage";
-import { LuMic, LuLoaderCircle, LuGauge } from "react-icons/lu";
+import { LuAudioLines, LuLoaderCircle, LuGauge } from "react-icons/lu";
 import type { MicVAD } from "@ricky0123/vad-web";
 import { api, type PortalEvent } from "../api";
 import type { Item } from "../transcript";
@@ -362,7 +362,7 @@ export function VoiceControl({ canvasOpen, onCanvasMinimize, onCanvasToggle, ses
       <button type="button" className="prompt-action" aria-label="Profile voice latency" title="Profile voice latency" aria-pressed={profileOpen} onClick={()=>{setProfileOpen(v=>!v);if(profileOpen)profiler.current!.close('disabled');}}><LuGauge/></button>
       {error && !enabled && !starting && <p role="alert" className="absolute bottom-full right-0 mb-3 w-64 rounded-xl border border-line bg-surface p-3 text-xs text-danger shadow-pop">{error}</p>}
       <button ref={startButton} type="button" onClick={start} aria-label="Turn on hands-free voice" title="Start voice conversation" className="prompt-action">
-        {starting ? <LuLoaderCircle aria-hidden className="animate-spin" /> : <LuMic aria-hidden />}
+        {starting ? <LuLoaderCircle aria-hidden className="animate-spin" /> : <LuAudioLines aria-hidden />}
       </button>
     </div>
   </>;

web/src/dictation.ts

@@ -0,0 +1,35 @@
+/**
+ * Whisper answers a stretch of silence or noise with a note in brackets —
+ * "[BLANK_AUDIO]", "(silence)", "[Music]" — instead of nothing. Dictated into
+ * a message box that is junk, so a transcript made only of such notes is empty.
+ * Brackets inside a sentence are left alone: "see (a) below" is what was said.
+ */
+export function cleanTranscript(text: string): string {
+  const trimmed = text.trim();
+  return /^(?:\s*[[(][^\])]*[\])])+\s*$/.test(trimmed) ? "" : trimmed;
+}
+
+/**
+ * `text` put into `value` where the caret is, spaced like a word.
+ *
+ * Spoken text carries no leading or trailing space, so pasting it in raw would
+ * glue it to the neighbouring words. A space is added only where one is missing
+ * and only between words, never at the start or end of the box or after a
+ * newline. The returned caret sits just after what was inserted, so the next
+ * phrase continues from there.
+ */
+export function insertAtCaret(
+  value: string,
+  start: number,
+  end: number,
+  text: string,
+): { value: string; caret: number } {
+  const from = Math.max(0, Math.min(start, value.length));
+  const to = Math.max(from, Math.min(end, value.length));
+  const before = value.slice(0, from);
+  const after = value.slice(to);
+  const lead = before && !/\s$/.test(before) ? " " : "";
+  const trail = after && !/^\s/.test(after) ? " " : "";
+  const inserted = lead + text + trail;
+  return { value: before + inserted + after, caret: before.length + inserted.length };
+}