From f1968258a0de7809b16adabec9d098a618c70778 Mon Sep 17 00:00:00 2001 From: Piggidragon Date: Sat, 19 Sep 2026 22:20:07 +0200 Subject: [PATCH] Add dictation: speak into the message box without a spoken reply A microphone button next to the voice-mode button transcribes what is said and either types it into the message box at the cursor (edit first) or sends it when the speaker pauses. It reuses voice mode's in-browser speech detection and transcription route, so it needs no speech synthesis and loads nothing onto the GPU. The voice-mode button becomes the waveform icon so the two are told apart. Co-Authored-By: Claude Sonnet 5 --- docs/guide/voice.md | 30 +++ package.json | 2 +- tests/dictation.test.mts | 45 ++++ web/src/components/Chat.tsx | 91 +++++++- web/src/components/Dictation.tsx | 87 +++++++ web/src/components/VoiceControl.tsx | 4 +- web/src/dictation.ts | 35 +++ web/src/index.css | 1 + web/src/use-dictation.ts | 349 ++++++++++++++++++++++++++++ 9 files changed, 632 insertions(+), 12 deletions(-) create mode 100644 tests/dictation.test.mts create mode 100644 web/src/components/Dictation.tsx create mode 100644 web/src/dictation.ts create mode 100644 web/src/use-dictation.ts diff --git a/docs/guide/voice.md b/docs/guide/voice.md index e579361..419e6b1 100644 --- a/docs/guide/voice.md +++ b/docs/guide/voice.md @@ -57,6 +57,36 @@ an already accepted agent task. Existing transcript history is never read aloud on activation. Status text shows listening, speech detection, transcription, and playback; errors remain visible in the voice screen. +## Dictation + +Voice mode is a conversation. When you only want to get words into the message +box, use **dictation**: the **microphone icon** left of the waveform icon. Speak, +and what you say is transcribed and typed in; the agent never speaks back and a +run in progress is not interrupted. + +It listens the same way voice mode does, with the same in-browser speech +detection and the same recognition service, so it works as soon as Voice is +enabled. Speech synthesis is not used and nothing is loaded onto the GPU for it. +While you talk, a line above the box shows whether you are being heard and the +words recognised so far. Pausing for about a second ends a phrase, and the next +phrase follows it. + +Choose where the words go with the switch on that line. The choice is remembered. + +| | What happens | +| --- | --- | +| **Edit first** (default) | Each phrase is typed into the box **at the cursor**, spaced like a word, and the cursor moves to the end of it. Click into the text to dictate in the middle of a sentence, correct a word by keyboard, then send with Enter as usual. Dictation keeps listening after you send. | +| **Send at once** | A message is sent once you have stopped talking and it has been transcribed. If you start speaking again while a phrase is still being transcribed, the two go out together. A draft already in the box is left alone. If sending fails, the words are put back in the box rather than lost. | + +Stopping dictation still delivers a sentence you were in the middle of. Starting +voice mode turns dictation off, since both use the microphone, and leaving the +session drops anything not yet transcribed instead of sending it elsewhere. +Whisper's placeholders for silence, such as `[BLANK_AUDIO]`, are not typed. + +Messages sent by dictation are ordinary text messages. Unlike voice-mode turns +they are not marked as audio and the agent is not asked to reply in speakable +style. + ## Alternative: manually managed Python services ::: details Show alternative deployment details diff --git a/package.json b/package.json index 3db5ef1..57684f0 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ ], "scripts": { "build": "npm run build -w server && npm run build -w web", - "test": "npm run build -w server && npm test -w server && node --import tsx --test tests/completed-transcript.test.mts", + "test": "npm run build -w server && npm test -w server && node --import tsx --test tests/*.test.mts", "dev:server": "npm run dev -w server", "dev:web": "npm run dev -w web", "start": "node server/dist/index.js", diff --git a/tests/dictation.test.mts b/tests/dictation.test.mts new file mode 100644 index 0000000..62ab250 --- /dev/null +++ b/tests/dictation.test.mts @@ -0,0 +1,45 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { cleanTranscript, insertAtCaret } from "../web/src/dictation.ts"; + +test("silence notes from Whisper are not text", () => { + for (const note of ["[BLANK_AUDIO]", " (silence) ", "[Music]", "[BLANK_AUDIO] [BLANK_AUDIO]", "(laughs) [applause]"]) { + assert.equal(cleanTranscript(note), "", note); + } +}); + +test("words are kept, brackets inside a sentence included", () => { + assert.equal(cleanTranscript(" hello there "), "hello there"); + assert.equal(cleanTranscript("see (a) below"), "see (a) below"); + assert.equal(cleanTranscript("[BLANK_AUDIO] and then"), "[BLANK_AUDIO] and then"); +}); + +test("dictating into an empty box adds no spaces", () => { + assert.deepEqual(insertAtCaret("", 0, 0, "hello"), { value: "hello", caret: 5 }); +}); + +test("a phrase after a word gets a space, and the next one continues after it", () => { + const first = insertAtCaret("write a test", 12, 12, "for the parser"); + assert.equal(first.value, "write a test for the parser"); + const second = insertAtCaret(first.value, first.caret, first.caret, "please"); + assert.equal(second.value, "write a test for the parser please"); +}); + +test("no space is added after a newline or before one", () => { + assert.equal(insertAtCaret("line one\n", 9, 9, "line two").value, "line one\nline two"); + assert.equal(insertAtCaret("a\nb", 1, 1, "x").value, "a x\nb"); +}); + +test("in the middle of a sentence the caret follows the words and the existing space stays after it", () => { + const out = insertAtCaret("fix the bug", 7, 7, "annoying"); + assert.equal(out.value, "fix the annoying bug"); + assert.equal(out.caret, "fix the annoying".length); +}); + +test("a selection is replaced", () => { + assert.equal(insertAtCaret("fix the old bug", 8, 11, "new").value, "fix the new bug"); +}); + +test("a caret outside the text is pulled back to it", () => { + assert.deepEqual(insertAtCaret("ab", 40, 50, "c"), { value: "ab c", caret: 4 }); +}); diff --git a/web/src/components/Chat.tsx b/web/src/components/Chat.tsx index e6374d4..173d479 100644 --- a/web/src/components/Chat.tsx +++ b/web/src/components/Chat.tsx @@ -4,7 +4,10 @@ import { CanvasPanel } from "./CanvasPanel"; import { displaySpeechText } from "../voice"; import { latestBrowserActivity, latestTerminalActivity } from "../voice-browser"; import { VoiceControl } from "./VoiceControl"; -import { useEffect, useMemo, useRef, useState } from "react"; +import { DictationButton, DictationStrip } from "./Dictation"; +import { insertAtCaret } from "../dictation"; +import { useDictation } from "../use-dictation"; +import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import { Streamdown, type DiagramPlugin } from "streamdown"; import { LuGlobe, LuSquareTerminal, LuSquare, LuFileText, LuArrowUp, LuAudioLines } from "react-icons/lu"; import { api, type PiCommand, type PortalEvent, type Session } from "../api"; @@ -95,6 +98,13 @@ export function Chat({ onClientCommand: (name: string, args: string) => void | Promise; }) { const [input, setInput] = useState(""); + // Where dictated words go. Kept beside the state because several phrases can + // arrive before React has drawn the first, and each must land after the last. + const box = useRef(null); + const draft = useRef(input); + draft.current = input; + const caret = useRef<{ start: number; end: number } | null>(null); + const caretTo = useRef(null); const [voiceMode, setVoiceMode] = useState(false); const [canvasOpen, setCanvasOpen] = useState(false); const [voiceHost, setVoiceHost] = useState(null); @@ -235,10 +245,8 @@ export function Chat({ settled.current = true; }, [items.length, events.length]); - const send = async () => { - const msg = input.trim(); - if (!msg || sending) return; - + /** Send `msg` as a message, or run it if it is one of the portal's own commands. */ + const submit = async (msg: string, fromBox: boolean) => { // Some builtins are UI, not prompts: /model opens the picker the pill uses, // /settings opens the modal. Sending them to pi would just be a chat line. const parsed = /^\/([\w-]+)\s*(.*)$/.exec(msg); @@ -246,14 +254,14 @@ export function Chat({ ? commands.find((c) => c.name === parsed[1] && c.where === "client") : undefined; if (client && parsed) { - setInput(""); + if (fromBox) clearBox(); if (client.name === "model") setPanelRequest("model"); else await onClientCommand(client.name, parsed[2]); return; } setSending(true); - setInput(""); + if (fromBox) clearBox(); try { await onSend(msg, voiceMode ? { voice: true } : undefined); } finally { @@ -261,6 +269,55 @@ export function Chat({ } }; + const send = async () => { + const msg = input.trim(); + if (!msg || sending) return; + await submit(msg, true); + }; + + const clearBox = () => { + caret.current = null; + setInput(""); + }; + + /** Dictated words, put in the box where the cursor was and the cursor left after them. */ + const insertSpoken = (text: string) => { + const at = caret.current ?? { start: draft.current.length, end: draft.current.length }; + const next = insertAtCaret(draft.current, at.start, at.end, text); + draft.current = next.value; + caret.current = { start: next.caret, end: next.caret }; + caretTo.current = next.caret; + setInput(next.value); + }; + useLayoutEffect(() => { + if (caretTo.current === null) return; + box.current?.setSelectionRange(caretTo.current, caretTo.current); + caretTo.current = null; + }, [input]); + + // Phrases sent as they are said go one at a time: a second must not overtake + // the first, and one that fails goes back in the box rather than being lost. + const spoken = useRef>(Promise.resolve()); + const sendSpoken = (text: string) => { + spoken.current = spoken.current.then(async () => { + try { + await submit(text, false); + } catch { + insertSpoken(text); + } + }); + }; + const dictation = useDictation({ + sessionId: session.id, + disabled: voiceMode, + onText: insertSpoken, + onSend: sendSpoken, + }); + // One microphone: voice mode takes over from dictation. + useEffect(() => { + if (voiceMode) void dictation.stop(); + }, [voiceMode, dictation.stop]); + return (
@@ -451,6 +508,7 @@ export function Chat({ type="button" onMouseDown={(e) => { e.preventDefault(); + caret.current = null; setInput(`/${c.name} `); }} className="flex w-full items-baseline gap-2 px-3 py-2 text-left transition hover:bg-fg/5" @@ -462,9 +520,17 @@ export function Chat({ ))}
)} +