diff --git a/docs/guide/voice.md b/docs/guide/voice.md index e579361..419e6b1 100644 --- a/docs/guide/voice.md +++ b/docs/guide/voice.md @@ -57,6 +57,36 @@ an already accepted agent task. Existing transcript history is never read aloud on activation. Status text shows listening, speech detection, transcription, and playback; errors remain visible in the voice screen. +## Dictation + +Voice mode is a conversation. When you only want to get words into the message +box, use **dictation**: the **microphone icon** left of the waveform icon. Speak, +and what you say is transcribed and typed in; the agent never speaks back and a +run in progress is not interrupted. + +It listens the same way voice mode does, with the same in-browser speech +detection and the same recognition service, so it works as soon as Voice is +enabled. Speech synthesis is not used and nothing is loaded onto the GPU for it. +While you talk, a line above the box shows whether you are being heard and the +words recognised so far. Pausing for about a second ends a phrase, and the next +phrase follows it. + +Choose where the words go with the switch on that line. The choice is remembered. + +| | What happens | +| --- | --- | +| **Edit first** (default) | Each phrase is typed into the box **at the cursor**, spaced like a word, and the cursor moves to the end of it. Click into the text to dictate in the middle of a sentence, correct a word by keyboard, then send with Enter as usual. Dictation keeps listening after you send. | +| **Send at once** | A message is sent once you have stopped talking and it has been transcribed. If you start speaking again while a phrase is still being transcribed, the two go out together. A draft already in the box is left alone. If sending fails, the words are put back in the box rather than lost. | + +Stopping dictation still delivers a sentence you were in the middle of. Starting +voice mode turns dictation off, since both use the microphone, and leaving the +session drops anything not yet transcribed instead of sending it elsewhere. +Whisper's placeholders for silence, such as `[BLANK_AUDIO]`, are not typed. + +Messages sent by dictation are ordinary text messages. Unlike voice-mode turns +they are not marked as audio and the agent is not asked to reply in speakable +style. + ## Alternative: manually managed Python services ::: details Show alternative deployment details diff --git a/package.json b/package.json index 3db5ef1..57684f0 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ ], "scripts": { "build": "npm run build -w server && npm run build -w web", - "test": "npm run build -w server && npm test -w server && node --import tsx --test tests/completed-transcript.test.mts", + "test": "npm run build -w server && npm test -w server && node --import tsx --test tests/*.test.mts", "dev:server": "npm run dev -w server", "dev:web": "npm run dev -w web", "start": "node server/dist/index.js", diff --git a/tests/dictation.test.mts b/tests/dictation.test.mts new file mode 100644 index 0000000..62ab250 --- /dev/null +++ b/tests/dictation.test.mts @@ -0,0 +1,45 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { cleanTranscript, insertAtCaret } from "../web/src/dictation.ts"; + +test("silence notes from Whisper are not text", () => { + for (const note of ["[BLANK_AUDIO]", " (silence) ", "[Music]", "[BLANK_AUDIO] [BLANK_AUDIO]", "(laughs) [applause]"]) { + assert.equal(cleanTranscript(note), "", note); + } +}); + +test("words are kept, brackets inside a sentence included", () => { + assert.equal(cleanTranscript(" hello there "), "hello there"); + assert.equal(cleanTranscript("see (a) below"), "see (a) below"); + assert.equal(cleanTranscript("[BLANK_AUDIO] and then"), "[BLANK_AUDIO] and then"); +}); + +test("dictating into an empty box adds no spaces", () => { + assert.deepEqual(insertAtCaret("", 0, 0, "hello"), { value: "hello", caret: 5 }); +}); + +test("a phrase after a word gets a space, and the next one continues after it", () => { + const first = insertAtCaret("write a test", 12, 12, "for the parser"); + assert.equal(first.value, "write a test for the parser"); + const second = insertAtCaret(first.value, first.caret, first.caret, "please"); + assert.equal(second.value, "write a test for the parser please"); +}); + +test("no space is added after a newline or before one", () => { + assert.equal(insertAtCaret("line one\n", 9, 9, "line two").value, "line one\nline two"); + assert.equal(insertAtCaret("a\nb", 1, 1, "x").value, "a x\nb"); +}); + +test("in the middle of a sentence the caret follows the words and the existing space stays after it", () => { + const out = insertAtCaret("fix the bug", 7, 7, "annoying"); + assert.equal(out.value, "fix the annoying bug"); + assert.equal(out.caret, "fix the annoying".length); +}); + +test("a selection is replaced", () => { + assert.equal(insertAtCaret("fix the old bug", 8, 11, "new").value, "fix the new bug"); +}); + +test("a caret outside the text is pulled back to it", () => { + assert.deepEqual(insertAtCaret("ab", 40, 50, "c"), { value: "ab c", caret: 4 }); +}); diff --git a/web/src/components/Chat.tsx b/web/src/components/Chat.tsx index e6374d4..173d479 100644 --- a/web/src/components/Chat.tsx +++ b/web/src/components/Chat.tsx @@ -4,7 +4,10 @@ import { CanvasPanel } from "./CanvasPanel"; import { displaySpeechText } from "../voice"; import { latestBrowserActivity, latestTerminalActivity } from "../voice-browser"; import { VoiceControl } from "./VoiceControl"; -import { useEffect, useMemo, useRef, useState } from "react"; +import { DictationButton, DictationStrip } from "./Dictation"; +import { insertAtCaret } from "../dictation"; +import { useDictation } from "../use-dictation"; +import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import { Streamdown, type DiagramPlugin } from "streamdown"; import { LuGlobe, LuSquareTerminal, LuSquare, LuFileText, LuArrowUp, LuAudioLines } from "react-icons/lu"; import { api, type PiCommand, type PortalEvent, type Session } from "../api"; @@ -95,6 +98,13 @@ export function Chat({ onClientCommand: (name: string, args: string) => void | Promise; }) { const [input, setInput] = useState(""); + // Where dictated words go. Kept beside the state because several phrases can + // arrive before React has drawn the first, and each must land after the last. + const box = useRef(null); + const draft = useRef(input); + draft.current = input; + const caret = useRef<{ start: number; end: number } | null>(null); + const caretTo = useRef(null); const [voiceMode, setVoiceMode] = useState(false); const [canvasOpen, setCanvasOpen] = useState(false); const [voiceHost, setVoiceHost] = useState(null); @@ -235,10 +245,8 @@ export function Chat({ settled.current = true; }, [items.length, events.length]); - const send = async () => { - const msg = input.trim(); - if (!msg || sending) return; - + /** Send `msg` as a message, or run it if it is one of the portal's own commands. */ + const submit = async (msg: string, fromBox: boolean) => { // Some builtins are UI, not prompts: /model opens the picker the pill uses, // /settings opens the modal. Sending them to pi would just be a chat line. const parsed = /^\/([\w-]+)\s*(.*)$/.exec(msg); @@ -246,14 +254,14 @@ export function Chat({ ? commands.find((c) => c.name === parsed[1] && c.where === "client") : undefined; if (client && parsed) { - setInput(""); + if (fromBox) clearBox(); if (client.name === "model") setPanelRequest("model"); else await onClientCommand(client.name, parsed[2]); return; } setSending(true); - setInput(""); + if (fromBox) clearBox(); try { await onSend(msg, voiceMode ? { voice: true } : undefined); } finally { @@ -261,6 +269,55 @@ export function Chat({ } }; + const send = async () => { + const msg = input.trim(); + if (!msg || sending) return; + await submit(msg, true); + }; + + const clearBox = () => { + caret.current = null; + setInput(""); + }; + + /** Dictated words, put in the box where the cursor was and the cursor left after them. */ + const insertSpoken = (text: string) => { + const at = caret.current ?? { start: draft.current.length, end: draft.current.length }; + const next = insertAtCaret(draft.current, at.start, at.end, text); + draft.current = next.value; + caret.current = { start: next.caret, end: next.caret }; + caretTo.current = next.caret; + setInput(next.value); + }; + useLayoutEffect(() => { + if (caretTo.current === null) return; + box.current?.setSelectionRange(caretTo.current, caretTo.current); + caretTo.current = null; + }, [input]); + + // Phrases sent as they are said go one at a time: a second must not overtake + // the first, and one that fails goes back in the box rather than being lost. + const spoken = useRef>(Promise.resolve()); + const sendSpoken = (text: string) => { + spoken.current = spoken.current.then(async () => { + try { + await submit(text, false); + } catch { + insertSpoken(text); + } + }); + }; + const dictation = useDictation({ + sessionId: session.id, + disabled: voiceMode, + onText: insertSpoken, + onSend: sendSpoken, + }); + // One microphone: voice mode takes over from dictation. + useEffect(() => { + if (voiceMode) void dictation.stop(); + }, [voiceMode, dictation.stop]); + return (
@@ -451,6 +508,7 @@ export function Chat({ type="button" onMouseDown={(e) => { e.preventDefault(); + caret.current = null; setInput(`/${c.name} `); }} className="flex w-full items-baseline gap-2 px-3 py-2 text-left transition hover:bg-fg/5" @@ -462,9 +520,17 @@ export function Chat({ ))}
)} +