diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e274081a0..c0b68cf2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,6 +137,9 @@ jobs: exit 1 fi + - name: Run speech regression and E2E suite + run: bash tools/speech-regress.sh + - name: Run namespace audit checks run: | echo "Running nsaudit security checks..." diff --git a/QUICKSTART.md b/QUICKSTART.md index 216847837..33f1a8b4e 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -106,6 +106,47 @@ After building (see Building section below): **Note:** The runtime `.dis` files in `dis/` are tracked in git, so basic commands work after clone. If you see `link typecheck` errors, run `./hooks/install.sh` and pull again, or rebuild manually with `mk install` in the affected `appl/` subdirectory. +## Voice Mode + +Once Veltro's LLM API key is set up (keyring → factotum inside InferNode), +talking to Lucia takes one host-side install: + +```sh +tools/install-speech-helpers.sh # from the repo root, on the host +``` + +then restart InferNode. The installer downloads/builds everything and writes +its configuration to `~/.local/share/infernode-speech/speech.ctl.sh`, which +`boot.sh` applies automatically — no manual ctl writes. The default stack is +the most lightweight high-quality option at each stage: + +| Stage | Default | Notes | +|-------|---------|-------| +| TTS | **Kokoro** (`af_bella`, kokoro-onnx) | natural voice; the robotic macOS `say` is only a fallback when no helpers are installed | +| STT | **Parakeet** realtime EOU 120M (`tools/parakeet_stream.cpp` adapter, built against [parakeet.cpp](https://github.com/mudler/parakeet.cpp)) | streaming transcription; the model itself detects end-of-utterance. Falls back to whisper.cpp `base.en` when parakeet can't be built | +| Wake | openWakeWord | wake phrase is **"hey jarvis"** (the only pretrained model) | + +### Using it + +- **Enter/exit voice mode:** `Esc` `v`, Option/Alt+V (SDL), the **Voice chip** + in the context panel, the **voice button** on the chat input row, or + Ctrl+Space in the conversation view. All toggle the same thing: + `/mnt/ui/input-mode` between `k` and `v`. +- Say **"hey jarvis"**, speak, and pause. The transcript appears in the + compose box and on the Voice chip, then sends after a **3-second grace + window** — say **"cancel"** to discard it, or keep talking to extend it. + (`voicemode -g 0` restores instant send.) +- Low-confidence transcripts ask for a spoken yes/no first. +- `Esc` exits voice mode at any point and releases the microphone. + +### Verifying without an LLM + +`tools/speech-test.sh` exercises microphone → STT → TTS with no login, no +API key, and no per-turn cost (`-g` for the GUI variant). See +`docs/SPEECH-ARCHITECTURE.md` for the full architecture, remote-audio +topologies, and every ctl knob, and `docs/SPEECH-REMOTE-AUDIO.md` for +running the microphone and the speech engines on different machines. + ## Building ### Linux x86_64 (Intel/AMD) diff --git a/appl/cmd/lucibridge.b b/appl/cmd/lucibridge.b index 5c1884c43..184553ee2 100644 --- a/appl/cmd/lucibridge.b +++ b/appl/cmd/lucibridge.b @@ -47,6 +47,21 @@ autospeak := 0; maxsteps := DEFAULT_MAX_STEPS; stderr: ref Sys->FD; +Speakreq: adt { + gen: int; + text: string; +}; + +speakq: chan of ref Speakreq; +speakdone: chan of int; +speakgen := 0; + +# Persistent input readers and cooperative active-turn control. +inputc: chan of (int, string); +turngen := 0; +turnpaused := 0; +turnactive := 0; + # LLM session state sessionid := ""; llmfd: ref Sys->FD; @@ -126,6 +141,26 @@ toolctlmount(mpt: string): string return "/mnt/toolctl"; } +readsmall(path: string): string +{ + fd := sys->open(path, Sys->OREAD); + if(fd == nil) + return nil; + buf := array[128] of byte; + n := sys->read(fd, buf, len buf); + if(n <= 0) + return nil; + return agentlib->strip(string buf[0:n]); +} + +inputmode(): string +{ + mode := readsmall("/mnt/ui/input-mode"); + if(mode == "v") + return "v"; + return "k"; +} + # Extract value for key from "key1=val1 key2=val2 ..." string getkv(line, key: string): string { @@ -260,23 +295,207 @@ registernamespace() log(sys->sprint("context: registered %d namespace entries", nreg)); } -# Speak text via speech9p (fire-and-forget, runs in spawned goroutine) -speaktext(text: string) +# Append without reversing the existing FIFO order. +appendspeak(l: list of ref Speakreq, r: ref Speakreq): list of ref Speakreq +{ + if(l == nil) + return r :: nil; + rev: list of ref Speakreq; + for(; l != nil; l = tl l) + rev = hd l :: rev; + rev = r :: rev; + result: list of ref Speakreq; + for(; rev != nil; rev = tl rev) + result = hd rev :: result; + return result; +} + +# Speak one queue entry via speech9p. sayq is a per-fid transaction: the +# read after the write blocks until playback finishes, keeping the resource +# truthfully active for the whole utterance. Older providers with only /say +# still work, but cannot report playback completion. +speaktext(r: ref Speakreq) { + if(r.gen != speakgen) { + speakdone <-= r.gen; + return; + } + # Update context zone status ctxpath := sys->sprint("/mnt/ui/activity/%d/context/ctl", actid); - writefile(ctxpath, "resource update path=speech status=active"); + writefile(ctxpath, "resource update path=/n/speech status=active"); - fd := sys->open("/n/speech/say", Sys->OWRITE); + queued := 1; + fd := sys->open("/n/speech/sayq", Sys->ORDWR); if(fd == nil) { - log("speaktext: cannot open /n/speech/say"); - writefile(ctxpath, "resource update path=speech status=idle"); + queued = 0; + fd = sys->open("/n/speech/say", Sys->OWRITE); + } + if(fd == nil) { + log("speaktext: cannot open /n/speech/sayq or /n/speech/say"); + if(r.gen == speakgen) + writefile(ctxpath, "resource update path=/n/speech status=error"); + speakdone <-= r.gen; return; } - b := array of byte text; - sys->write(fd, b, len b); + if(r.gen != speakgen) { + speakdone <-= r.gen; + return; + } + b := array of byte r.text; + if(sys->write(fd, b, len b) < 0) { + log("speaktext: write failed"); + if(r.gen == speakgen) + writefile(ctxpath, "resource update path=/n/speech status=error"); + speakdone <-= r.gen; + return; + } + + if(queued) { + sys->seek(fd, big 0, Sys->SEEKSTART); + status := array[256] of byte; + n := sys->read(fd, status, len status); + if(n < 0) + log("speaktext: sayq completion read failed"); + else if(n > 0) + log("speaktext: " + string status[0:n]); + } + if(r.gen != speakgen) + writefile("/n/speech/cancel", "cancel"); + else + writefile(ctxpath, "resource update path=/n/speech status=idle"); + speakdone <-= r.gen; +} + +# Single queue manager: accepts new speech while playback is in progress, but +# launches at most one blocking sayq transaction at a time. +speechmanager() +{ + pending: list of ref Speakreq; + busy := 0; + for(;;) { + while(!busy && pending != nil) { + r := hd pending; + pending = tl pending; + if(r.gen != speakgen) + continue; + busy = 1; + spawn speaktext(r); + } + alt { + r := <-speakq => + if(r != nil && r.text != "" && r.gen == speakgen) + pending = appendspeak(pending, r); + <-speakdone => + busy = 0; + } + } +} - writefile(ctxpath, "resource update path=speech status=idle"); +queuespeech(text: string) +{ + if(text != "") + speakq <-= ref Speakreq(speakgen, text); +} + +# Queue complete sentence-sized prefixes while text is still streaming. +# The returned byte index is the first unspoken byte; final flushes a short +# tail so response audio starts before the entire model turn is complete. +queuespeechavailable(text: string, start, final: int): int +{ + if(start < 0 || start > len text) + start = 0; + end := -1; + for(i := start; i < len text; i++) + if((text[i] == '.' || text[i] == '!' || text[i] == '?' || text[i] == '\n') && + i - start >= 24) + end = i + 1; + if(final && end < len text) + end = len text; + if(end > start) { + chunk := agentlib->strip(text[start:end]); + if(chunk != "") + queuespeech(chunk); + return end; + } + return start; +} + +cancelspeechqueue() +{ + speakgen++; + writefile("/n/speech/cancel", "cancel"); +} + +# Persistent conversation-input reader. Re-opens the file for every read +# because the 9P offset advances after a read (subsequent reads return EOF). +# Feeds one shared channel so the main loop can block on keyboard input and +# voice input simultaneously — picking a single path before blocking would +# leave the bridge stuck on the stale path across an input-mode switch. +# Sends nil once (open failure or EOF) and exits. +inputreader(path: string, isvoice: int, ch: chan of (int, string)) +{ + for(;;) { + fd := sys->open(path, Sys->OREAD); + if(fd == nil) { + ch <-= (isvoice, nil); + return; + } + s := blockread(fd); + fd = nil; + ch <-= (isvoice, s); + if(s == nil) + return; + } +} + +controlreader(path: string) +{ + for(;;) { + fd := sys->open(path, Sys->OREAD); + if(fd == nil) { + sys->sleep(500); + continue; + } + ctl := agentlib->strip(blockread(fd)); + fd = nil; + if(ctl == nil || ctl == "") + continue; + case ctl { + "cancel" or "refine" => + turngen++; + turnpaused = 0; + cancelspeechqueue(); + if(turnactive) + setstatus("cancelling"); + "pause" => + turnpaused = 1; + cancelspeechqueue(); + if(turnactive) + setstatus("paused"); + "resume" => + turnpaused = 0; + if(turnactive) + setstatus("working"); + * => + log("unknown turn control: " + ctl); + } + } +} + +# Tool backends do not all expose hard interruption. Cancellation therefore +# takes effect at the next LLM/tool boundary and prevents subsequent actions. +turncheckpoint(gen: int): int +{ + if(gen != turngen) + return -1; + while(turnpaused) { + setstatus("paused"); + sys->sleep(100); + if(gen != turngen) + return -1; + } + return 0; } # Read from a blocking fd, strip trailing newline @@ -451,6 +670,8 @@ updatedialogue(idx: int, progress, title, text: string) readuserinput(): string { path := sys->sprint("/mnt/ui/activity/%d/conversation/input", actid); + if(inputmode() == "v") + path = sys->sprint("/mnt/ui/activity/%d/conversation/voiceinput", actid); fd := sys->open(path, Sys->OREAD); if(fd == nil) return ""; @@ -479,8 +700,16 @@ needsapproval(toolname, args: string): int return 0; } -# Pre-tool approval gate. Returns "allow" or "deny". -pretoolapproval(toolname, args: string): string +# Send one timeout tick without leaving a resident timer process behind. +approvaltick(c: chan of int) +{ + sys->sleep(100); + c <-= 1; +} + +# Pre-tool approval gate. Returns "allow" or "deny". Cancellation is checked +# while blocked so a spoken Stop cannot strand the turn in an approval read. +pretoolapproval(toolname, args: string, gen: int): string { if(!needsapproval(toolname, args)) return "allow"; @@ -490,11 +719,29 @@ pretoolapproval(toolname, args: string): string log("pretool: awaiting approval for " + toolname); setstatus("blocked"); seturgency(2); - response := readuserinput(); + # Use the already-running shared readers so typed buttons and spoken + # Allow/Deny reach the same gate; a second direct reader races them. + response := ""; + done := 0; + while(!done) { + tick := chan[1] of int; + spawn approvaltick(tick); + alt { + (nil, response) = <-inputc => + done = 1; + <-tick => + if(gen != turngen) { + response = "Deny"; + done = 1; + } + } + } log("pretool: user responded: " + response); setstatus("working"); seturgency(0); - if(response == "Deny" || response == "deny" || response == "no") { + # Fail closed: only an explicit affirmative releases a sensitive tool. + if(response != "Allow" && response != "allow" && response != "approve" && + response != "yes") { if(didx >= 0) updatedialogue(didx, "", "Denied", ""); return "deny"; @@ -749,7 +996,7 @@ initsession(): string # Register speech resource if speech9p is available if(autospeak) { ctxpath := sys->sprint("/mnt/ui/activity/%d/context/ctl", actid); - writefile(ctxpath, "resource add path=speech label=Speech type=tool status=idle"); + writefile(ctxpath, "resource upsert path=/n/speech label=Voice type=audio status=idle"); log("context: registered speech resource"); } @@ -1323,7 +1570,25 @@ handleslash(cmd: string): int ack = "usage: /tools +name or /tools -name"; } "voice" => - if(cmdarg == "" || cmdarg == "on") { + (vcmd, vrest) := str->splitl(cmdarg, " \t"); + varg := str->drop(vrest, " \t"); + if(vcmd == "mode") { + if(varg == "on") { + autospeak = 1; + writefile("/mnt/ui/input-mode", "v"); + writefile(sys->sprint("/mnt/ui/activity/%d/context/ctl", actid), + "resource upsert path=/n/speech label=Voice type=audio status=waiting via=voice-mode"); + ack = "voice mode: on"; + } else if(varg == "off") { + cancelspeechqueue(); + writefile("/mnt/ui/input-mode", "k"); + writefile(sys->sprint("/mnt/ui/activity/%d/context/ctl", actid), + "resource upsert path=/n/speech label=Voice type=audio status=idle via=voice-mode"); + ack = "voice mode: off"; + } else { + ack = "usage: /voice mode on|off"; + } + } else if(cmdarg == "" || cmdarg == "on") { autospeak = 1; ack = "voice: auto-speak enabled"; } else if(cmdarg == "off") { @@ -1346,6 +1611,7 @@ handleslash(cmd: string): int "/tools +name — add tool\n" + "/tools -name — remove tool\n" + "/voice on|off — toggle auto-speak\n" + + "/voice mode on|off — toggle hands-free voice mode\n" + "/voice — change voice\n" + "/diff — show cowfs changes\n" + "/promote [path] — promote cowfs changes\n" + @@ -1468,6 +1734,8 @@ cowrevert(arg: string): string agentturn(input: string) { + mygen := turngen; + turnactive = 1; agentlib->dedupreset(); # fresh read-cache per turn # Sync convcount with actual server message count before streaming. @@ -1499,7 +1767,13 @@ agentturn(input: string) streambase := "/mnt/llm/" + sessionid; hitlimit := 1; + cancelled := 0; for(step := 0; step < maxsteps; step++) { + if(turncheckpoint(mygen) < 0) { + cancelled = 1; + break; + } + stepstarted := sys->millisec(); log(sys->sprint("step %d: writing %d bytes to LLM", step + 1, len array of byte prompt)); # Start async generation — returns immediately with new llmsrv, @@ -1518,11 +1792,14 @@ agentturn(input: string) # For step > 0 (tool-execution follow-ups) defer creation to the first # actual chunk, so tool-only steps produce no spurious bubble. placeholder_idx := -1; + speechconsumed := 0; + streamtext := ""; if(streamfd != nil) { log("stream: reading " + streampath); buf := array[512] of byte; growing := ""; nchunks := 0; + spoken := 0; # Show activity cursor immediately on the first step. if(step == 0) { placeholder_idx = convcount; @@ -1534,6 +1811,20 @@ agentturn(input: string) break; growing += string buf[0:n]; nchunks++; + if(nchunks == 1) + log(sys->sprint("voice-timing first-token %dms", sys->millisec() - stepstarted)); + if(autospeak) { + # Stream only the parsed assistant text. The raw stream may + # contain STOP/TOOL protocol records which must never be read + # aloud as if they were a user-facing answer. + (nil, nil, speechtext) := agentlib->parsellmresponse(growing); + oldspoken := spoken; + spoken = queuespeechavailable(speechtext, spoken, 0); + if(oldspoken == 0 && spoken > 0) + log(sys->sprint("voice-timing first-audio-queued %dms", sys->millisec() - stepstarted)); + speechconsumed = spoken; + streamtext = speechtext; + } # Create placeholder on the first chunk if not already created # (steps > 0), seeded with actual text. if(placeholder_idx < 0) { @@ -1577,6 +1868,10 @@ agentturn(input: string) writemsg("veltro", "(no response from LLM)"); break; } + if(turncheckpoint(mygen) < 0) { + cancelled = 1; + break; + } log("llm: " + agentlib->truncate(response, 200)); @@ -1627,6 +1922,17 @@ agentturn(input: string) updateliveconvmsg(placeholder_idx, text); else writemsg("veltro", text); + if(autospeak) { + # `spoken` exists only on the streaming branch. Derive the + # consumed prefix from growing when available; legacy backends + # simply queue the complete parsed response here. + consumed := 0; + if(speechconsumed > 0 && speechconsumed <= len text && + speechconsumed <= len streamtext && + text[0:speechconsumed] == streamtext[0:speechconsumed]) + consumed = speechconsumed; + queuespeechavailable(text, consumed, 1); + } } else if(placeholder_idx >= 0) { # Tool-only or empty response: clear placeholder so tile is hidden updateliveconvmsg(placeholder_idx, ""); @@ -1641,9 +1947,14 @@ agentturn(input: string) # Execute tools, intercepting say locally. results: list of (string, string); for(tc := tools; tc != nil; tc = tl tc) { + if(turncheckpoint(mygen) < 0) { + cancelled = 1; + break; + } (id, name, args) := hd tc; if(str->tolower(name) == "say") { writemsg("veltro", args); + queuespeech(args); results = (id, "said") :: results; } else { # Mark the tool as active in the context zone for the full duration. @@ -1670,7 +1981,7 @@ agentturn(input: string) } # Pre-tool approval for destructive operations - approval := pretoolapproval(nm, eargs); + approval := pretoolapproval(nm, eargs, mygen); if(approval == "deny") { results = (id, "error: operation denied by operator") :: results; writefile(ctxpath, "resource update path=" + nm + " status=idle"); @@ -1692,6 +2003,8 @@ agentturn(input: string) setstatus(nm); log("tool " + name + ": calling with " + string len eargs + " bytes"); result := agentlib->calltool(name, eargs); + if(turncheckpoint(mygen) < 0) + cancelled = 1; agentlib->deduprecord(nm, eargs, result, step); setstatus("working"); writefile(ctxpath, "resource update path=" + nm + " status=idle"); @@ -1722,6 +2035,8 @@ agentturn(input: string) results = (id, result) :: results; } } + if(cancelled) + break; # Reverse results (list was built by prepending). rev: list of (string, string); @@ -1761,7 +2076,11 @@ agentturn(input: string) prompt = agentlib->buildtoolresults(rev); } - if(hitlimit) { + turnactive = 0; + if(cancelled) { + writemsg("veltro", "(cancelled)"); + setstatus("idle"); + } else if(hitlimit) { writemsg("veltro", sys->sprint( "(reached %d-step limit — send another message to continue)", maxsteps)); setstatus("idle"); @@ -1785,6 +2104,9 @@ init(nil: ref Draw->Context, args: list of string) { sys = load Sys Sys->PATH; stderr = sys->fildes(2); + speakq = chan of ref Speakreq; + speakdone = chan of int; + spawn speechmanager(); str = load String String->PATH; if(str == nil) @@ -1875,6 +2197,7 @@ init(nil: ref Draw->Context, args: list of string) if(backend == nil || backend == "") backend = "api"; dial := readndbfield("/lib/ndb/llm", "dial"); + log("llm configuration mode=" + mode + " backend=" + backend); llmconfigured := 0; if(mode == "remote") { # Remote 9P mount: configured iff a dial address is set. @@ -1884,6 +2207,10 @@ init(nil: ref Draw->Context, args: list of string) } else if(backend == "api") { # Check factotum for an anthropic API key ctldata := agentlib->readfile("/mnt/factotum/ctl"); + if(ctldata != nil && len ctldata > 0) + log("factotum ctl readable"); + else + log("factotum ctl unreadable or empty"); if(ctldata != nil && len ctldata > 0) { (nil, ctllines) := sys->tokenize(ctldata, "\n"); for(; ctllines != nil; ctllines = tl ctllines) { @@ -1892,6 +2219,10 @@ init(nil: ref Draw->Context, args: list of string) llmconfigured = 1; } } + if(llmconfigured) + log("anthropic factotum key present"); + else + log("anthropic factotum key absent"); } else if(backend == "openai" || backend == "cli") { # Ollama/OpenAI, or the claude-gate CLI gateway (backend=cli, # OpenAI-shaped on localhost): configured if a URL is set @@ -2047,6 +2378,11 @@ init(nil: ref Draw->Context, args: list of string) } inputpath := sys->sprint("/mnt/ui/activity/%d/conversation/input", actid); + voiceinputpath := sys->sprint("/mnt/ui/activity/%d/conversation/voiceinput", actid); + inputc = chan of (int, string); + spawn inputreader(inputpath, 0, inputc); + spawn inputreader(voiceinputpath, 1, inputc); + spawn controlreader(sys->sprint("/mnt/ui/activity/%d/conversation/control", actid)); log(sys->sprint("ready — activity %d, session %s, max %d steps, %d existing msgs", actid, sessionid, maxsteps, convcount)); @@ -2074,19 +2410,38 @@ init(nil: ref Draw->Context, args: list of string) agentturn(kickoff); } - # Main loop: re-open input fd each iteration because 9P offset - # advances after read, causing subsequent reads to return EOF. + # Main loop: both input paths are read concurrently so an input-mode + # switch takes effect immediately instead of after the next message on + # the previously selected path. Voice-originated turns arrive on + # conversation/voiceinput (written by voicemode); typed turns on + # conversation/input. for(;;) { - inputfd := sys->open(inputpath, Sys->OREAD); - if(inputfd == nil) - fatal("cannot open " + inputpath); - human := blockread(inputfd); - inputfd = nil; + (isvoice, human) := <-inputc; if(human == nil) { + if(isvoice) { + # Older luciuisrv without voiceinput, or a flushed + # read — voice input is unavailable but the bridge + # keeps serving typed input. + log("voiceinput unavailable"); + continue; + } log("input closed"); break; } - log("human: " + human); + if(isvoice) + log("voice: " + human); + else + log("human: " + human); + + # Mutual exclusion: while voice mode is active, plain typed text + # is paused. Slash commands still work so /voice mode off can + # always be typed. + if(!isvoice && inputmode() == "v" && + !(len human > 0 && human[0] == '/')) { + writemsg("assistant", + "voice mode active — press Esc or say \"keyboard\" to resume typing"); + continue; + } # Slash commands (/bind, /unbind, /tools, /help) are handled locally. # They update tools9p state and reply immediately; agent is not invoked. diff --git a/appl/cmd/luciconv.b b/appl/cmd/luciconv.b index cc778e241..ecc722018 100644 --- a/appl/cmd/luciconv.b +++ b/appl/cmd/luciconv.b @@ -111,6 +111,8 @@ msgstore: array of ref ConvMsg; nmsg := 0; inputbuf: string; inputpos := 0; # cursor position within inputbuf +draftbuf: string; # replaceable STT hypothesis; never submitted +draftstatusbuf: string; # listening/countdown state shown with the hypothesis scrollpx := 0; maxscrollpx := 0; viewport_h := 400; @@ -118,12 +120,7 @@ lastrendw := 0; username := "human"; agentname := ""; # agent display name from /lib/veltro/agent-name (branding) -# Voice input state -VOICE_IDLE: con 0; -VOICE_REC: con 1; -voicestate := VOICE_IDLE; -voicech: chan of string; -micrect: Rect; # Hit area for mic button +micrect: Rect; # Hit area for the voice-mode toggle button inputrect: Rect; # Hit area for input field # Tile layout (populated by drawconversation, used for click hit-testing) @@ -211,6 +208,8 @@ init(img: ref Draw->Image, dsp: ref Draw->Display, inputbuf = ""; inputpos = 0; + draftbuf = ""; + draftstatusbuf = ""; username = readdevuser(); # Agent display name (branding); empty => fall back to the raw role string. an := readfile("/lib/veltro/agent-name"); @@ -219,12 +218,13 @@ init(img: ref Draw->Image, dsp: ref Draw->Display, an = an[:len an-1]; agentname = an; } - voicech = chan of string; msgstore = array[32] of ref ConvMsg; nmsg = 0; - if(actid >= 0) + if(actid >= 0) { loadmessages(); + loaddraft(); + } redrawconv(); @@ -248,10 +248,11 @@ init(img: ref Draw->Image, dsp: ref Draw->Display, } # Button-1 just pressed if(p.buttons == 1 && wasdown == 0) { + locked := voiceactive(); # Mobile soft keyboard: raise it only when the input field # is tapped; hide it on any other tap in the chat zone. if(mobile) { - if(inputrect.dx() > 0 && inputrect.contains(p.xy)) { + if(!locked && inputrect.dx() > 0 && inputrect.contains(p.xy)) { # INFR-166: tell SDL the actual focused # widget rect (in window points) so it # slides this widget — not a hard-coded @@ -274,16 +275,16 @@ init(img: ref Draw->Image, dsp: ref Draw->Display, # pressing Return on desktop: submit inputbuf if # non-empty, then clear it. if(mobile && sendrect.dx() > 0 && sendrect.contains(p.xy)) { - if(len inputbuf > 0) { + if(!locked && len inputbuf > 0) { sendinput(inputbuf); inputbuf = ""; inputpos = 0; } redrawconv(); } else - # Check mic button first + # Check the voice-mode toggle button first if(micrect.dx() > 0 && micrect.contains(p.xy)) { - startvoice(); + togglevoice(); redrawconv(); } else { # Check for dialogue button clicks first @@ -310,7 +311,7 @@ init(img: ref Draw->Image, dsp: ref Draw->Display, } # Button-3 press: context menu if((p.buttons & 4) != 0 && (wasdown & 4) == 0) { - if(inputrect.dx() > 0 && inputrect.contains(p.xy)) { + if(!voiceactive() && inputrect.dx() > 0 && inputrect.contains(p.xy)) { # Input field context menu if(menumod != nil) { items := array[] of {"Copy", "Paste"}; @@ -349,10 +350,15 @@ init(img: ref Draw->Image, dsp: ref Draw->Display, prevbuttons = 0; } k := <-kbd => + if(voiceactive() && k != 0 && k != 16rF00E && k != 16rF00F) { + # Voice owns the pending turn. Preserve the typed compose + # verbatim until the user exits voice mode. + redrawconv(); + } else { case k { 0 => - # Ctrl+Space — toggle voice input - startvoice(); + # Ctrl+Space — toggle voice mode (same as Esc-V / Voice chip) + togglevoice(); 1 => # Ctrl-A — beginning of line inputpos = 0; @@ -441,16 +447,7 @@ init(img: ref Draw->Image, dsp: ref Draw->Display, } } redrawconv(); - vtext := <-voicech => - # Voice transcription result received - voicestate = VOICE_IDLE; - if(vtext != nil && vtext != "" && !hasprefix(vtext, "error:")) { - inputbuf = vtext; - sendinput(inputbuf); - inputbuf = ""; - inputpos = 0; } - redrawconv(); ev := <-evch => handleevent(ev); redrawconv(); @@ -479,9 +476,24 @@ handleevent(ev: string) if(hasprefix(ev, "switchactivity ")) { newid := strtoint(ev[len "switchactivity ":]); if(newid >= 0) { + # A draft belongs only to the active listening context. Clear + # the old activity without touching the user's typed compose. + writedraft(""); + writedraftstatus(""); actid_g = newid; + draftbuf = ""; + draftstatusbuf = ""; loadmessages(); + loaddraft(); } + } else if(hasprefix(ev, "input-mode ")) { + if(strip(ev[len "input-mode ":]) == "v") { + reqkbd(0); + if(softkbd != nil) + softkbd->clear_rect(); + } + } else if(ev == "conversation draft") { + loaddraft(); } else if(hasprefix(ev, "conversation update ")) { idx := strtoint(ev[len "conversation update ":]); if(idx >= 0) @@ -563,7 +575,11 @@ drawconversation(zone: Rect) inputr := Rect((zone.min.x + pad, zone.max.y - inputh), (zone.max.x - pad, zone.max.y)); inputrect = inputr; - mainwin.draw(inputr, inputcol, nil, (0, 0)); + locked := voiceactive(); + inputfill := inputcol; + if(locked) + inputfill = bordercol; + mainwin.draw(inputr, inputfill, nil, (0, 0)); # Send button — mobile only. Sits between the input area and # the mic button. Tapping it submits the current inputbuf, same @@ -578,9 +594,15 @@ drawconversation(zone: Rect) sendx := inputr.max.x - sendw; sendy := inputr.min.y; sendrect = Rect((sendx, sendy), (inputr.max.x, inputr.max.y)); - mainwin.draw(sendrect, accentcol, nil, (0, 0)); + sendfill := accentcol; + sendcol := bgcol; + if(locked) { + sendfill = bordercol; + sendcol = dimcol; + } + mainwin.draw(sendrect, sendfill, nil, (0, 0)); sty := sendy + (inputh - mainfont.height) / 2; - mainwin.text((sendx + pad, sty), bgcol, (0, 0), mainfont, "Send"); + mainwin.text((sendx + pad, sty), sendcol, (0, 0), mainfont, "Send"); } # Mic button at right edge of input (or left of Send on mobile). @@ -589,16 +611,14 @@ drawconversation(zone: Rect) # what's tappable instead of guessing. REC state inverts to the # accent fill so a recording session reads as the active action, # matching how Send is drawn. - miclabel: string; + miclabel := "voice"; micfill: ref Image; miccol: ref Image; - case voicestate { - VOICE_REC => - miclabel = "REC"; + if(locked) { + # Voice mode on: inverted accent fill, same treatment as REC/Send. micfill = accentcol; miccol = bgcol; - * => - miclabel = "mic"; + } else { micfill = bordercol; # muted chrome — clearly a button, not the primary action miccol = textcol; } @@ -625,44 +645,61 @@ drawconversation(zone: Rect) maxitw := inputr.dx() - 2 * pad - 8 - micw - sendw; cw := 8; - # Clamp inputpos + # Clamp the real keyboard compose cursor. Voice mode leaves the typed + # compose visible but locks and dims it while the pending turn is shown + # in the conversation above. if(inputpos < 0) inputpos = 0; if(inputpos > len inputbuf) inputpos = len inputbuf; - # Find a visible window of inputbuf that keeps the cursor in view. + displaytext := inputbuf; + displaypos := inputpos; + displaycol := textcol; + showcursor := 1; + if(locked) { + displaycol = dimcol; + showcursor = 0; + } + + # Find a visible window that keeps the compose cursor in view. # Start by including the cursor position, then expand left/right. - vstart := inputpos; - vend := inputpos; + vstart := displaypos; + vend := displaypos; # Expand right first - while(vend < len inputbuf && mainfont.width(inputbuf[vstart:vend+1]) + cw <= maxitw) + while(vend < len displaytext && mainfont.width(displaytext[vstart:vend+1]) + cw <= maxitw) vend++; # Expand left - while(vstart > 0 && mainfont.width(inputbuf[vstart-1:vend]) + cw <= maxitw) + while(vstart > 0 && mainfont.width(displaytext[vstart-1:vend]) + cw <= maxitw) vstart--; - itext := inputbuf[vstart:vend]; - mainwin.text((itx, ity), textcol, (0, 0), mainfont, itext); + itext := displaytext[vstart:vend]; + mainwin.text((itx, ity), displaycol, (0, 0), mainfont, itext); # Block cursor at cursor position within visible text - pre := inputbuf[vstart:inputpos]; - ch := mainfont.height; - cx := itx + mainfont.width(pre); - cy := ity; - mainwin.draw(Rect((cx, cy), (cx + cw, cy + ch)), cursorcol, nil, (0, 0)); + if(showcursor) { + pre := displaytext[vstart:displaypos]; + ch := mainfont.height; + cx := itx + mainfont.width(pre); + cy := ity; + mainwin.draw(Rect((cx, cy), (cx + cw, cy + ch)), cursorcol, nil, (0, 0)); + } - if(nmsg == 0) { + hasdraft := draftbuf != nil && draftbuf != ""; + drawn := nmsg; + if(hasdraft) + drawn++; + if(drawn == 0) { drawcentertext(Rect((zone.min.x, zone.min.y), (zone.max.x, msgy)), "No messages yet"); return; } # Reset tile layout - tilelayout = array[nmsg + 1] of ref TileRect; + tilelayout = array[drawn + 1] of ref TileRect; ntiles = 0; - dlgbuttons = array[nmsg * 4] of ref DlgButton; # up to 4 buttons per dialogue + dlgbuttons = array[drawn * 4] of ref DlgButton; # up to 4 buttons per dialogue ndlgbuttons = 0; tilegap := 4; @@ -677,12 +714,30 @@ drawconversation(zone: Rect) lastrendw = tilew; } - marr := msgstore; + marr := array[drawn] of ref ConvMsg; + for(mi := 0; mi < nmsg; mi++) + marr[mi] = msgstore[mi]; + if(hasdraft) + marr[nmsg] = ref ConvMsg("human", draftbuf, "", nil, + "voice-draft", draftstatusbuf, "", ""); # Pass 1: estimate heights - harr := array[nmsg] of int; + harr := array[drawn] of int; total_h := 0; - for(pi := 0; pi < nmsg; pi++) { + for(pi := 0; pi < drawn; pi++) { + if(marr[pi].dtype == "voice-draft") { + t := strip(marr[pi].text); + ls := wraptext(t, tilew - 8); + n := 0; + for(wl := ls; wl != nil; wl = tl wl) + n++; + h := mainfont.height + n * mainfont.height + 2 * tpadv; + if(marr[pi].title != "") + h += mainfont.height; + harr[pi] = h; + total_h += h + tilegap; + continue; + } # Dialogue tiles have their own height calculation if(marr[pi].dtype == "dialogue" || marr[pi].dtype == "form") { DLGPAD := 8; @@ -750,7 +805,7 @@ drawconversation(zone: Rect) # Pass 2: render visible messages codebg := codebgcol_g; ey := msgy + scrollpx; - for(ri := nmsg - 1; ri >= 0; ri--) { + for(ri := drawn - 1; ri >= 0; ri--) { if(harr[ri] == 0) continue; tiletop_e := ey - harr[ri] - tilegap; @@ -782,7 +837,7 @@ drawconversation(zone: Rect) # Draw messages bottom-up y := msgy + scrollpx; - for(i := nmsg - 1; i >= 0; i--) { + for(i := drawn - 1; i >= 0; i--) { tileh := harr[i]; if(tileh == 0) continue; @@ -797,6 +852,7 @@ drawconversation(zone: Rect) msg := marr[i]; isdialogue := msg.dtype == "dialogue" || msg.dtype == "form"; + pending := msg.dtype == "voice-draft"; human := msg.role == "human" && !isdialogue; errrole := msg.role == "error" && !isdialogue; tilecol: ref Image; @@ -804,6 +860,9 @@ drawconversation(zone: Rect) if(isdialogue) { tilecol = veltrocol; rolecol = accentcol; + } else if(pending) { + tilecol = inputcol; + rolecol = accentcol; } else if(human) { tilecol = humancol; rolecol = text2col; @@ -822,6 +881,16 @@ drawconversation(zone: Rect) if(drawtop < drawbot) { tiler := Rect((tilex, drawtop), (tilex + tilew, drawbot)); mainwin.draw(tiler, tilecol, nil, (0, 0)); + if(pending) { + mainwin.draw(Rect((tiler.min.x, tiler.min.y), + (tiler.max.x, tiler.min.y + 1)), accentcol, nil, (0, 0)); + mainwin.draw(Rect((tiler.min.x, tiler.max.y - 1), + (tiler.max.x, tiler.max.y)), accentcol, nil, (0, 0)); + mainwin.draw(Rect((tiler.min.x, tiler.min.y), + (tiler.min.x + 1, tiler.max.y)), accentcol, nil, (0, 0)); + mainwin.draw(Rect((tiler.max.x - 1, tiler.min.y), + (tiler.max.x, tiler.max.y)), accentcol, nil, (0, 0)); + } } if(ntiles < len tilelayout) tilelayout[ntiles++] = ref TileRect( @@ -922,7 +991,9 @@ drawconversation(zone: Rect) ty := tiletop + tpadv; rolelabel := msg.role; - if(human) + if(pending) + rolelabel = username + " - not sent"; + else if(human) rolelabel = username; else if(!errrole && agentname != "") rolelabel = agentname; @@ -945,6 +1016,13 @@ drawconversation(zone: Rect) } ty += mainfont.height; } + if(pending && msg.title != "") { + if(ty < msgy && ty + mainfont.height > zone.min.y) { + lx := tilex + tilew - mainfont.width(msg.title); + mainwin.text((lx, ty), accentcol, (0, 0), mainfont, msg.title); + } + ty += mainfont.height; + } } else if(errrole) { lines := wraptext(msg.text, tilew - 8); for(ll := lines; ll != nil; ll = tl ll) { @@ -1162,87 +1240,79 @@ sendinput(text: string) sys->write(fd, b, len b); } -# --- Voice input --- - -startvoice() +loaddraft() { - if(voicestate != VOICE_IDLE) - return; - # Check if speech9p is mounted - (ok, nil) := sys->stat("/n/speech/hear"); - if(ok < 0) { - sys->fprint(stderr, "luciconv: /n/speech not mounted\n"); + if(actid_g < 0) { + draftbuf = ""; + draftstatusbuf = ""; return; } - voicestate = VOICE_REC; - spawn voiceworker(voicech); + path := sys->sprint("%s/activity/%d/conversation/draft", mountpt_g, actid_g); + draftbuf = strip(readfile(path)); + path = sys->sprint("%s/activity/%d/conversation/draft-status", mountpt_g, actid_g); + draftstatusbuf = strip(readfile(path)); } -VOICE_TIMEOUT_MS: con 30000; - -VoiceFD: adt { - fd: ref Sys->FD; -}; - -voiceworker(ch: chan of string) +writedraft(text: string) { - fd := sys->open("/n/speech/hear", Sys->ORDWR); - if(fd == nil) { - ch <-= "error: cannot open /n/speech/hear"; + if(actid_g < 0) return; - } + path := sys->sprint("%s/activity/%d/conversation/draft", mountpt_g, actid_g); + fd := sys->open(path, Sys->OWRITE | Sys->OTRUNC); + if(fd == nil) + return; + b := array of byte text; + sys->write(fd, b, len b); +} - # Write start command to begin recording - cmd := array of byte "start 5000"; - if(sys->write(fd, cmd, len cmd) < 0) { - ch <-= "error: write to hear failed"; +writedraftstatus(text: string) +{ + if(actid_g < 0) return; - } + path := sys->sprint("%s/activity/%d/conversation/draft-status", mountpt_g, actid_g); + fd := sys->open(path, Sys->OWRITE | Sys->OTRUNC); + if(fd == nil) + return; + b := array of byte text; + sys->write(fd, b, len b); +} - # Read transcription result with timeout. - # Use a shared VoiceFD ref so timeout can nil the fd, - # preventing voiceread from looping after timeout. - # NOTE: if voiceread is blocked inside sys->read when timeout - # fires, it will remain blocked until the underlying kernel FD - # is closed or the read returns. This is a known limitation - # of Limbo's FD model -- there is no sys->close(). - sys->seek(fd, big 0, Sys->SEEKSTART); - vfd := ref VoiceFD(fd); - resultch := chan of string; - spawn voiceread(vfd, resultch); - - timeoutch := chan of int; - spawn voicetimeout(timeoutch, VOICE_TIMEOUT_MS); - - alt { - result := <-resultch => - ch <-= result; - <-timeoutch => - vfd.fd = nil; - ch <-= "error: voice recognition timed out"; - } +# --- Voice input --- +# The compose-row voice button toggles the SAME voice mode as Esc-V and +# lucictx's Voice chip: /mnt/ui/input-mode flips between "v" and "k", and +# the resident voicemode daemon owns the microphone, wake gating, drafts, +# and transcript submission. luciconv keeps no speech state of its own — +# the old press-to-dictate flow (a one-shot /n/speech/hear read pasted +# into the compose box) duplicated voice mode through a second, subtly +# different microphone pathway and is gone. + +voiceactive(): int +{ + return strip(readfile(mountpt_g + "/input-mode")) == "v"; } -voiceread(vfd: ref VoiceFD, ch: chan of string) +togglevoice() { - result := ""; - buf := array[8192] of byte; - for(;;) { - fd := vfd.fd; - if(fd == nil) - break; - n := sys->read(fd, buf, len buf); - if(n <= 0) - break; - result += string buf[0:n]; + path := mountpt_g + "/input-mode"; + ctl := sys->sprint("%s/activity/%d/context/ctl", mountpt_g, actid_g); + if(voiceactive()) { + writestring(path, "k"); + writestring(ctl, "resource upsert path=/n/speech label=Voice " + + "type=audio status=idle via=voice-mode"); + } else { + writestring(path, "v"); + writestring(ctl, "resource upsert path=/n/speech label=Voice " + + "type=audio status=starting via=voice-mode"); } - ch <-= result; } -voicetimeout(ch: chan of int, ms: int) +writestring(path, text: string) { - sys->sleep(ms); - ch <-= 1; + fd := sys->open(path, Sys->OWRITE); + if(fd == nil) + return; + b := array of byte text; + sys->write(fd, b, len b); } # --- Word wrapping --- diff --git a/appl/cmd/lucictx.b b/appl/cmd/lucictx.b index b2c406bc2..3f3577f8f 100644 --- a/appl/cmd/lucictx.b +++ b/appl/cmd/lucictx.b @@ -176,6 +176,10 @@ ntoolplusrects := 0; toolentryrects: array of Rect; ntoolentryrects := 0; +# Resource row rects +resourcerects: array of Rect; +nresourcerects := 0; + # Browse rect (inside user namespace) browserect: Rect; mobile := 0; # /env/infmobile=1 — floor browser/list rows at a tap target @@ -470,6 +474,26 @@ init(img: ref Draw->Image, dsp: ref Draw->Display, } } + # Resource row click — /n/speech toggles voice input mode. + if(!tabclicked) { + for(ri := 0; ri < nresourcerects; ri++) { + if(resourcerects[ri].contains(clickpt)) { + ridx := 0; + for(rl := resources; rl != nil; rl = tl rl) { + if(ridx == ri) { + if((hd rl).path == "/n/speech") { + togglevoicemode(); + tabclicked = 1; + } + break; + } + ridx++; + } + break; + } + } + } + # Agent NS entry [ro]/[rw] badge click — toggle permission # (checked before path-click since the badge is right-aligned # inside the same row band) @@ -849,6 +873,7 @@ drawcontext(zone: Rect) ntoolentryrects = 0; nctxentryrects = 0; nnsentryrects = 0; + nresourcerects = 0; toolavailhdrrect = Rect((0, 0), (0, 0)); browserect = Rect((0, 0), (0, 0)); agentnshdrrect = Rect((0, 0), (0, 0)); @@ -899,6 +924,52 @@ drawcontext(zone: Rect) y += secgap; } + # --- Resources section --- + if(resources != nil) { + if(y + mainfont.height > vis_top && y < vis_bot) + mainwin.text((zone.min.x + pad, y), labelcol, (0, 0), mainfont, "Resources"); + y += mainfont.height + 4; + + resourcerects = array[64] of Rect; + for(rp := resources; rp != nil; rp = tl rp) { + res := hd rp; + visible := y + entryH > vis_top && y < vis_bot; + rowtexty := y + (entryH - mainfont.height) / 2; + if(nresourcerects < len resourcerects) + resourcerects[nresourcerects++] = Rect( + (zone.min.x, y), (zone.max.x, y + entryH)); + + indcol3 := dimcol; + if(res.status == "active" || res.status == "listening" || + res.status == "speaking" || res.status == "processing") + indcol3 = accentcol; + else if(res.status == "error") + indcol3 = redcol; + else if(res.status == "waiting" || res.status == "starting") + indcol3 = greencol; + + if(visible) { + tindy := y + (entryH - indh) / 2; + mainwin.draw(Rect( + (zone.min.x + pad, tindy), + (zone.min.x + pad + indw, tindy + indh)), + indcol3, nil, (0, 0)); + label := res.label; + if(label == nil || label == "") + label = res.path; + mainwin.text((zone.min.x + pad + indw + 6, rowtexty), + text2col, (0, 0), mainfont, label); + if(res.status != nil && res.status != "") { + statusw := mainfont.width(res.status); + mainwin.text((zone.max.x - pad - statusw, rowtexty), + dimcol, (0, 0), mainfont, res.status); + } + } + y += entryH; + } + y += secgap; + } + # --- Agent Namespace section --- { nind := "▸"; @@ -2374,6 +2445,29 @@ writetofile(path: string, text: string): int return n; } +# Voice-chip click. The input-mode write is what voicemode acts on, but it is +# asynchronous — voicemode may take a moment to answer, and if it is not running +# at all it never will. Upsert the chip row here too so the click always produces +# an immediate visible state change ("starting"/"idle") rather than appearing to +# do nothing. voicemode overwrites the row with the real state (waiting, +# listening, error) as soon as it picks the mode change up; a row still reading +# "starting" seconds later means voicemode never responded. +togglevoicemode() +{ + path := mountpt_g + "/input-mode"; + ctl := sys->sprint("%s/activity/%d/context/ctl", mountpt_g, actid_g); + mode := strip(readfile(path)); + if(mode == "v") { + writetofile(path, "k"); + writetofile(ctl, "resource upsert path=/n/speech label=Voice " + + "type=audio status=idle via=voice-mode"); + } else { + writetofile(path, "v"); + writetofile(ctl, "resource upsert path=/n/speech label=Voice " + + "type=audio status=starting via=voice-mode"); + } +} + strip(s: string): string { while(len s > 0 && (s[len s - 1] == '\n' || s[len s - 1] == ' ' || s[len s - 1] == '\t')) diff --git a/appl/cmd/lucifer.b b/appl/cmd/lucifer.b index abbfcdd60..091c31048 100644 --- a/appl/cmd/lucifer.b +++ b/appl/cmd/lucifer.b @@ -275,6 +275,10 @@ MOBILE_TITLEBARH: con MOBILE_TAPMIN; # per-zone title bar = one tap target tall # nslistener process ID — killed and respawned on activity switch nslistenerpid := -1; +# Hands-free voice mode active (input-mode "v") — tracked from global +# events by globallistener; kbdproc uses it for the Esc escape hatch. +voicemodeon := 0; + # Zone channels convMouseCh: chan of ref Pointer; convKbdCh: chan of int; @@ -2186,6 +2190,11 @@ globallistener() lucipres_g->deliverevent("activity new " + string newid); alt { uievent <-= 1 => ; * => ; } } + if(hasprefix(ev, "input-mode ")) { + # Voice-mode tracking for kbdproc's Esc escape hatch. + voicemodeon = strip(ev[len "input-mode ":]) == "v"; + convEvCh <-= ev; + } if(hasprefix(ev, "applaunch ")) { # "applaunch " # Launch the app in the EXACT activity that created it, @@ -2568,6 +2577,14 @@ kbdproc() case escstate { 0 => if(c == 27) { + # Esc is the unconditional voice-mode escape + # hatch: return to keyboard input immediately, + # even while speech helpers are active. voicemode + # sees the input-mode broadcast and cancels TTS. + if(voicemodeon) { + writefile(mountpt + "/input-mode", "k"); + continue; + } escstate = 1; continue; } @@ -2578,6 +2595,10 @@ kbdproc() escarg = 0; continue; } + if(!voicemodeon && (c == 'v' || c == 'V')) { + writefile(mountpt + "/input-mode", "v"); + continue; + } # Bare ESC+char: deliver char as-is (fall through to route) 2 => escstate = 0; diff --git a/appl/cmd/luciuisrv.b b/appl/cmd/luciuisrv.b index 225ddadda..104ede4dc 100644 --- a/appl/cmd/luciuisrv.b +++ b/appl/cmd/luciuisrv.b @@ -23,6 +23,9 @@ implement Luciuisrv; # conversation/ # ctl write new messages # input user text (blocking read) +# voiceinput voice-originated text (blocking read) +# control cancel/pause/resume/refine (blocking read) +# draft replaceable, non-submitting text # 0, 1, 2... numbered message files # presentation/ # ctl create/remove artifacts @@ -106,6 +109,11 @@ Qcatalogentry: con 31; Qartdispath: con 32; # presentation//dispath (app type only) Qartappstatus: con 33; # presentation//appstatus (app type only) Qacturgency: con 34; # /activity/{id}/urgency +Qinputmode: con 35; # /input-mode, k=keyboard, v=voice +Qconvvoiceinput: con 36; # conversation/voiceinput, voice-originated input +Qconvdraft: con 37; # conversation/draft, replaceable non-submitting text +Qconvcontrol: con 38; # conversation/control, active-turn voice controls +Qconvdraftstatus: con 39; # conversation/draft-status, pending-turn presentation # --- QID encoding --- # 64-bit path: [activity_id:16][sub_id:16][unused:24][filetype:8] @@ -194,6 +202,10 @@ Activity: adt { messages: array of ref ConvMsg; nmsg: int; inputq: list of string; + voiceinputq: list of string; + controlq: list of string; + draft: string; + draftstatus: string; # Presentation currentArtifact: string; @@ -234,6 +246,7 @@ activities: array of ref Activity; nact: int; nextactid: int; currentact: int; # id of current activity +inputmode: string; # "k" keyboard, "v" voice # Available resources catalog (loaded once at init from /lib/veltro/resources/) catalog: array of ref CatalogEntry; @@ -373,6 +386,7 @@ init(nil: ref Draw->Context, args: list of string) nact = 0; nextactid = 0; currentact = -1; + inputmode = "k"; vers = 0; # Load available resources catalog from /lib/veltro/resources/ @@ -414,7 +428,7 @@ newactivity(label: string): ref Activity a := ref Activity( id, label, "active", 0, # urgency - array[32] of ref ConvMsg, 0, nil, # conversation + array[32] of ref ConvMsg, 0, nil, nil, nil, "", "", # conversation "", array[16] of ref Artifact, 0, # presentation array[16] of ref Resource, 0, # resources array[8] of ref Gap, 0, # gaps @@ -868,6 +882,8 @@ doread(srv: ref Styxserver, m: ref Tmsg.Read, c: ref Fid) Qactcurrent => srv.reply(styxservers->readbytes(m, array of byte (string currentact + "\n"))); + Qinputmode => + srv.reply(styxservers->readbytes(m, array of byte (inputmode + "\n"))); Qactlabel => a := findactivity(actid); @@ -923,6 +939,46 @@ doread(srv: ref Styxserver, m: ref Tmsg.Read, c: ref Fid) a.inputq = tl a.inputq; srv.reply(styxservers->readbytes(m, data)); } + Qconvvoiceinput => + a := findactivity(actid); + if(a == nil) { + srv.reply(ref Rmsg.Error(m.tag, Enotfound)); + break; + } + if(a.voiceinputq == nil) { + addpending(m.fid, m.tag, Qconvvoiceinput, actid, m); + } else { + data := array of byte (hd a.voiceinputq + "\n"); + a.voiceinputq = tl a.voiceinputq; + srv.reply(styxservers->readbytes(m, data)); + } + Qconvcontrol => + a := findactivity(actid); + if(a == nil) { + srv.reply(ref Rmsg.Error(m.tag, Enotfound)); + break; + } + if(a.controlq == nil) + addpending(m.fid, m.tag, Qconvcontrol, actid, m); + else { + data := array of byte (hd a.controlq + "\n"); + a.controlq = tl a.controlq; + srv.reply(styxservers->readbytes(m, data)); + } + Qconvdraft => + a := findactivity(actid); + if(a == nil) { + srv.reply(ref Rmsg.Error(m.tag, Enotfound)); + break; + } + srv.reply(styxservers->readbytes(m, array of byte a.draft)); + Qconvdraftstatus => + a := findactivity(actid); + if(a == nil) { + srv.reply(ref Rmsg.Error(m.tag, Enotfound)); + break; + } + srv.reply(styxservers->readbytes(m, array of byte a.draftstatus)); Qconvmsg => a := findactivity(actid); @@ -1106,6 +1162,15 @@ dowrite(srv: ref Styxserver, m: ref Tmsg.Write, c: ref Fid) vers++; pushglobalevent("activity " + data); srv.reply(ref Rmsg.Write(m.tag, len m.data)); + Qinputmode => + if(data != "k" && data != "v") { + srv.reply(ref Rmsg.Error(m.tag, "input-mode must be k or v")); + break; + } + inputmode = data; + vers++; + pushglobalevent("input-mode " + inputmode); + srv.reply(ref Rmsg.Write(m.tag, len m.data)); Qactevent => # Write to the event file = flush: discard buffered events and kick @@ -1204,6 +1269,86 @@ dowrite(srv: ref Styxserver, m: ref Tmsg.Write, c: ref Fid) a.inputq = appendstr(a.inputq, data); pushevent(actid, "input"); srv.reply(ref Rmsg.Write(m.tag, len m.data)); + Qconvvoiceinput => + a := findactivity(actid); + if(a == nil) { + srv.reply(ref Rmsg.Error(m.tag, Enotfound)); + break; + } + vdelivered := 0; + vprev: ref PendingRead; + vp := pending; + while(vp != nil) { + next := vp.next; + if(vp.ft == Qconvvoiceinput && vp.actid == actid) { + reply := array of byte (data + "\n"); + srv_g.reply(styxservers->readbytes(vp.m, reply)); + if(vprev == nil) + pending = next; + else + vprev.next = next; + vdelivered = 1; + vp = next; + break; + } + vprev = vp; + vp = next; + } + if(!vdelivered) + a.voiceinputq = appendstr(a.voiceinputq, data); + pushevent(actid, "voiceinput"); + srv.reply(ref Rmsg.Write(m.tag, len m.data)); + Qconvcontrol => + a := findactivity(actid); + if(a == nil) { + srv.reply(ref Rmsg.Error(m.tag, Enotfound)); + break; + } + cdelivered := 0; + cprev: ref PendingRead; + cp := pending; + while(cp != nil) { + next := cp.next; + if(cp.ft == Qconvcontrol && cp.actid == actid) { + reply := array of byte (data + "\n"); + srv_g.reply(styxservers->readbytes(cp.m, reply)); + if(cprev == nil) + pending = next; + else + cprev.next = next; + cdelivered = 1; + cp = next; + break; + } + cprev = cp; + cp = next; + } + if(!cdelivered) + a.controlq = appendstr(a.controlq, data); + pushevent(actid, "conversation control " + data); + srv.reply(ref Rmsg.Write(m.tag, len m.data)); + Qconvdraft => + a := findactivity(actid); + if(a == nil) { + srv.reply(ref Rmsg.Error(m.tag, Enotfound)); + break; + } + # Replacement semantics are intentional: streaming STT hypotheses + # revise one draft without appending messages or submitting turns. + a.draft = data; + vers++; + pushevent(actid, "conversation draft"); + srv.reply(ref Rmsg.Write(m.tag, len m.data)); + Qconvdraftstatus => + a := findactivity(actid); + if(a == nil) { + srv.reply(ref Rmsg.Error(m.tag, Enotfound)); + break; + } + a.draftstatus = data; + vers++; + pushevent(actid, "conversation draft"); + srv.reply(ref Rmsg.Write(m.tag, len m.data)); Qpresctl => a := findactivity(actid); @@ -1345,6 +1490,8 @@ globalctl(data: string): string activities[idx].nbg = 0; activities[idx].pendingevent = nil; activities[idx].inputq = nil; + activities[idx].voiceinputq = nil; + activities[idx].controlq = nil; vers++; pushglobalevent("activity delete " + idstr); return nil; @@ -1993,6 +2140,8 @@ dirgen(p: big): (ref Sys->Dir, string) return (dir(Qid(p, vers, Sys->QTFILE), "notification", big 0, 8r666), nil); Qtoast => return (dir(Qid(p, vers, Sys->QTFILE), "toast", big 0, 8r666), nil); + Qinputmode => + return (dir(Qid(p, vers, Sys->QTFILE), "input-mode", big 0, 8r666), nil); Qactdir => return (dir(Qid(p, vers, Sys->QTDIR), "activity", big 0, 8r755), nil); Qactcurrent => @@ -2013,6 +2162,14 @@ dirgen(p: big): (ref Sys->Dir, string) return (dir(Qid(p, vers, Sys->QTFILE), "ctl", big 0, 8r644), nil); Qconvinput => return (dir(Qid(p, vers, Sys->QTFILE), "input", big 0, 8r666), nil); + Qconvvoiceinput => + return (dir(Qid(p, vers, Sys->QTFILE), "voiceinput", big 0, 8r666), nil); + Qconvcontrol => + return (dir(Qid(p, vers, Sys->QTFILE), "control", big 0, 8r666), nil); + Qconvdraft => + return (dir(Qid(p, vers, Sys->QTFILE), "draft", big 0, 8r666), nil); + Qconvdraftstatus => + return (dir(Qid(p, vers, Sys->QTFILE), "draft-status", big 0, 8r666), nil); Qconvmsg => return (dir(Qid(p, vers, Sys->QTFILE), string subid, big 0, 8r444), nil); Qpresdir => @@ -2090,6 +2247,8 @@ navigator(navops: chan of ref Navop) n.path = MKPATH(0, 0, Qnotification); "toast" => n.path = MKPATH(0, 0, Qtoast); + "input-mode" => + n.path = MKPATH(0, 0, Qinputmode); "activity" => n.path = MKPATH(0, 0, Qactdir); "catalog" => @@ -2148,6 +2307,14 @@ navigator(navops: chan of ref Navop) n.path = MKPATH(actid, 0, Qconvctl); "input" => n.path = MKPATH(actid, 0, Qconvinput); + "voiceinput" => + n.path = MKPATH(actid, 0, Qconvvoiceinput); + "control" => + n.path = MKPATH(actid, 0, Qconvcontrol); + "draft" => + n.path = MKPATH(actid, 0, Qconvdraft); + "draft-status" => + n.path = MKPATH(actid, 0, Qconvdraftstatus); * => # Numbered message file idx := strtoint(n.name); @@ -2293,7 +2460,8 @@ navigator(navops: chan of ref Navop) n.path = MKPATH(0, 0, Qactdir); Qactlabel or Qactstatus or Qacturgency or Qactevent => n.path = MKPATH(actid, 0, Qact); - Qconvctl or Qconvinput or Qconvmsg => + Qconvctl or Qconvinput or Qconvvoiceinput or Qconvcontrol or + Qconvdraft or Qconvdraftstatus or Qconvmsg => n.path = MKPATH(actid, 0, Qconvdir); Qpresctl or Qprescurrent => n.path = MKPATH(actid, 0, Qpresdir); @@ -2329,6 +2497,7 @@ navigator(navops: chan of ref Navop) MKPATH(0, 0, Qevent), MKPATH(0, 0, Qnotification), MKPATH(0, 0, Qtoast), + MKPATH(0, 0, Qinputmode), MKPATH(0, 0, Qactdir), MKPATH(0, 0, Qcatalogdir), }; @@ -2375,8 +2544,9 @@ navigator(navops: chan of ref Navop) Qconvdir => a := findactivity(actid); - # ctl + input + message files - total := 2; + # ctl + keyboard input + voice input + control + draft + + # draft-status + messages + total := 6; if(a != nil) total += a.nmsg; i := n.offset; @@ -2391,9 +2561,29 @@ navigator(navops: chan of ref Navop) cnt--; i++; } + if(i == 2 && cnt > 0) { + n.reply <-= dirgen(MKPATH(actid, 0, Qconvvoiceinput)); + cnt--; + i++; + } + if(i == 3 && cnt > 0) { + n.reply <-= dirgen(MKPATH(actid, 0, Qconvcontrol)); + cnt--; + i++; + } + if(i == 4 && cnt > 0) { + n.reply <-= dirgen(MKPATH(actid, 0, Qconvdraft)); + cnt--; + i++; + } + if(i == 5 && cnt > 0) { + n.reply <-= dirgen(MKPATH(actid, 0, Qconvdraftstatus)); + cnt--; + i++; + } if(a != nil) { for(; i < total && cnt > 0; i++) { - midx := i - 2; + midx := i - 6; n.reply <-= dirgen(MKPATH(actid, midx, Qconvmsg)); cnt--; } diff --git a/appl/cmd/mkfile b/appl/cmd/mkfile index 88d0cfbd4..8133c7d60 100644 --- a/appl/cmd/mkfile +++ b/appl/cmd/mkfile @@ -187,6 +187,7 @@ TARG=\ vacget.dis\ vacput.dis\ vid9p.dis\ + voicemode.dis\ wav2iaf.dis\ wc.dis\ webfs.dis\ diff --git a/appl/cmd/speechtest.b b/appl/cmd/speechtest.b new file mode 100644 index 000000000..c234d8ba6 --- /dev/null +++ b/appl/cmd/speechtest.b @@ -0,0 +1,477 @@ +implement Speechtest; + +# +# speechtest - exercise the /n/speech STT/TTS surface without an LLM. +# +# Reads streaming transcripts from /listen, prints partials to +# stdout as they arrive, and answers every non-junk final transcript by +# speaking a fixed phrase (or the transcript itself with -e) through +# /say. No LLM, no GUI, no login, no API key: a self-contained +# microphone -> STT -> TTS loop for validating speech providers, audio +# topologies, and helper installs before paying for a model. +# +# If /ctl does not exist and -b is given, speechtest bootstraps +# the standard provider stack in its own namespace: it spawns +# speechshim9p at /n/speechshim and speech9p at , then points +# the provider at the shim and sets duplex half (the same sequence as +# lib/lucifer/boot.sh). -C replays the exact configuration +# produced by tools/install-speech-helpers.sh; -H retains the +# legacy Whisper helper block for explicit test fixtures; -c 'key value' appends +# raw ctl lines (repeatable) — that is how remote-audio topologies are +# selected, and -M 'dialaddr mountpt' (repeatable, unauthenticated) +# mounts a remote 9P export first. See docs/SPEECH-REMOTE-AUDIO.md. +# +# Host-side launcher: tools/speech-test.sh. +# +# The listen wire format and the junk-final filter are kept in sync +# with appl/cmd/voicemode.b (newline-delimited "partial " / +# "final " / "error: " records; bare text is a final). +# + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "arg.m"; + +include "string.m"; + str: String; + +include "sh.m"; + +Speechtest: module +{ + PATH: con "/dis/speechtest.dis"; + init: fn(nil: ref Draw->Context, args: list of string); +}; + +Srv: module +{ + init: fn(nil: ref Draw->Context, args: list of string); +}; + +SHIMPATH: con "/dis/veltro/speechshim9p.dis"; +SPEECH9PPATH: con "/dis/veltro/speech9p.dis"; +SHIMMNT: con "/n/speechshim"; + +stderr: ref Sys->FD; +debug := 0; +speech := "/n/speech"; +phrase := "Speech test complete. I heard you."; +echoback := 0; +turns := 0; +bootstrap := 0; +bootstrapped := 0; + +LISTEN_EMPTY, LISTEN_PARTIAL, LISTEN_FINAL, LISTEN_ERROR: con iota; + +silencefinals := array[] of { + "thank you", + "thanks for watching", + "you", +}; + +log(msg: string) +{ + if(debug) + sys->fprint(stderr, "speechtest: %s\n", msg); +} + +writefile(path, data: string): int +{ + fd := sys->open(path, Sys->OWRITE); + if(fd == nil) + return -1; + b := array of byte data; + return sys->write(fd, b, len b); +} + +readfile(path: string): string +{ + fd := sys->open(path, Sys->OREAD); + if(fd == nil) + return nil; + buf := array[8192] of byte; + n := sys->read(fd, buf, len buf); + if(n <= 0) + return nil; + return string buf[0:n]; +} + +strip(s: string): string +{ + if(s == nil) + return nil; + i := 0; + while(i < len s && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n')) + i++; + j := len s; + while(j > i && (s[j-1] == ' ' || s[j-1] == '\t' || s[j-1] == '\r' || s[j-1] == '\n')) + j--; + if(i >= j) + return ""; + return s[i:j]; +} + +hasprefix(s, prefix: string): int +{ + return len s >= len prefix && s[0:len prefix] == prefix; +} + +finaltext(s: string): string +{ + s = strip(s); + if(s == nil || s == "") + return nil; + if(hasprefix(s, "final ")) + return strip(s[6:]); + if(hasprefix(s, "text ")) + return strip(s[5:]); + if(hasprefix(s, "partial ")) + return nil; + if(hasprefix(s, "error:")) + return nil; + return s; +} + +ispartial(s: string): int +{ + s = strip(s); + return s != nil && hasprefix(s, "partial "); +} + +parselisten(s: string): (int, string) +{ + s = strip(s); + if(s == nil || s == "") + return (LISTEN_EMPTY, nil); + kind := LISTEN_EMPTY; + text := ""; + (nil, lines) := sys->tokenize(s, "\n"); + for(; lines != nil; lines = tl lines) { + line := strip(hd lines); + if(line == "") + continue; + if(hasprefix(line, "error:")) { + if(kind != LISTEN_FINAL) + kind = LISTEN_ERROR; + continue; + } + if(ispartial(line)) { + if(kind != LISTEN_FINAL) { + kind = LISTEN_PARTIAL; + text = strip(line[8:]); + } + continue; + } + t := finaltext(line); + if(t != nil && t != "") { + kind = LISTEN_FINAL; + text = t; + } + } + return (kind, text); +} + +errline(s: string): string +{ + (nil, lines) := sys->tokenize(s, "\n"); + for(; lines != nil; lines = tl lines) { + line := strip(hd lines); + if(hasprefix(line, "error:")) + return line; + } + return "error: unknown listen failure"; +} + +ispunct(c: int): int +{ + return c == '.' || c == ',' || c == '!' || c == '?' || c == ';' || c == ':'; +} + +normalize(text: string): string +{ + text = strip(str->tolower(text)); + if(text == nil || text == "") + return text; + i := 0; + j := len text; + while(i < j && ispunct(text[i])) + i++; + while(j > i && ispunct(text[j-1])) + j--; + if(i >= j) + return ""; + return strip(text[i:j]); +} + +stripbrackets(text: string): string +{ + out := ""; + for(i := 0; i < len text; i++) { + c := text[i]; + if(c == '[' || c == '(') { + close := ']'; + if(c == '(') + close = ')'; + for(j := i + 1; j < len text && text[j] != close; j++) + ; + if(j < len text) { + i = j; + out += " "; + continue; + } + } + out[len out] = c; + } + return strip(out); +} + +junkfinal(text: string): int +{ + text = strip(text); + if(text == nil || text == "") + return 1; + text = stripbrackets(text); + if(text == "") + return 1; + n := normalize(text); + for(i := 0; i < len silencefinals; i++) + if(n == silencefinals[i]) + return 1; + return 0; +} + +chime(kind: string) +{ + writefile(speech + "/chime", kind); +} + +exists(path: string): int +{ + (ok, nil) := sys->stat(path); + return ok == 0; +} + +waitfile(path: string, ms: int): int +{ + for(waited := 0; waited < ms; waited += 100) { + if(exists(path)) + return 1; + sys->sleep(100); + } + return exists(path); +} + +startsrv(dis: string, argv: list of string, ready: string): string +{ + srv := load Srv dis; + if(srv == nil) + return sys->sprint("cannot load %s: %r", dis); + spawn srv->init(nil, argv); + if(!waitfile(ready, 5000)) + return sys->sprint("%s did not serve %s within 5s", dis, ready); + return nil; +} + +ctlwrite(line: string) +{ + if(writefile(speech + "/ctl", line) < 0) + sys->fprint(stderr, "speechtest: ctl write failed: %s: %r\n", line); + else + log("ctl: " + line); +} + +# The standard host-helper configuration, mirroring the ctl block that +# tools/install-speech-helpers.sh prints (listen + TTS only; wake is not +# used here). bindir is a HOST path — helpers run through devcmd. +helperctl(bindir: string) +{ + modeldir := bindir + "/../models"; + if(len bindir > 4 && bindir[len bindir - 4:] == "/bin") + modeldir = bindir[:len bindir - 4] + "/models"; + ctlwrite("kokorobin " + bindir + "/kokoro-cli"); + ctlwrite("whisperstreambin " + bindir + "/whisper-stream-cli"); + ctlwrite("whispermodel " + modeldir + "/ggml-base.en.bin"); + ctlwrite("voice af_bella"); +} + +# Unauthenticated 9P mount for remote-topology tests (a remote provider +# or a remote capture device exported with styxlisten -A on a trusted +# network). spec: "dialaddr mountpt". +domount(spec: string): string +{ + (n, flds) := sys->tokenize(spec, " \t"); + if(n != 2) + return "usage: -M 'dialaddr mountpt'"; + addr := hd flds; + mnt := hd tl flds; + (ok, conn) := sys->dial(addr, nil); + if(ok < 0) + return sys->sprint("dial %s: %r", addr); + sys->create(mnt, Sys->OREAD, Sys->DMDIR | 8r755); + if(sys->mount(conn.dfd, nil, mnt, Sys->MREPL | Sys->MCREATE, "") < 0) + return sys->sprint("mount %s on %s: %r", addr, mnt); + sys->print("speechtest: mounted %s at %s\n", addr, mnt); + return nil; +} + +fatal(msg: string) +{ + sys->fprint(stderr, "speechtest: %s\n", msg); + raise "fail:" + msg; +} + +# Take the servers we spawned (same process group) down with us so a +# bootstrapped headless emu halts instead of idling forever. +killgrp() +{ + pid := sys->pctl(0, nil); + fd := sys->open("/prog/" + string pid + "/ctl", Sys->OWRITE); + if(fd != nil) { + b := array of byte "killgrp"; + sys->write(fd, b, len b); + } +} + +finish(completed: int) +{ + chime("off"); + sys->print("speechtest: %d turn(s) completed\n", completed); + if(bootstrapped) + killgrp(); +} + +rev(l: list of string): list of string +{ + r: list of string; + for(; l != nil; l = tl l) + r = hd l :: r; + return r; +} + +init(nil: ref Draw->Context, args: list of string) +{ + sys = load Sys Sys->PATH; + stderr = sys->fildes(2); + str = load String String->PATH; + arg := load Arg Arg->PATH; + if(str == nil || arg == nil) { + sys->fprint(stderr, "speechtest: cannot load modules: %r\n"); + raise "fail:load"; + } + + arg->init(args); + arg->setusage("speechtest [-bde] [-n turns] [-p phrase] [-s /n/speech] " + + "[-C ctlfile] [-H helperbindir] [-c 'key value'] [-M 'dialaddr mountpt']"); + ctllines: list of string; + mounts: list of string; + helperbin := ""; + ctlfile := ""; + while((c := arg->opt()) != 0) + case c { + 'b' => bootstrap = 1; + 'd' => debug = 1; + 'e' => echoback = 1; + 'n' => turns = int arg->earg(); + 'p' => phrase = arg->earg(); + 's' => speech = arg->earg(); + 'C' => ctlfile = arg->earg(); + 'H' => helperbin = arg->earg(); + 'c' => ctllines = arg->earg() :: ctllines; + 'M' => mounts = arg->earg() :: mounts; + * => arg->usage(); + } + ctllines = rev(ctllines); + mounts = rev(mounts); + + for(m := mounts; m != nil; m = tl m) { + err := domount(hd m); + if(err != nil) + fatal(err); + } + + if(bootstrap && !exists(speech + "/ctl")) { + sys->print("speechtest: starting speech stack (speechshim9p + speech9p)\n"); + err := startsrv(SHIMPATH, "speechshim9p" :: "-m" :: SHIMMNT :: nil, + SHIMMNT + "/ctl"); + if(err == nil) + err = startsrv(SPEECH9PPATH, "speech9p" :: "-m" :: speech :: nil, + speech + "/ctl"); + if(err != nil) + fatal(err); + bootstrapped = 1; + ctlwrite("provider " + SHIMMNT); + ctlwrite("duplex half"); + } + + if(ctlfile != "") { + sh := load Sh Sh->PATH; + if(sh == nil) + fatal(sys->sprint("cannot load sh for %s: %r", ctlfile)); + err := sh->run(nil, "sh" :: ctlfile :: nil); + if(err != nil) + fatal(sys->sprint("speech ctl file failed: %s: %s", ctlfile, err)); + log("ctl file: " + ctlfile); + } + else if(helperbin != "") + helperctl(helperbin); + for(; ctllines != nil; ctllines = tl ctllines) + ctlwrite(hd ctllines); + + saywhat := "\"" + phrase + "\""; + if(echoback) + saywhat = "the transcript back"; + sys->print("speechtest: listening on %s — speak; every final transcript answers with %s\n", + speech, saywhat); + chime("on"); + + completed := 0; + lastpartial := ""; + lastjunk := ""; + for(;;) { + rec := readfile(speech + "/listen"); + (kind, text) := parselisten(rec); + case kind { + LISTEN_EMPTY => + sys->sleep(250); + LISTEN_ERROR => + sys->print("%s\n", errline(rec)); + sys->sleep(1000); + LISTEN_PARTIAL => + if(text != lastpartial) { + sys->print("partial: %s\n", text); + lastpartial = text; + } else + sys->sleep(100); + LISTEN_FINAL => + lastpartial = ""; + if(junkfinal(text)) { + if(text != lastjunk) { + sys->print("final: %s (junk — ignored)\n", text); + lastjunk = text; + } + sys->sleep(250); + } else { + lastjunk = ""; + completed++; + sys->print("final: %s\n", text); + saytext := phrase; + if(echoback) + saytext = text; + sys->print("say: %s\n", saytext); + t0 := sys->millisec(); + if(writefile(speech + "/say", saytext) < 0) + sys->print("say error: %r\n"); + else + sys->print("say done (%d ms)\n", sys->millisec() - t0); + chime("done"); + if(turns > 0 && completed >= turns) { + finish(completed); + return; + } + } + } + } +} diff --git a/appl/cmd/voicemode.b b/appl/cmd/voicemode.b new file mode 100644 index 000000000..a87da9379 --- /dev/null +++ b/appl/cmd/voicemode.b @@ -0,0 +1,1102 @@ +implement Voicemode; + +# +# voicemode - bridge /n/speech wake/listen events into Lucia activity input. +# +# Phase 1 resident daemon. Pre-spawned at boot in an idle state; it activates +# when /mnt/ui/input-mode becomes "v" (the Voice chip click or Esc-V in +# lucifer, lucibridge's "/voice mode on", or a spoken control intent) and +# returns to idle on "k" (the same chip/key, Esc, "/voice mode off", or a +# spoken "keyboard"). The microphone is only open during a voice session: +# helpers start on the first wake/listen read, and exit writes `mic off` to +# the speech ctl so the provider tears them down again. While active it runs +# the Phase 1 state machine: +# +# WAITING_WAKE -> LISTENING -> PROCESSING/SPEAKING -> WAITING_WAKE +# +# Wake is re-armed as soon as a transcript is injected, so a wake event that +# arrives while the assistant is speaking acts as barge-in: /n/speech/cancel +# is written (cutting off TTS) and the machine goes straight to LISTENING. +# +# Mode changes are observed through the /mnt/ui/event global stream when it +# exists; otherwise (mock file trees in tests, older servers) the daemon polls +# /mnt/ui/input-mode. Final transcripts are injected through the privileged +# conversation/voiceinput path so lucibridge accepts them while keyboard input +# is paused. +# +# Test mode (-p phrase, -e): the LLM-free loop for dogfooding the speech +# stack in the GUI without API cost. Finals never reach voiceinput; the +# transcript is posted to the conversation as a "Heard" dialogue line and +# the canned phrase (-p), or the transcript itself (-e), is spoken via +# /n/speech/say. Wake, live partials, chimes, barge-in and control intents +# behave exactly as in normal mode. tools/speech-test.sh --gui boots this. +# + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "arg.m"; + +include "string.m"; + str: String; + +Voicemode: module +{ + init: fn(nil: ref Draw->Context, args: list of string); +}; + +stderr: ref Sys->FD; +debug := 0; +ui := "/mnt/ui"; +speech := "/n/speech"; + +Listenrec: adt { + gen: int; + text: string; +}; + +# Watcher plumbing. Buffered so a watcher finishing a read after the voice +# loop has exited never deadlocks; stale results are drained on re-entry. +evch: chan of string; # "input-mode v|k" and other global events +wakech: chan of string; # one wake read result per request +listench: chan of ref Listenrec; # one listen read result per request +startwake: chan of int; +startlisten: chan of int; +timerch: chan of int; +listenseq := 0; + +listentimeout := 10000; +wakecooldown := 1500; +gracems := 3000; # grace window before a final is submitted; 0 = immediate +confidencethreshold := 650; # thousandths; confidence metadata is optional +pendingconfirm := ""; +busyqueued := 0; # at most one voice follow-up while the activity is busy + +testmode := 0; +echoback := 0; +testphrase := "Speech test complete. I heard you."; + +LISTEN_EMPTY, LISTEN_PARTIAL, LISTEN_FINAL, LISTEN_ERROR: con iota; + +silencefinals := array[] of { + "thank you", + "thanks for watching", + "you", +}; + +usage() +{ + sys->fprint(stderr, "Usage: voicemode [-d] [-e] [-p phrase] [-g grace-ms] [-q confidence-permille] [-t ms] [-w ms] [-u /mnt/ui] [-s /n/speech]\n"); + raise "fail:usage"; +} + +log(msg: string) +{ + if(debug) + sys->fprint(stderr, "voicemode: %s\n", msg); +} + +# Failures are always logged, not just under -d. The daemon is started by +# boot.sh without flags, so a debug-gated error path means a silent stack: no +# log, and nothing to explain why voice mode did nothing. +logerr(msg: string) +{ + sys->fprint(stderr, "voicemode: %s\n", msg); +} + +writefile(path, data: string): int +{ + fd := sys->open(path, Sys->OWRITE); + if(fd == nil) + return -1; + b := array of byte data; + return sys->write(fd, b, len b); +} + +readfile(path: string): string +{ + fd := sys->open(path, Sys->OREAD); + if(fd == nil) + return nil; + buf := array[8192] of byte; + n := sys->read(fd, buf, len buf); + if(n <= 0) + return nil; + return string buf[0:n]; +} + +strip(s: string): string +{ + if(s == nil) + return nil; + i := 0; + while(i < len s && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n')) + i++; + j := len s; + while(j > i && (s[j-1] == ' ' || s[j-1] == '\t' || s[j-1] == '\r' || s[j-1] == '\n')) + j--; + if(i >= j) + return ""; + return s[i:j]; +} + +hasprefix(s, prefix: string): int +{ + return len s >= len prefix && s[0:len prefix] == prefix; +} + +currentactivity(): int +{ + s := strip(readfile(ui + "/activity/current")); + if(s == nil || s == "") + return 0; + return int s; +} + +ctxstatus(actid: int, state: string) +{ + path := sys->sprint("%s/activity/%d/context/ctl", ui, actid); + writefile(path, "resource upsert path=/n/speech label=Voice type=audio status=" + + state + " via=voice-mode"); +} + +ctxqueued(actid: int, full: int) +{ + label := "Voice: queued"; + if(full) + label = "Voice: busy; one turn queued"; + path := sys->sprint("%s/activity/%d/context/ctl", ui, actid); + writefile(path, "resource upsert path=/n/speech label=" + label + + " type=audio status=queued via=voice-mode"); +} + +ctxpartial(actid: int) +{ + path := sys->sprint("%s/activity/%d/context/ctl", ui, actid); + writefile(path, "resource upsert path=/n/speech label=Voice" + + " type=audio status=listening via=voice-mode"); + draftstatus(actid, "Listening..."); +} + +# Grace window: the transcript remains in the pending conversation bubble +# while its separate status line carries the deadline. The Voice resource row +# stays a concise state indicator instead of trying to render user content in +# the narrow context column. +ctxsending(actid: int, remainms: int) +{ + secs := (remainms + 999) / 1000; + path := sys->sprint("%s/activity/%d/context/ctl", ui, actid); + writefile(path, "resource upsert path=/n/speech label=Voice" + + " type=audio status=sending via=voice-mode"); + draftstatus(actid, "Sending in " + string secs + "s - say cancel to stop"); +} + +# Put the failure reason on the Voice chip itself. The conversation notice is +# easy to miss and is not where the user is looking after clicking the chip — +# the chip is, and a bare red "error" does not say what to fix. +ctxerror(actid: int, reason: string) +{ + reason = strip(reason); + if(hasprefix(reason, "error:")) + reason = strip(reason[6:]); + if(reason == nil || reason == "") + reason = "speech helper unavailable"; + for(i := 0; i < len reason; i++) + if(reason[i] == '=' || reason[i] == '\n' || + reason[i] == '\r' || reason[i] == '\t') + reason[i] = ' '; + if(len reason > 48) + reason = reason[0:48]; + path := sys->sprint("%s/activity/%d/context/ctl", ui, actid); + writefile(path, "resource upsert path=/n/speech label=Voice: " + reason + + " type=audio status=error via=voice-mode"); +} + +# Timeout is non-fatal but must be visible: the chip returns to waiting +# with a note saying nothing was heard, so a broken STT path does not +# masquerade as a successful empty utterance. The next wake overwrites it. +ctxtimeout(actid: int) +{ + path := sys->sprint("%s/activity/%d/context/ctl", ui, actid); + writefile(path, "resource upsert path=/n/speech label=Voice: no speech heard" + + " type=audio status=waiting via=voice-mode"); +} + +inputmode(): string +{ + return strip(readfile(ui + "/input-mode")); +} + +setinputmode(mode: string) +{ + writefile(ui + "/input-mode", mode); +} + +voiceinput(actid: int, text: string): int +{ + path := sys->sprint("%s/activity/%d/conversation/voiceinput", ui, actid); + return writefile(path, text); +} + +draftinput(actid: int, text: string): int +{ + path := sys->sprint("%s/activity/%d/conversation/draft", ui, actid); + fd := sys->open(path, Sys->OWRITE | Sys->OTRUNC); + if(fd == nil) + return -1; + b := array of byte text; + return sys->write(fd, b, len b); +} + +draftstatus(actid: int, text: string): int +{ + path := sys->sprint("%s/activity/%d/conversation/draft-status", ui, actid); + fd := sys->open(path, Sys->OWRITE | Sys->OTRUNC); + if(fd == nil) + return -1; + b := array of byte text; + return sys->write(fd, b, len b); +} + +cleardraft(actid: int) +{ + # Clear text first so a stale transcript cannot survive if an older UI + # server does not yet expose draft-status. + draftinput(actid, ""); + draftstatus(actid, ""); +} + +cancelspeech() +{ + writefile(speech + "/cancel", "cancel"); +} + +# Release the microphone on voice-mode exit: the provider kills its +# mic-side helpers, and the next wake/listen read (the next session) +# re-arms them. +micoff() +{ + writefile(speech + "/ctl", "mic off"); +} + +# Stop the STT helper between turns: anything it hears while no turn is +# active (ambient speech, our own TTS) would queue as a stale record and +# replay into the next turn. The next listen read restarts it. +listenoff() +{ + writefile(speech + "/ctl", "listen off"); +} + +chime(kind: string) +{ + writefile(speech + "/chime", kind); +} + +# Parse a listen-stream record into a final transcript, or nil if the record +# is a partial, an error, or empty. Wire format (see appl/veltro/speech9p.b): +# newline-delimited "partial " / "final " records; bare text from +# batch-style helpers is treated as final. +finaltext(s: string): string +{ + s = strip(s); + if(s == nil || s == "") + return nil; + if(hasprefix(s, "final ")) + return recordtext(s[6:]); + if(hasprefix(s, "text ")) + return strip(s[5:]); + if(hasprefix(s, "partial ")) + return nil; + if(hasprefix(s, "error:")) + return nil; + return s; +} + +recordtext(s: string): string +{ + s = strip(s); + if(!hasprefix(s, "confidence=")) + return s; + for(i := 0; i < len s; i++) + if(s[i] == ' ') + return strip(s[i+1:]); + return nil; +} + +recordconfidence(s: string): int +{ + s = strip(s); + if(!hasprefix(s, "confidence=")) + return -1; + i := len "confidence="; + whole := 0; + while(i < len s && s[i] >= '0' && s[i] <= '9') { + whole = whole * 10 + s[i] - '0'; + i++; + } + if(whole >= 1) + return 1000; + if(i >= len s || s[i] != '.') + return 0; + i++; + frac := 0; + n := 0; + while(i < len s && n < 3 && s[i] >= '0' && s[i] <= '9') { + frac = frac * 10 + s[i] - '0'; + i++; + n++; + } + while(n++ < 3) + frac *= 10; + return frac; +} + +ispartial(s: string): int +{ + s = strip(s); + return s != nil && hasprefix(s, "partial "); +} + +parselisten(s: string): (int, string, int) +{ + s = strip(s); + if(s == nil || s == "") + return (LISTEN_EMPTY, nil, -1); + kind := LISTEN_EMPTY; + text := ""; + confidence := -1; + (nil, lines) := sys->tokenize(s, "\n"); + for(; lines != nil; lines = tl lines) { + line := strip(hd lines); + if(line == "") + continue; + if(hasprefix(line, "error:")) { + if(kind != LISTEN_FINAL) + kind = LISTEN_ERROR; + continue; + } + if(ispartial(line)) { + if(kind != LISTEN_FINAL) { + kind = LISTEN_PARTIAL; + confidence = recordconfidence(line[8:]); + text = recordtext(line[8:]); + } + continue; + } + t := finaltext(line); + if(t != nil && t != "") { + kind = LISTEN_FINAL; + if(hasprefix(line, "final ")) + confidence = recordconfidence(line[6:]); + text = t; + } + } + return (kind, text, confidence); +} + +iserror(s: string): int +{ + return s == nil || strip(s) == "" || hasprefix(strip(s), "error:"); +} + +ispunct(c: int): int +{ + return c == '.' || c == ',' || c == '!' || c == '?' || c == ';' || c == ':'; +} + +normalize(text: string): string +{ + text = strip(str->tolower(text)); + if(text == nil || text == "") + return text; + i := 0; + j := len text; + while(i < j && ispunct(text[i])) + i++; + while(j > i && ispunct(text[j-1])) + j--; + if(i >= j) + return ""; + return strip(text[i:j]); +} + +stripbrackets(text: string): string +{ + out := ""; + for(i := 0; i < len text; i++) { + c := text[i]; + if(c == '[' || c == '(') { + close := ']'; + if(c == '(') + close = ')'; + for(j := i + 1; j < len text && text[j] != close; j++) + ; + if(j < len text) { + i = j; + out += " "; + continue; + } + } + out[len out] = c; + } + return strip(out); +} + +junkfinal(text: string): int +{ + text = strip(text); + if(text == nil || text == "") + return 1; + text = stripbrackets(text); + if(text == "") + return 1; + n := normalize(text); + for(i := 0; i < len silencefinals; i++) + if(n == silencefinals[i]) + return 1; + return 0; +} + +# Words that discard the pending transcript during the send grace window. +gracecancel(text: string): int +{ + n := normalize(text); + return n == "cancel" || n == "no" || n == "stop" || n == "wrong" || + n == "never mind" || n == "nevermind" || n == "discard" || + n == "scratch that"; +} + +approvalpending(actid: int): int +{ + path := sys->sprint("%s/activity/%d/status", ui, actid); + return strip(readfile(path)) == "blocked"; +} + +controlinput(actid: int, ctl: string): int +{ + path := sys->sprint("%s/activity/%d/conversation/control", ui, actid); + n := writefile(path, ctl); + if(n < 0) + logerr("cannot write active-turn control " + ctl + " to " + path + ": " + sys->sprint("%r")); + return n; +} + +activitystatus(actid: int): string +{ + return strip(readfile(sys->sprint("%s/activity/%d/status", ui, actid))); +} + +agentbusy(actid: int): int +{ + s := activitystatus(actid); + return s != nil && s != "" && s != "idle" && s != "active" && s != "complete"; +} + +# The say write blocks for the TTS duration on real providers, so test +# mode runs it spawned; barge-in still works because a wake event writes +# /n/speech/cancel, which kills the in-flight synthesis. +saytts(text: string) +{ + writefile(speech + "/say", text); +} + +# Test mode: surface the recognized transcript in the conversation view +# without submitting it as an LLM turn. +noticeheard(actid: int, text: string) +{ + for(i := 0; i < len text; i++) + if(text[i] == '\n' || text[i] == '\r' || text[i] == '\t') + text[i] = ' '; + path := sys->sprint("%s/activity/%d/conversation/ctl", ui, actid); + writefile(path, "role=veltro dtype=dialogue title=Heard text=" + text); +} + +noticeconfirm(actid: int, text: string) +{ + for(i := 0; i < len text; i++) + if(text[i] == '\n' || text[i] == '\r' || text[i] == '\t') + text[i] = ' '; + path := sys->sprint("%s/activity/%d/conversation/ctl", ui, actid); + writefile(path, "role=veltro dtype=dialogue title=Confirm speech text=I heard: " + + text + ". Say yes to continue, or say the correction."); +} + +noticevoiceerror(actid: int, reason: string) +{ + reason = strip(reason); + if(reason == nil || reason == "") + reason = "speech helper unavailable"; + for(i := 0; i < len reason; i++) + if(reason[i] == '\n' || reason[i] == '\r' || reason[i] == '\t') + reason[i] = ' '; + path := sys->sprint("%s/activity/%d/conversation/ctl", ui, actid); + writefile(path, "role=veltro dtype=dialogue title=Voice text=" + reason); +} + +# Spoken control intents that act on the session instead of becoming a chat +# turn. Returns 1 when the utterance was consumed. +handlecontrol(actid: int, text: string): int +{ + lower := normalize(text); + if(lower == "stop" || lower == "cancel") { + cancelspeech(); + controlinput(actid, "cancel"); + ctxstatus(actid, "waiting"); + return 1; + } + if(lower == "pause") { + controlinput(actid, "pause"); + ctxstatus(actid, "paused"); + return 1; + } + if(lower == "resume" || (lower == "continue" && activitystatus(actid) == "paused")) { + controlinput(actid, "resume"); + ctxstatus(actid, "waiting"); + return 1; + } + if(lower == "status" || lower == "repeat status" || lower == "what is happening") { + s := activitystatus(actid); + if(s == nil || s == "") + s = "idle"; + spawn saytts("Current activity is " + s + "."); + return 1; + } + if(lower == "keyboard" || lower == "voice mode off") { + cancelspeech(); + # The input-mode change is observed by the event watcher and + # exits the voice loop; lucifer and lucibridge see the same + # broadcast. + setinputmode("k"); + return 1; + } + if(lower == "approve" || lower == "allow" || + (lower == "yes" && approvalpending(actid))) { + voiceinput(actid, "Allow"); + return 1; + } + if(lower == "deny" || (lower == "no" && approvalpending(actid))) { + voiceinput(actid, "Deny"); + return 1; + } + return 0; +} + +# Global event watcher. Prefers the /mnt/ui/event broadcast stream (persistent +# fd, blocking reads). When the event file is unavailable — mock file trees in +# tests, or a ui server without it — falls back to polling input-mode and +# synthesizing "input-mode " events on change. +eventwatcher() +{ + last := ""; + for(;;) { + fd := sys->open(ui + "/event", Sys->OREAD); + if(fd == nil) { + m := inputmode(); + if(m != nil && m != "" && m != last) { + last = m; + evch <-= "input-mode " + m; + } + sys->sleep(300); + continue; + } + buf := array[1024] of byte; + while((n := sys->read(fd, buf, len buf)) > 0) { + (nil, lines) := sys->tokenize(string buf[0:n], "\n"); + for(; lines != nil; lines = tl lines) { + ev := strip(hd lines); + if(ev != "") + evch <-= ev; + if(hasprefix(ev, "input-mode ")) + last = strip(ev[11:]); + } + } + fd = nil; + # EOF: plain-file mock or server restart; re-open after a beat. + sys->sleep(300); + } +} + +# One blocking speech read per start request. Gated so the microphone-side +# helpers only run while the voice loop has asked for an event. +speechwatcher(file: string, startch: chan of int, ch: chan of string) +{ + for(;;) { + <-startch; + ch <-= readfile(speech + "/" + file); + } +} + +listenwatcher() +{ + for(;;) { + gen := <-startlisten; + listench <-= ref Listenrec(gen, readfile(speech + "/listen")); + } +} + +# Non-blocking start request; a no-op if a request is already queued. +request(startch: chan of int) +{ + alt { + startch <-= 1 => + ; + * => + ; + } +} + +requestlisten(gen: int) +{ + alt { + startlisten <-= gen => + ; + * => + ; + } +} + +timer(ch: chan of int, ms: int, gen: int) +{ + sys->sleep(ms); + alt { + ch <-= gen => + ; + * => + ; + } +} + +# Drop results left over from a previous voice session. +drainresults() +{ + for(;;) { + alt { + <-wakech => + ; + <-listench => + ; + <-timerch => + ; + * => + return; + } + } +} + +WAITING, LISTENING, SENDING: con iota; + +# Submit a completed utterance as the turn (test mode: canned reply, no LLM). +submitfinal(actid: int, text: string) +{ + if(testmode) { + ctxstatus(actid, "processing"); + noticeheard(actid, text); + saytext := testphrase; + if(echoback) + saytext = text; + ctxstatus(actid, "speaking"); + spawn saytts(saytext); + } else { + busy := agentbusy(actid); + if(!busy) + busyqueued = 0; + if(busy && busyqueued) { + log("busy: discarding additional queued voice turn: " + text); + ctxqueued(actid, 1); + chime("done"); + return; + } + ctxstatus(actid, "processing"); + # A follow-up against a busy activity takes conversational + # control at the next safe model/tool boundary, then remains + # queued on voiceinput as the next turn in the same session. + if(busy && !approvalpending(actid)) + controlinput(actid, "refine"); + if(voiceinput(actid, text) < 0) { + ctxstatus(actid, "error"); + return; + } + if(busy) { + busyqueued = 1; + ctxqueued(actid, 0); + } + } +} + +# Active voice session. Runs until input-mode leaves "v". +voiceloop() +{ + log("voice mode on"); + pendingconfirm = ""; + busyqueued = 0; + pendingsend := ""; + drainresults(); + actid := currentactivity(); + state := WAITING; + listengen := 0; + # Grace-window bookkeeping. One timer per SENDING entry (identified by + # sendgen); appended speech only moves senddeadline, and the timer + # re-arms itself for the remainder when it fires early. Spawning a new + # timer per append instead would flood timerch's small buffer with + # stale ticks and the live one gets dropped (timer() sends + # non-blocking so an exited session never leaks a blocked proc). + sendgen := 0; + senddeadline := 0; + lastappend := ""; + lastwake := -wakecooldown; + errorshown := 0; + cleardraft(actid); + ctxstatus(actid, "waiting"); + chime("on"); + request(startwake); + for(;;) { + alt { + ev := <-evch => + if(hasprefix(ev, "input-mode ") && strip(ev[11:]) != "v") { + listenseq++; + listengen = listenseq; + cleardraft(actid); + cancelspeech(); + chime("off"); + micoff(); + busyqueued = 0; + ctxstatus(actid, "idle"); + log("voice mode off"); + return; + } + w := <-wakech => + if(state != WAITING) + continue; + if(iserror(w)) { + logerr("wake: " + strip(w)); + # Report the first failure, not the third: a wake + # helper that cannot start fails identically every + # time, and three silent seconds read as "the button + # did nothing". errorshown keeps it to one notice. + if(!errorshown) { + ctxerror(actid, strip(w)); + noticevoiceerror(actid, strip(w)); + errorshown = 1; + } + sys->sleep(1000); + request(startwake); + continue; + } + errorshown = 0; + now := sys->millisec(); + if(now - lastwake < wakecooldown) { + log("wake debounce: " + strip(w)); + sys->sleep(100); + request(startwake); + continue; + } + lastwake = now; + log("wake: " + strip(w)); + chime("wake"); + # Barge-in: any active TTS is cut off before listening. + cancelspeech(); + actid = currentactivity(); + cleardraft(actid); + state = LISTENING; + listenseq++; + listengen = listenseq; + ctxstatus(actid, "listening"); + spawn timer(timerch, listentimeout, listengen); + requestlisten(listengen); + rec := <-listench => + if(state == SENDING) { + if(rec.gen != listengen) + continue; + (gkind, gtext, nil) := parselisten(rec.text); + if(gkind == LISTEN_EMPTY || junkfinal(gtext) && gkind == LISTEN_FINAL) { + sys->sleep(100); + requestlisten(listengen); + continue; + } + if(gkind == LISTEN_PARTIAL) { + draftstatus(actid, "Listening for more..."); + draftinput(actid, pendingsend + " " + gtext); + sys->sleep(100); + requestlisten(listengen); + continue; + } + if(gkind == LISTEN_ERROR) { + # The pending text still sends when the grace + # timer fires; only the barge-in ear is lost. + logerr("listen during grace: " + strip(rec.text)); + continue; + } + if(gracecancel(gtext)) { + log("grace cancel: " + gtext); + pendingsend = ""; + listenseq++; + listengen = listenseq; + state = WAITING; + listenoff(); + cleardraft(actid); + ctxstatus(actid, "waiting"); + spawn saytts("Cancelled."); + chime("done"); + sys->sleep(100); + request(startwake); + continue; + } + # The whisper wrapper's sliding window re-emits the + # same utterance after a pause; appending it would + # turn one sentence into three. (Parakeet emits each + # final once, so this only guards the fallback.) + if(normalize(gtext) == lastappend) { + sys->sleep(100); + requestlisten(listengen); + continue; + } + # More speech: the turn was not over. Append and + # push the deadline out; the running tick timer + # covers the remainder. + log("grace append: " + gtext); + pendingsend += " " + gtext; + lastappend = normalize(gtext); + listenseq++; + listengen = listenseq; + senddeadline = sys->millisec() + gracems; + ctxsending(actid, gracems); + draftinput(actid, pendingsend); + requestlisten(listengen); + continue; + } + if(state != LISTENING) + continue; + if(rec.gen != listengen) + continue; + (kind, text, confidence) := parselisten(rec.text); + if(kind == LISTEN_EMPTY || kind == LISTEN_PARTIAL) { + errorshown = 0; + if(kind == LISTEN_PARTIAL) { + ctxpartial(actid); + draftinput(actid, text); + } + sys->sleep(100); + requestlisten(listengen); + continue; + } + if(kind == LISTEN_ERROR) { + logerr("listen: " + strip(rec.text)); + listenseq++; + listengen = listenseq; + state = WAITING; + listenoff(); + cleardraft(actid); + if(!errorshown) { + ctxerror(actid, strip(rec.text)); + noticevoiceerror(actid, strip(rec.text)); + errorshown = 1; + } else + ctxstatus(actid, "waiting"); + sys->sleep(100); + request(startwake); + continue; + } + listenseq++; + listengen = listenseq; + errorshown = 0; + if(junkfinal(text)) { + state = WAITING; + listenoff(); + cleardraft(actid); + log("listen junk: " + text); + ctxstatus(actid, "waiting"); + chime("done"); + sys->sleep(100); + request(startwake); + continue; + } + log("transcript: " + text); + if(handlecontrol(actid, text)) { + state = WAITING; + listenoff(); + cleardraft(actid); + pendingconfirm = ""; + ctxstatus(actid, "waiting"); + chime("done"); + sys->sleep(100); + request(startwake); + continue; + } + confirmed := 0; + if(pendingconfirm != "") { + answer := normalize(text); + if(answer == "yes" || answer == "confirm" || answer == "correct" || + answer == "do it") { + text = pendingconfirm; + pendingconfirm = ""; + confidence = 1000; + # An explicit yes skips the grace window: the + # user already vetted this exact text. + confirmed = 1; + } else if(answer == "no" || answer == "wrong" || answer == "cancel") { + pendingconfirm = ""; + state = WAITING; + listenoff(); + cleardraft(actid); + ctxstatus(actid, "waiting"); + spawn saytts("Okay. Please say it again."); + chime("done"); + sys->sleep(100); + request(startwake); + continue; + } else { + # Treat any other answer as a spoken correction and apply its + # own confidence rather than submitting the old interpretation. + pendingconfirm = ""; + } + } + if(confidence >= 0 && confidence < confidencethreshold) { + state = WAITING; + listenoff(); + cleardraft(actid); + pendingconfirm = text; + ctxstatus(actid, "confirming"); + noticeconfirm(actid, text); + spawn saytts("I heard " + text + ". Is that right?"); + chime("done"); + sys->sleep(100); + request(startwake); + continue; + } + if(gracems > 0 && !confirmed) { + # Grace window: show the transcript (chip + compose + # draft) and keep listening. "cancel" discards it, + # more speech appends to it, the timer submits it. + # The listen helper stays armed so no restart + # latency lands inside the window. + pendingsend = text; + lastappend = normalize(text); + state = SENDING; + sendgen = listengen; + senddeadline = sys->millisec() + gracems; + ctxsending(actid, gracems); + draftinput(actid, pendingsend); + chime("done"); + # Tick timer: fires every ≤500ms to refresh the + # countdown, re-arming until the deadline (which + # appended speech may keep moving) passes. + tick := gracems; + if(tick > 500) + tick = 500; + spawn timer(timerch, tick, sendgen); + requestlisten(listengen); + continue; + } + state = WAITING; + listenoff(); + cleardraft(actid); + submitfinal(actid, text); + # Re-arm immediately: a wake during the spoken response is + # barge-in. The pacing sleep keeps mock file trees (always- + # ready reads) from spinning. + sys->sleep(100); + request(startwake); + gen := <-timerch => + if(state == LISTENING && gen == listengen) { + # Always logged: a timeout with no transcript is the + # signature of a broken STT path, and must not be + # indistinguishable from a completed empty turn. + logerr("listen timeout: no transcript received"); + listenseq++; + listengen = listenseq; + state = WAITING; + listenoff(); + cleardraft(actid); + ctxtimeout(actid); + chime("done"); + request(startwake); + } else if(state == SENDING && gen == sendgen) { + now := sys->millisec(); + if(now < senddeadline) { + # Deadline not reached (or moved by appended + # speech): refresh the countdown and re-arm. + wait := senddeadline - now; + if(wait > 500) + wait = 500; + ctxsending(actid, senddeadline - now); + spawn timer(timerch, wait, sendgen); + continue; + } + # Grace window elapsed with no cancel: submit. + sendtext := pendingsend; + pendingsend = ""; + listenseq++; + listengen = listenseq; + state = WAITING; + listenoff(); + cleardraft(actid); + submitfinal(actid, sendtext); + sys->sleep(100); + request(startwake); + } + } + } +} + +init(nil: ref Draw->Context, args: list of string) +{ + sys = load Sys Sys->PATH; + stderr = sys->fildes(2); + + str = load String String->PATH; + if(str == nil) { + sys->fprint(stderr, "voicemode: cannot load string: %r\n"); + raise "fail:load"; + } + + arg := load Arg Arg->PATH; + if(arg == nil) { + sys->fprint(stderr, "voicemode: cannot load arg: %r\n"); + raise "fail:load"; + } + arg->init(args); + while((o := arg->opt()) != 0) + case o { + 'd' => debug = 1; + 'e' => echoback = 1; + testmode = 1; + 'p' => testphrase = arg->earg(); + testmode = 1; + 'g' => gracems = int arg->earg(); + 'q' => confidencethreshold = int arg->earg(); + 't' => listentimeout = int arg->earg(); + 'w' => wakecooldown = int arg->earg(); + 'u' => ui = arg->earg(); + 's' => speech = arg->earg(); + * => usage(); + } + if(confidencethreshold < 0 || confidencethreshold > 1000) + usage(); + if(gracems < 0) + usage(); + + evch = chan[8] of string; + wakech = chan[2] of string; + listench = chan[2] of ref Listenrec; + startwake = chan[1] of int; + startlisten = chan[1] of int; + timerch = chan[2] of int; + + spawn eventwatcher(); + spawn speechwatcher("wake", startwake, wakech); + spawn listenwatcher(); + + # Resident loop: idle until input-mode becomes "v". The startup check + # catches a daemon (re)started while voice mode is already on. + for(;;) { + if(inputmode() == "v") + voiceloop(); + else { + ev := <-evch; + if(!(hasprefix(ev, "input-mode ") && strip(ev[11:]) == "v")) + continue; + } + } +} diff --git a/appl/veltro/mkfile b/appl/veltro/mkfile index 66470c9fc..ee305f663 100644 --- a/appl/veltro/mkfile +++ b/appl/veltro/mkfile @@ -15,6 +15,8 @@ MAIN=\ subagent.dis\ mc9p.dis\ speech9p.dis\ + speechprovider.dis\ + speechshim9p.dis\ msg9p.dis\ msgwatch.dis\ ninepsrc.dis\ @@ -98,6 +100,12 @@ mc9p.dis: mc9p.b speech9p.dis: speech9p.b limbo $LIMBOFLAGS -gw speech9p.b +speechprovider.dis: speechprovider.b + limbo $LIMBOFLAGS -gw speechprovider.b + +speechshim9p.dis: speechshim9p.b + limbo $LIMBOFLAGS -gw speechshim9p.b + msg9p.dis: msg9p.b limbo $LIMBOFLAGS -gw msg9p.b @@ -265,7 +273,7 @@ tools/wallet.dis: tools/wallet.b tool.m install:V: all mkdir -p $DISBIN $DISBIN/tools $DISBIN/sources - cp agentlib.dis veltro.dis repl.dis tools9p.dis nsconstruct.dis cowfs.dis subagent.dis mc9p.dis speech9p.dis msg9p.dis msgwatch.dis ninepsrc.dis wiki9p.dis $DISBIN/ + cp agentlib.dis veltro.dis repl.dis tools9p.dis nsconstruct.dis cowfs.dis subagent.dis mc9p.dis speech9p.dis speechprovider.dis speechshim9p.dis msg9p.dis msgwatch.dis ninepsrc.dis wiki9p.dis $DISBIN/ cp tools/*.dis $DISBIN/tools/ cp sources/*.dis $DISBIN/sources/ diff --git a/appl/veltro/speech9p.b b/appl/veltro/speech9p.b index 43b6b7f75..415a80190 100644 --- a/appl/veltro/speech9p.b +++ b/appl/veltro/speech9p.b @@ -55,12 +55,14 @@ include "styxservers.m"; include "string.m"; str: String; +include "speech.m"; + Speech9p: module { init: fn(nil: ref Draw->Context, args: list of string); }; # Qid layout for synthetic files -Qroot, Qctl, Qsay, Qhear, Qvoices: con iota; +Qroot, Qctl, Qsay, Qhear, Qvoices, Qlisten, Qwake, Qsayq, Qcancel, Qchime: con iota; # Per-fid state for say and hear operations FidState: adt { @@ -75,6 +77,8 @@ FidState: adt { ENGINE_CMD: con 0; # Host OS commands via #C (devcmd) ENGINE_API: con 1; # HTTP API (OpenAI, etc.) ENGINE_LOCAL: con 2; # Local ML models (Piper TTS, whisper.cpp STT) +ENGINE_KOKORO: con 3; # Kokoro TTS + streaming helper STT/wake +ENGINE_MODULE: con 4; # Dynamically loaded SpeechEngine `.dis` module # Current configuration engine := ENGINE_CMD; @@ -86,6 +90,8 @@ apikey := ""; audrate := 22050; audchans := 1; audbits := 16; +enginepath := ""; +engineplugin: SpeechEngine; # Platform-specific defaults for cmd engine cmdtts := ""; # Set in initplatform() @@ -98,11 +104,57 @@ pipermodel := ""; # Path to .onnx voice model whisperbin := "whisper-cli"; whispermodel := ""; # Path to .bin GGML model +# Voice-mode speech provider. speech9p does not run streaming helpers +# itself: wake, streaming listen, and voice-mode TTS are consumed from a +# provider mount serving the contract documented in +# docs/SPEECH-ARCHITECTURE.md — listen/wake/say/cancel/chime plus optional +# ctl/voices. speechshim9p adapts external helper CLIs (whisper-stream, +# kokoro, openwakeword) to that contract; a parakeet export or a remote +# 9P-mounted provider serves the same shape. The parakeet*/pipersay ctl +# keys below are compatibility aliases into the same state. +providermount := "/n/parakeet"; +providerlisten := "/n/parakeet/listen"; +providersay := "/n/parakeet/say"; +providerwake := "/n/parakeet/wake"; +providerlistenfd: ref Sys->FD; +providerwakefd: ref Sys->FD; + +# Helper configuration retained for introspection and forwarded to the +# provider's ctl when one is mounted (speechshim9p consumes these keys). +kokorobin := "kokoro-cli"; +ttsengine := "engine"; +listenengine := "whisper"; +whisperstreambin := "whisper-stream"; +wakebin := "openwakeword-cli"; +wakeword := "hey lucia"; +wakethreshold := "0.5"; +cancelreq := 0; + stderr: ref Sys->FD; user: string; mountpt := "/n/speech"; fidstates: list of ref FidState; +# Async helper completion. listen/wake/hear reads run external helpers that +# can block indefinitely (a wake read blocks until the wake word is spoken). +# Running them inline would freeze the serveloop and with it every other 9P +# request — including the /n/speech/cancel write that barge-in depends on. +# Instead the read is parked on asyncpending, the helper runs in a spawned +# proc, and its completion is delivered to the serveloop through helperc. +# Flush and Clunk remove pending entries; completions whose entry is gone +# are dropped. +Helperdone: adt { + kind: int; # Qlisten, Qwake, Qhear, Qsay, Qsayq + fid: int; + m: ref Tmsg.Read; # parked request; reply is built from it + result: array of byte; +}; +helperc: chan of ref Helperdone; +asyncpending: list of (int, int); # (tag, fid) of reads awaiting a helper +listenbusy := 0; +wakebusy := 0; +hearbusy := 0; + nomod(s: string) { sys->fprint(stderr, "speech9p: can't load %s: %r\n", s); @@ -111,10 +163,11 @@ nomod(s: string) usage() { - sys->fprint(stderr, "Usage: speech9p [-D] [-m mountpoint] [-e engine] [-k apikey]\n"); + sys->fprint(stderr, "Usage: speech9p [-D] [-m mountpoint] [-e engine] [-E module.dis] [-k apikey]\n"); sys->fprint(stderr, " -D Enable 9P debug tracing\n"); sys->fprint(stderr, " -m mountpoint Mount point (default: /n/speech)\n"); - sys->fprint(stderr, " -e engine Engine: cmd (default), api, local\n"); + sys->fprint(stderr, " -e engine Engine: cmd (default), api, local, kokoro\n"); + sys->fprint(stderr, " -E module.dis Load a SpeechEngine module and select it\n"); sys->fprint(stderr, " -k key API key (for api engine)\n"); sys->fprint(stderr, " -u url API base URL\n"); sys->fprint(stderr, " -v voice Default voice\n"); @@ -157,11 +210,13 @@ init(nil: ref Draw->Context, args: list of string) "cmd" => engine = ENGINE_CMD; "api" => engine = ENGINE_API; "local" => engine = ENGINE_LOCAL; + "kokoro" => engine = ENGINE_KOKORO; * => sys->fprint(stderr, "speech9p: unknown engine '%s'\n", e); usage(); } engineexplicit = 1; + 'E' => enginepath = arg->earg(); 'k' => apikey = arg->earg(); 'u' => apiurl = arg->earg(); 'v' => @@ -178,6 +233,14 @@ init(nil: ref Draw->Context, args: list of string) # Detect platform and set defaults initplatform(); + if(enginepath != "") { + path := enginepath; + err := loadengine(path); + if(err != nil && err != "") { + sys->fprint(stderr, "speech9p: %s\n", err); + raise "fail:engine"; + } + } sys->pctl(Sys->FORKFD, nil); @@ -191,6 +254,8 @@ init(nil: ref Draw->Context, args: list of string) raise "fail:pipe"; } + helperc = chan of ref Helperdone; + navops := chan of ref Navop; spawn navigator(navops); @@ -225,8 +290,12 @@ initplatform() cmdtts = "say"; if(cmdstt == "") cmdstt = "whisper-cli"; - if(voice == "") - voice = "samantha"; + if(voice == "") { + if(engine == ENGINE_KOKORO) + voice = "af_bella"; + else + voice = "samantha"; + } "linux" => # Linux: prefer local ML engine if Piper/whisper are available # On Jetson (ARM64 + GPU), these get CUDA acceleration automatically @@ -277,6 +346,39 @@ detectplatform(): string return "unknown"; } +speechconfig(): ref Speech->Config +{ + infmt := ref Speech->AudioFmt(audrate, audchans, audbits, "pcm"); + outfmt := ref Speech->AudioFmt(audrate, audchans, audbits, "pcm"); + return ref Speech->Config("module", voice, lang, apiurl, apikey, + cmdtts, cmdstt, providermount, infmt, outfmt); +} + +configureplugin(): string +{ + if(engineplugin == nil) + return "speech engine module is not loaded"; + return engineplugin->configure(speechconfig()); +} + +loadengine(path: string): string +{ + m := load SpeechEngine path; + if(m == nil) + return sys->sprint("cannot load speech engine %s: %r", path); + err := m->init(); + if(err != nil && err != "") + return err; + err = m->configure(speechconfig()); + if(err != nil && err != "") + return err; + engineplugin = m; + enginepath = path; + engine = ENGINE_MODULE; + engineexplicit = 1; + return nil; +} + # Read current config as text readconfig(): string { @@ -285,6 +387,10 @@ readconfig(): string ename = "api"; else if(engine == ENGINE_LOCAL) ename = "local"; + else if(engine == ENGINE_KOKORO) + ename = "kokoro"; + else if(engine == ENGINE_MODULE) + ename = "module"; result := "engine " + ename + "\n"; result += "voice " + voice + "\n"; @@ -292,6 +398,9 @@ readconfig(): string result += "rate " + string audrate + "\n"; result += "chans " + string audchans + "\n"; result += "bits " + string audbits + "\n"; + result += "module " + enginepath + "\n"; + if(engineplugin != nil) + result += "modulename " + engineplugin->name() + "\n"; if(engine == ENGINE_CMD) { result += "cmdtts " + cmdtts + "\n"; @@ -313,6 +422,18 @@ readconfig(): string result += "whispermodel " + whispermodel + "\n"; } + result += "kokorobin " + kokorobin + "\n"; + result += "ttsengine " + ttsengine + "\n"; + result += "listenengine " + listenengine + "\n"; + result += "whisperstreambin " + whisperstreambin + "\n"; + result += "wakebin " + wakebin + "\n"; + result += "wakeword " + wakeword + "\n"; + result += "wakethreshold " + wakethreshold + "\n"; + result += "provider " + providermount + "\n"; + result += "parakeetmount " + providermount + "\n"; + result += "parakeetlisten " + providerlisten + "\n"; + result += "pipersay " + providersay + "\n"; + return result; } @@ -336,7 +457,21 @@ applyconfig(cmd: string): string case key { "engine" => - return "error: engine is startup-only"; + case val { + "cmd" => engine = ENGINE_CMD; + "api" => engine = ENGINE_API; + "local" => engine = ENGINE_LOCAL; + "kokoro" => engine = ENGINE_KOKORO; + "module" => + if(engineplugin == nil) + return "error: load a module first with: module /path/engine.dis"; + engine = ENGINE_MODULE; + * => return "error: unknown engine: " + val; + } + "module" => + err := loadengine(val); + if(err != nil && err != "") + return "error: " + err; "voice" => if(!safename(val)) return "error: unsafe voice"; @@ -375,14 +510,79 @@ applyconfig(cmd: string): string "whisperbin" => return "error: whisperbin is startup-only"; "whispermodel" => - return "error: whispermodel is startup-only"; + whispermodel = val; + forwardprovider(key, val); + "kokorobin" => + kokorobin = val; + forwardprovider(key, val); + "ttsengine" => + case val { + "engine" or "piper" => + ttsengine = val; + * => + return "error: unknown ttsengine: " + val; + } + "listenengine" => + case val { + "whisper" or "parakeet" => + if(listenengine != val) + resetprovider(); + listenengine = val; + * => + return "error: unknown listenengine: " + val; + } + "whisperstreambin" => + whisperstreambin = val; + forwardprovider(key, val); + "wakebin" => + wakebin = val; + forwardprovider(key, val); + "wakeword" => + wakeword = val; + forwardprovider(key, val); + "wakethreshold" => + wakethreshold = val; + forwardprovider(key, val); + "audiodev" or "capturedev" or "micmode" or "capturerate" or "duplex" or "mic" or "listen" => + # Audio routing lives in the provider (docs/SPEECH-REMOTE-AUDIO.md); + # speech9p only passes the knobs through. `mic off` is written by + # voicemode on voice-mode exit so the provider releases the + # microphone; the next listen/wake read re-arms it. `listen off` + # is written at the end of each voice turn so the STT helper is + # not transcribing between turns. + forwardprovider(key, val); + "provider" or "parakeetmount" => + resetprovider(); + providermount = val; + providerlisten = val + "/listen"; + providersay = val + "/say"; + providerwake = val + "/wake"; + "parakeetlisten" => + resetprovider(); + providerlisten = val; + "pipersay" => + providersay = val; * => return "error: unknown config key: " + key; } + if(engine == ENGINE_MODULE) { + err := configureplugin(); + if(err != nil && err != "") + return "error: " + err; + } return "ok"; } +# Best-effort write-through of helper configuration to the mounted +# provider's ctl (speechshim9p consumes these keys; other providers may +# ignore them). +forwardprovider(key, val: string) +{ + if(providermount != "") + writemounted(providermount + "/ctl", key + " " + val + "\n"); +} + safename(s: string): int { if(s == nil || s == "" || s == "." || s == "..") @@ -407,6 +607,15 @@ listvoices(): string return listapivoices(); ENGINE_LOCAL => return listlocalvoices(); + ENGINE_KOKORO => + return listkokorovoices(); + ENGINE_MODULE => + if(engineplugin == nil || !(engineplugin->caps() & Speech->CAPTTS)) + return "(loaded module has no TTS capability)\n"; + result := ""; + for(vs := engineplugin->voices(); vs != nil; vs = tl vs) + result += hd vs + "\n"; + return result; } return ""; } @@ -460,6 +669,8 @@ dosay(text: string): string { if(text == "") return "error: no text to speak"; + if(ttsengine == "piper") + return sayprovider(text); case engine { ENGINE_CMD => @@ -468,10 +679,40 @@ dosay(text: string): string return sayapi(text); ENGINE_LOCAL => return saylocal(text); + ENGINE_KOKORO => + return sayprovider(text); + ENGINE_MODULE => + if(engineplugin == nil || !(engineplugin->caps() & Speech->CAPTTS)) + return "error: loaded module has no TTS capability"; + r := engineplugin->synthesize(text); + if(r == nil) + return "error: speech engine returned no TTS result"; + if(r.err != nil && r.err != "") + return "error: " + r.err; + if(r.audio != nil && len r.audio > 0) + return playpcm(r.audio); + return "ok"; } return "error: no engine configured"; } +# Delegate TTS to the provider's say file (speechshim9p runs Kokoro; a +# parakeet export runs Piper; a remote provider does whatever it likes). +sayprovider(text: string): string +{ + if(providersay == "") + return "error: speech provider say mount not configured"; + if(writemounted(providersay, text + "\n") < 0) + return "error: speech provider say unavailable: " + providersay; + result := readmounted(providersay); + if(result == nil) + return "ok"; + result = strip(result); + if(result == "") + return "ok"; + return result; +} + # TTS via host command (platform-specific) saycmd(text: string): string { @@ -544,6 +785,80 @@ sayapi(text: string): string return playpcm(audiodata); } +# List voices from the provider's voices file when it serves one. +listkokorovoices(): string +{ + if(providermount == "") + return "(no speech provider configured)\n"; + result := readmounted(providermount + "/voices"); + if(result == nil || strip(result) == "" || hasprefix(strip(result), "error:")) + return "af_bella\n(default; provider voices unavailable)\n"; + return result; +} + +# Streaming listen. The provider owns the persistent microphone/STT process; +# speech9p only reads its stream file. Records are newline-delimited +# "partial ..." / "final ..." lines (see module/speech.m Partial); speech9p +# does not interpret them — voicemode consumes the stream. +dolisten(): string +{ + if(providerlisten == "") + return "error: listen provider not configured"; + if(providerlistenfd == nil) { + providerlistenfd = sys->open(providerlisten, Sys->OREAD); + if(providerlistenfd == nil) + return "error: listen provider unavailable: " + providerlisten; + } + result := readmountedfd(providerlistenfd); + if(result == nil) { + providerlistenfd = nil; + return "error: listen provider unavailable: " + providerlisten; + } + if(strip(result) == "") + return "error: listen provider produced no transcript"; + return result; +} + +resetprovider() +{ + providerlistenfd = nil; + providerwakefd = nil; +} + +cancelprovider() +{ + if(providermount != "") + writemounted(providermount + "/cancel", "cancel\n"); + resetprovider(); +} + +chimeprovider(kind: string) +{ + if(providermount != "") + writemounted(providermount + "/chime", strip(kind) + "\n"); +} + +# Wake. The provider blocks the read until its wake-word engine fires, then +# returns one event line containing model/score. +dowake(): string +{ + if(providerwake == "") + return "error: wake provider not configured"; + if(providerwakefd == nil) { + providerwakefd = sys->open(providerwake, Sys->OREAD); + if(providerwakefd == nil) + return "error: wake provider unavailable: " + providerwake; + } + result := readmountedfd(providerwakefd); + if(result == nil) { + providerwakefd = nil; + return "error: wake provider unavailable: " + providerwake; + } + if(strip(result) == "") + return "error: wake provider produced no event"; + return result; +} + # === STT: Speech to Text === # Record from /dev/audio and transcribe @@ -556,10 +871,43 @@ dohear(): string return hearapi(); ENGINE_LOCAL => return hearlocal(); + ENGINE_KOKORO => + return dolisten(); + ENGINE_MODULE => + if(engineplugin == nil || !(engineplugin->caps() & Speech->CAPSTT)) + return "error: loaded module has no STT capability"; + fmt := ref Speech->AudioFmt(audrate, audchans, audbits, "pcm"); + audio := capturepcm(hearduration); + if(audio == nil) + return "error: recording failed"; + r := engineplugin->recognize(audio, fmt); + if(r == nil) + return "error: speech engine returned no STT result"; + if(r.err != nil && r.err != "") + return "error: " + r.err; + return r.text; } return "error: no engine configured"; } +capturepcm(duration: int): array of byte +{ + configaudio("in"); + fd := sys->open("/dev/audio", Sys->OREAD); + if(fd == nil) + return nil; + total := (audrate * audchans * (audbits / 8) * duration) / 1000; + data := array[total] of byte; + nread := 0; + while(nread < total) { + n := sys->read(fd, data[nread:], total - nread); + if(n <= 0) + break; + nread += n; + } + return data[0:nread]; +} + # STT via host commands (record + transcribe, all host-side) hearcmd(): string { @@ -1413,6 +1761,32 @@ hasprefix(s, prefix: string): int return len s >= len prefix && s[0:len prefix] == prefix; } +readmounted(path: string): string +{ + fd := sys->open(path, Sys->OREAD); + if(fd == nil) + return nil; + return readmountedfd(fd); +} + +readmountedfd(fd: ref Sys->FD): string +{ + buf := array[8192] of byte; + n := sys->read(fd, buf, len buf); + if(n < 0) + return nil; + return string buf[0:n]; +} + +writemounted(path, data: string): int +{ + fd := sys->open(path, Sys->OWRITE); + if(fd == nil) + return -1; + b := array of byte data; + return sys->write(fd, b, len b); +} + strip(s: string): string { i := 0; @@ -1426,6 +1800,100 @@ strip(s: string): string return s[i:j]; } +# === Async helper reads === + +addasync(tag, fid: int) +{ + asyncpending = (tag, fid) :: asyncpending; +} + +isasync(tag: int): int +{ + for(l := asyncpending; l != nil; l = tl l) { + (t, nil) := hd l; + if(t == tag) + return 1; + } + return 0; +} + +cancelasynctag(tag: int) +{ + newlist: list of (int, int); + for(l := asyncpending; l != nil; l = tl l) { + (t, nil) := hd l; + if(t != tag) + newlist = hd l :: newlist; + } + asyncpending = newlist; +} + +cancelasyncfid(fid: int) +{ + newlist: list of (int, int); + for(l := asyncpending; l != nil; l = tl l) { + (nil, f) := hd l; + if(f != fid) + newlist = hd l :: newlist; + } + asyncpending = newlist; +} + +asynclisten(donec: chan of ref Helperdone, m: ref Tmsg.Read) +{ + donec <-= ref Helperdone(Qlisten, m.fid, m, array of byte dolisten()); +} + +asyncwake(donec: chan of ref Helperdone, m: ref Tmsg.Read) +{ + donec <-= ref Helperdone(Qwake, m.fid, m, array of byte dowake()); +} + +asynchear(donec: chan of ref Helperdone, m: ref Tmsg.Read) +{ + donec <-= ref Helperdone(Qhear, m.fid, m, array of byte dohear()); +} + +# Forward an in-flight TTS completion (say/sayq) to the serveloop so a +# result read does not block other 9P traffic while playback finishes. +saywait(kind: int, donec: chan of ref Helperdone, m: ref Tmsg.Read, ch: chan of array of byte) +{ + donec <-= ref Helperdone(kind, m.fid, m, <-ch); +} + +# Serveloop-side completion: drop if the read was flushed or clunked, +# otherwise cache per-fid results and reply. +asyncdone(srv: ref Styxserver, h: ref Helperdone) +{ + case h.kind { + Qlisten => listenbusy = 0; + Qwake => wakebusy = 0; + Qhear => hearbusy = 0; + } + if(!isasync(h.m.tag)) + return; + cancelasynctag(h.m.tag); + fs := getfidstate(h.fid); + case h.kind { + Qhear => + fs.hearresp = h.result; + Qsay or Qsayq => + fs.sayresp = h.result; + } + if(h.kind == Qlisten || h.kind == Qwake) { + # Streaming replies must ignore the fid's read offset: clients + # hold one fd across many reads and offset-sliced replies would + # EOF the stream after the first read (INFR-28). Reply raw, + # clamped to the requested count. + data := h.result; + if(len data > h.m.count) + data = data[0:h.m.count]; + srv.reply(ref Rmsg.Read(h.m.tag, data)); + return; + } + srv.reply(styxservers->readbytes(h.m, h.result)); +} + # === Per-fid state management === getfidstate(fid: int): ref FidState @@ -1484,6 +1952,21 @@ walkto(n: ref Navop.Walk) "voices" => n.path = big Qvoices; n.reply <-= dirgen(int n.path); + "listen" => + n.path = big Qlisten; + n.reply <-= dirgen(int n.path); + "wake" => + n.path = big Qwake; + n.reply <-= dirgen(int n.path); + "sayq" => + n.path = big Qsayq; + n.reply <-= dirgen(int n.path); + "cancel" => + n.path = big Qcancel; + n.reply <-= dirgen(int n.path); + "chime" => + n.path = big Qchime; + n.reply <-= dirgen(int n.path); * => n.reply <-= (nil, Enotfound); } @@ -1518,6 +2001,21 @@ dirgen(path: int): (ref Sys->Dir, string) Qvoices => d.name = "voices"; d.mode = 8r444; + Qlisten => + d.name = "listen"; + d.mode = 8r444; + Qwake => + d.name = "wake"; + d.mode = 8r444; + Qsayq => + d.name = "sayq"; + d.mode = 8r666; + Qcancel => + d.name = "cancel"; + d.mode = 8r666; + Qchime => + d.name = "chime"; + d.mode = 8r222; * => return (nil, Enotfound); } @@ -1530,7 +2028,7 @@ readdir(n: ref Navop.Readdir, path: int) { case path { Qroot => - entries := array[] of {Qctl, Qsay, Qhear, Qvoices}; + entries := array[] of {Qctl, Qsay, Qhear, Qvoices, Qlisten, Qwake, Qsayq, Qcancel, Qchime}; for(i := 0; i < len entries; i++) { if(i >= n.offset) { (d, err) := dirgen(entries[i]); @@ -1550,7 +2048,14 @@ serveloop(tchan: chan of ref Tmsg, srv: ref Styxserver, pidc: chan of int, navop Serve: for(;;) { - gm := <-tchan; + gm: ref Tmsg; + alt { + gm = <-tchan => + ; + h := <-helperc => + asyncdone(srv, h); + continue; + } if(gm == nil) break Serve; @@ -1558,6 +2063,10 @@ Serve: Readerror => break Serve; + Flush => + cancelasynctag(m.oldtag); + srv.reply(ref Rmsg.Flush(m.tag)); + Read => fid := srv.getfid(m.fid); if(fid == nil) { @@ -1571,25 +2080,70 @@ Serve: srv.reply(styxservers->readstr(m, readconfig())); Qsay => fs := getfidstate(m.fid); - # If async TTS is pending, wait for completion + # If async TTS is pending, park the read and let a + # spawned waiter forward the completion. if(fs.sayresp == nil && fs.saydone != nil) { - fs.sayresp = <-fs.saydone; + ch := fs.saydone; fs.saydone = nil; - } - if(fs.sayresp != nil) + addasync(m.tag, m.fid); + spawn saywait(Qsay, helperc, m, ch); + } else if(fs.sayresp != nil) srv.reply(styxservers->readbytes(m, fs.sayresp)); else srv.reply(styxservers->readstr(m, "")); Qhear => - # Trigger listening and return transcription + # Trigger listening and return transcription. + # Recording takes seconds; run it async. fs := getfidstate(m.fid); - if(fs.hearresp == nil) { - text := dohear(); - fs.hearresp = array of byte text; + if(fs.hearresp != nil) + srv.reply(styxservers->readbytes(m, fs.hearresp)); + else if(hearbusy) + srv.reply(styxservers->readstr(m, "error: hear busy")); + else { + hearbusy = 1; + addasync(m.tag, m.fid); + spawn asynchear(helperc, m); } - srv.reply(styxservers->readbytes(m, fs.hearresp)); Qvoices => srv.reply(styxservers->readstr(m, listvoices())); + Qlisten => + # Blocks until the helper produces transcript records; + # must not block the serveloop (cancel/ctl stay live). + if(listenbusy) + srv.reply(styxservers->readstr(m, "error: listen busy")); + else { + listenbusy = 1; + addasync(m.tag, m.fid); + spawn asynclisten(helperc, m); + } + Qwake => + # Blocks until the wake word is detected; async for the + # same reason as Qlisten. + if(wakebusy) + srv.reply(styxservers->readstr(m, "error: wake busy")); + else { + wakebusy = 1; + addasync(m.tag, m.fid); + spawn asyncwake(helperc, m); + } + Qsayq => + fs := getfidstate(m.fid); + if(fs.sayresp == nil && fs.saydone != nil) { + ch := fs.saydone; + fs.saydone = nil; + addasync(m.tag, m.fid); + spawn saywait(Qsayq, helperc, m, ch); + } else if(fs.sayresp != nil) + srv.reply(styxservers->readbytes(m, fs.sayresp)); + else + srv.reply(styxservers->readstr(m, "")); + Qcancel => + if(cancelreq) + srv.reply(styxservers->readstr(m, "cancel pending\n")); + else + srv.reply(styxservers->readstr(m, "idle\n")); + Qchime => + srv.reply(ref Rmsg.Error(m.tag, Eperm)); * => srv.default(gm); } @@ -1621,6 +2175,22 @@ Serve: # the serveloop, blocking all other 9P traffic. srv.reply(ref Rmsg.Write(m.tag, len m.data)); spawn asyncsay(fs.saydone, strip(text)); + Qsayq => + text := string m.data; + fs := getfidstate(m.fid); + fs.sayreq = text; + fs.sayresp = nil; + fs.saydone = chan of array of byte; + cancelreq = 0; + srv.reply(ref Rmsg.Write(m.tag, len m.data)); + spawn asyncsay(fs.saydone, strip(text)); + Qcancel => + cancelreq = 1; + cancelprovider(); + srv.reply(ref Rmsg.Write(m.tag, len m.data)); + Qchime => + chimeprovider(string m.data); + srv.reply(ref Rmsg.Write(m.tag, len m.data)); Qhear => # Writing to hear resets/starts a new recording # Parse optional duration: "start 10000" = 10 seconds @@ -1640,8 +2210,10 @@ Serve: Clunk => fid := srv.getfid(m.fid); - if(fid != nil) + if(fid != nil) { + cancelasyncfid(m.fid); delfidstate(m.fid); + } srv.default(gm); * => diff --git a/appl/veltro/speechprovider.b b/appl/veltro/speechprovider.b new file mode 100644 index 000000000..07a9623bf --- /dev/null +++ b/appl/veltro/speechprovider.b @@ -0,0 +1,103 @@ +implement SpeechEngine; + +# Loadable speech engine that delegates to a namespace provider. This makes +# provider-backed speech usable through the same `.dis` contract as future +# in-process engines while keeping audio transport file-oriented. + +include "sys.m"; + sys: Sys; + +include "speech.m"; + +cfg: ref Speech->Config; + +init(): string +{ + sys = load Sys Sys->PATH; + if(sys == nil) + return "cannot load Sys"; + return nil; +} + +name(): string +{ + return "provider"; +} + +caps(): int +{ + return Speech->CAPTTS | Speech->CAPSTT; +} + +configure(c: ref Speech->Config): string +{ + if(c == nil || c.provider == nil || c.provider == "") + return "provider mount is not configured"; + cfg = c; + return nil; +} + +readfile(path: string): string +{ + fd := sys->open(path, Sys->OREAD); + if(fd == nil) + return nil; + out := ""; + buf := array[8192] of byte; + while((n := sys->read(fd, buf, len buf)) > 0) + out += string buf[0:n]; + return out; +} + +voices(): list of string +{ + if(cfg == nil) + return nil; + s := readfile(cfg.provider + "/voices"); + if(s == nil || s == "") + return nil; + (nil, lines) := sys->tokenize(s, "\n"); + out: list of string; + for(; lines != nil; lines = tl lines) + if(hd lines != "") + out = hd lines :: out; + rev: list of string; + for(; out != nil; out = tl out) + rev = hd out :: rev; + return rev; +} + +synthesize(text: string): ref Speech->TTSResult +{ + fmt := ref Speech->AudioFmt(22050, 1, 16, "pcm"); + if(cfg != nil && cfg.outfmt != nil) + fmt = cfg.outfmt; + if(cfg == nil) + return ref Speech->TTSResult(nil, fmt, "engine is not configured"); + fd := sys->open(cfg.provider + "/say", Sys->ORDWR); + if(fd == nil) + return ref Speech->TTSResult(nil, fmt, sys->sprint("provider say unavailable: %r")); + b := array of byte text; + if(sys->write(fd, b, len b) < 0) + return ref Speech->TTSResult(nil, fmt, sys->sprint("provider say failed: %r")); + sys->seek(fd, big 0, Sys->SEEKSTART); + buf := array[1024] of byte; + n := sys->read(fd, buf, len buf); + if(n > 0) { + status := string buf[0:n]; + if(len status >= 6 && status[0:6] == "error:") + return ref Speech->TTSResult(nil, fmt, status); + } + # The provider has already played the utterance, so no PCM is returned. + return ref Speech->TTSResult(nil, fmt, nil); +} + +recognize(nil: array of byte, nil: ref Speech->AudioFmt): ref Speech->STTResult +{ + if(cfg == nil) + return ref Speech->STTResult(nil, "engine is not configured"); + text := readfile(cfg.provider + "/listen"); + if(text == nil) + return ref Speech->STTResult(nil, sys->sprint("provider listen unavailable: %r")); + return ref Speech->STTResult(text, nil); +} diff --git a/appl/veltro/speechshim9p.b b/appl/veltro/speechshim9p.b new file mode 100644 index 000000000..b97baf168 --- /dev/null +++ b/appl/veltro/speechshim9p.b @@ -0,0 +1,1315 @@ +implement Speechshim9p; + +# +# speechshim9p - adapt external host speech helper CLIs to the speech +# provider contract (docs/SPEECH-ARCHITECTURE.md): +# +# /n/speechshim/ +# ├── ctl (rw) kokorobin, whisperstreambin, wakebin, wakeword, +# │ wakethreshold, whispermodel, voice, rate, +# │ audiodev, capturedev, micmode, capturerate, +# │ mic on|off +# ├── listen (r) newline records from the streaming STT helper: +# │ "partial [confidence=N] " / +# │ "final [confidence=N] " / "error: " +# ├── wake (r) blocks until the wake-word helper emits an event line +# ├── say (rw) write text: Kokoro synthesizes PCM, played through +# │ /dev/audio in chunks; read returns the status +# ├── cancel (w) kills the active TTS helper process and stops playback +# ├── chime (w) local earcons: wake, done, on, off +# └── voices (r) helper voice list +# +# speech9p consumes this mount exactly as it consumes a parakeet export or a +# remote provider — the helper binaries are an implementation detail behind +# the namespace. The helpers themselves are external installs (whisper.cpp +# stream, kokoro-onnx wrapper, openWakeWord wrapper); every path soft-fails +# with an "error: ..." record when a helper is absent. +# +# Host processes run through #C (devcmd). Streaming helpers (listen, wake) +# are started lazily by the first listen/wake read and read incrementally; +# killonclose is armed so they die with the shim. The microphone is thus +# only open while a client is actually reading: nothing runs at boot, and +# `mic off` on ctl tears the mic-side helpers down again (voicemode writes +# it when the user leaves voice mode) — the next read re-arms them. TTS is +# killed on cancel via the devcmd ctl "kill" command, and playback checks +# the cancel flag between chunks, so barge-in silence is bounded by one +# audio chunk rather than the remaining utterance. +# +# Audio routing (docs/SPEECH-REMOTE-AUDIO.md): playback always goes through +# the namespace (`audiodev`, default /dev/audio), so binding an imported +# remote audio device remotes the speakers with no shim changes. Capture has +# two modes: +# micmode helper (default) the helper CLI grabs the host microphone +# itself — right when the shim runs on the machine the +# user talks to. +# micmode device the shim reads s16le mono PCM from `capturedev` (falls +# back to `audiodev`) at `capturerate` and tees it into +# the stdin of the listen/wake helpers. The microphone is +# then just a namespace entry — an Android phone's or GUI +# terminal's exported /dev/audio works the same as the +# local device. +# + +include "sys.m"; + sys: Sys; + Qid: import Sys; + +include "draw.m"; + +include "arg.m"; + +include "math.m"; + math: Math; + +include "styx.m"; + styx: Styx; + Tmsg, Rmsg: import styx; + +include "styxservers.m"; + styxservers: Styxservers; + Fid, Styxserver, Navigator, Navop: import styxservers; + Enotfound, Eperm, Ebadarg: import styxservers; + +Speechshim9p: module { + init: fn(nil: ref Draw->Context, args: list of string); +}; + +Qroot, Qctl, Qlisten, Qwake, Qsay, Qcancel, Qvoices, Qchime: con iota; + +# Bytes of helper stderr retained for diagnostics (see Hostproc.errtail). +ERRTAIL: con 512; + +# Configuration +kokorobin := "kokoro-cli"; +whisperstreambin := "whisper-stream"; +wakebin := "openwakeword-cli"; +wakeword := "hey lucia"; +wakethreshold := "0.5"; +whispermodel := ""; +voice := "af_bella"; +# 22050, NOT Kokoro-native 24000: emu's devaudio only accepts the rates in +# audio_rate_tbl {8000, 11025, 16000, 22050, 44100}. An unsupported rate is +# rejected silently and playback runs at the 8000 default — Kokoro speech +# comes out as 3x slow-motion. kokoro-cli resamples to --rate, so 22050 is +# both accepted and near-native. +audrate := 22050; +audiodev := "/dev/audio"; +capturedev := ""; # capture override; empty = audiodev +micmode := "helper"; # helper | device +capturerate := 16000; +duplex := "full"; # full | half +standby := 0; # mic off: no helper (re)starts until the next listen/wake read +listenoff := 0; # listen off: the STT helper stays down until the next listen read + +stderr: ref Sys->FD; +user: string; +mountpt := "/n/speechshim"; +cmdbound := 0; +audiobound := 0; +cancelreq := 0; +playing := 0; + +# A host helper process behind #C. ctlfd is the clone fd (kept open — +# killonclose is armed on it); writing "kill" to it terminates the process. +Hostproc: adt { + ctlfd: ref Sys->FD; + datafd: ref Sys->FD; + dir: string; + # Tail of the helper's stderr, kept by a drain proc. A helper that fails + # to start (missing binary, bad model path) exits immediately and the only + # account of why is on its stderr — devcmd would otherwise discard it and + # leave us reporting a bare "helper exited". The drain also keeps the pipe + # from filling: whisper-stream is chatty on stderr, and a full pipe would + # block it. + errtail: string; + errdone: chan of string; + outbuf: string; +}; + +listenproc: ref Hostproc; +wakeproc: ref Hostproc; +sayproc: ref Hostproc; + +# Per-fid say state (same contract as speech9p's say file) +FidState: adt { + fid: int; + sayresp: array of byte; + saydone: chan of array of byte; +}; +fidstates: list of ref FidState; + +# Async read plumbing (same shape as speech9p's Helperdone machinery): +# blocking helper reads run in spawned procs and complete through helperc, +# so the serveloop — and with it ctl and cancel — stays live. +Helperdone: adt { + kind: int; # Qlisten, Qwake, Qsay + fid: int; + m: ref Tmsg.Read; + result: array of byte; +}; +helperc: chan of ref Helperdone; +asyncpending: list of (int, int); # (tag, fid) +listenbusy := 0; +wakebusy := 0; + +# Capture pump (micmode device): one proc owns the capture device and the +# helper stdin sinks; registration and reset arrive over pumpc so there is +# no shared mutable state between the pump and the 9P side. +SINKLISTEN, SINKWAKE, SINKRESET, SINKQUIT: con iota; +pumpc: chan of (int, ref Sys->FD); +pumprunning := 0; + +nomod(s: string) +{ + sys->fprint(stderr, "speechshim9p: can't load %s: %r\n", s); + raise "fail:load"; +} + +usage() +{ + sys->fprint(stderr, "Usage: speechshim9p [-D] [-m mountpoint]\n"); + raise "fail:usage"; +} + +init(nil: ref Draw->Context, args: list of string) +{ + sys = load Sys Sys->PATH; + sys->pctl(Sys->FORKFD|Sys->NEWPGRP, nil); + stderr = sys->fildes(2); + + styx = load Styx Styx->PATH; + if(styx == nil) + nomod(Styx->PATH); + styx->init(); + + styxservers = load Styxservers Styxservers->PATH; + if(styxservers == nil) + nomod(Styxservers->PATH); + styxservers->init(styx); + + arg := load Arg Arg->PATH; + if(arg == nil) + nomod(Arg->PATH); + math = load Math Math->PATH; + if(math == nil) + nomod(Math->PATH); + arg->init(args); + while((o := arg->opt()) != 0) + case o { + 'D' => styxservers->traceset(1); + 'm' => mountpt = arg->earg(); + * => usage(); + } + arg = nil; + + sys->pctl(Sys->FORKFD, nil); + + user = rf("/dev/user"); + if(user == nil) + user = "inferno"; + + fds := array[2] of ref Sys->FD; + if(sys->pipe(fds) < 0) { + sys->fprint(stderr, "speechshim9p: can't create pipe: %r\n"); + raise "fail:pipe"; + } + + helperc = chan of ref Helperdone; + pumpc = chan[4] of (int, ref Sys->FD); + + navops := chan of ref Navop; + spawn navigator(navops); + + (tchan, srv) := Styxserver.new(fds[0], Navigator.new(navops), big Qroot); + fds[0] = nil; + + pidc := chan of int; + spawn serveloop(tchan, srv, pidc, navops); + <-pidc; + + ensuredir(mountpt); + + if(sys->mount(fds[1], nil, mountpt, Sys->MREPL|Sys->MCREATE, nil) < 0) { + sys->fprint(stderr, "speechshim9p: mount failed: %r\n"); + raise "fail:mount"; + } +} + +# === Host process management (devcmd) === + +bindcmd() +{ + if(cmdbound) + return; + if(sys->stat("/cmd/clone").t0 == -1) + sys->bind("#C", "/", Sys->MBEFORE); + cmdbound = 1; +} + +bindaudio() +{ + if(audiobound) + return; + if(sys->stat("/dev/audio").t0 == -1) + sys->bind("#A", "/dev", Sys->MBEFORE); + audiobound = 1; +} + +openaudioout(rate: int): ref Sys->FD +{ + if(audiodev == "/dev/audio") + bindaudio(); + ctl := sys->open(audiodev + "ctl", Sys->OWRITE); + if(ctl != nil) { + writectl(ctl, sys->sprint("out rate %d", rate)); + writectl(ctl, "out chans 1"); + writectl(ctl, "out bits 16"); + writectl(ctl, "out enc pcm"); + ctl = nil; + } + return sys->open(audiodev, Sys->OWRITE); +} + +# Start a host command; the process dies with the shim (killonclose) or on +# killproc(). Returns (proc, nil) or (nil, error string). +startproc(cmd: string): (ref Hostproc, string) +{ + bindcmd(); + + cfd := sys->open("/cmd/clone", Sys->ORDWR); + if(cfd == nil) + return (nil, sys->sprint("error: cannot open /cmd/clone: %r")); + + buf := array[32] of byte; + n := sys->read(cfd, buf, len buf); + if(n <= 0) + return (nil, "error: cannot read cmd number"); + dir := "/cmd/" + string buf[0:n]; + + sys->fprint(cfd, "killonclose"); + if(sys->fprint(cfd, "exec /bin/sh -c '%s'", cmd) < 0) + return (nil, sys->sprint("error: exec failed: %r")); + + datafd := sys->open(dir + "/data", Sys->OREAD); + if(datafd == nil) + return (nil, sys->sprint("error: cannot open %s/data: %r", dir)); + + p := ref Hostproc(cfd, datafd, dir, "", chan[1] of string, ""); + errfd := sys->open(dir + "/stderr", Sys->OREAD); + if(errfd != nil) + spawn drainstderr(p, errfd); + + return (p, nil); +} + +# Keep the helper's stderr drained, retaining only the tail for diagnostics. +drainstderr(p: ref Hostproc, errfd: ref Sys->FD) +{ + buf := array[1024] of byte; + for(;;) { + n := sys->read(errfd, buf, len buf); + if(n <= 0) { + p.errdone <-= p.errtail; + return; + } + s := p.errtail + string buf[0:n]; + if(len s > ERRTAIL) + s = s[len s - ERRTAIL:]; + p.errtail = s; + } +} + +# Return one newline-delimited helper record. Host pipe reads may split a +# record arbitrarily, so retain an incomplete tail on the process rather than +# exposing it as a transcript or wake event. +readrecord(p: ref Hostproc): (string, int) +{ + for(;;) { + for(i := 0; i < len p.outbuf; i++) + if(p.outbuf[i] == '\n') { + record := p.outbuf[0:i+1]; + p.outbuf = p.outbuf[i+1:]; + return (record, 1); + } + + buf := array[8192] of byte; + n := sys->read(p.datafd, buf, len buf); + if(n <= 0) { + if(p.outbuf != "") { + record := p.outbuf; + p.outbuf = ""; + return (record, 1); + } + return ("", 0); + } + p.outbuf += string buf[0:n]; + } +} + +after(c: chan of int, ms: int) +{ + sys->sleep(ms); + c <-= 1; +} + +# One-line summary of why a helper died, for the "error:" record the client +# sees. Falls back to the caller's generic reason when stderr said nothing. +exitreason(p: ref Hostproc, dflt: string): string +{ + if(p == nil) + return dflt; + # stdout EOF and stderr EOF are delivered independently by #C. Wait briefly + # for the drainer's completion signal, but never let diagnostics wedge the + # voice daemon if a helper closes stdout while retaining stderr. + p.datafd = nil; + p.ctlfd = nil; # killonclose also lets /stderr reach EOF + if(p.errdone != nil) { + timeoutc := chan[1] of int; + spawn after(timeoutc, 250); + alt { + tail := <-p.errdone => + p.errtail = tail; + <-timeoutc => + ; + } + } + # Last non-blank line: host sh puts "not found" style errors there. + (nil, lines) := sys->tokenize(p.errtail, "\n\r"); + last := ""; + for(; lines != nil; lines = tl lines) { + l := strip(hd lines); + if(l != "") + last = l; + } + if(last == "") + return dflt; + return dflt + ": " + last; +} + +killproc(p: ref Hostproc) +{ + if(p != nil && p.ctlfd != nil) + sys->fprint(p.ctlfd, "kill"); +} + +closeproc(p: ref Hostproc) +{ + if(p == nil) + return; + p.datafd = nil; + p.ctlfd = nil; +} + +# One-shot host command, full stdout. +runcmd(cmd: string): string +{ + (p, err) := startproc(cmd); + if(p == nil) + return err; + result := ""; + rbuf := array[8192] of byte; + for(;;) { + n := sys->read(p.datafd, rbuf, len rbuf); + if(n <= 0) + break; + result += string rbuf[0:n]; + } + closeproc(p); + return result; +} + +# === Capture pump (micmode device) === + +capdev(): string +{ + if(capturedev != "") + return capturedev; + return audiodev; +} + +# Register a running helper's stdin as a pump sink. +addsink(kind: int, p: ref Hostproc) +{ + wfd := sys->open(p.dir + "/data", Sys->OWRITE); + if(wfd == nil) + return; + if(!pumprunning) { + pumprunning = 1; + spawn audiopump(); + } + pumpc <-= (kind, wfd); +} + +pumpreset() +{ + if(pumprunning) + pumpc <-= (SINKRESET, nil); +} + +opencapture(): ref Sys->FD +{ + dev := capdev(); + if(dev == "/dev/audio") + bindaudio(); + ctl := sys->open(dev + "ctl", Sys->OWRITE); + if(ctl != nil) { + writectl(ctl, sys->sprint("in rate %d", capturerate)); + writectl(ctl, "in chans 1"); + writectl(ctl, "in bits 16"); + writectl(ctl, "in enc pcm"); + ctl = nil; + } + return sys->open(dev, Sys->OREAD); +} + +# Read s16le mono PCM from the capture device and tee it into the stdin of +# the registered streaming helpers. The device is held open only while a +# sink is registered. On device EOF (an exported file exhausted, an import +# torn down) the sinks are closed so the helpers see stdin EOF and can +# flush a final record. +audiopump() +{ + sinks := array[2] of ref Sys->FD; + afd: ref Sys->FD; + for(;;) { + if(sinks[SINKLISTEN] == nil && sinks[SINKWAKE] == nil) { + afd = nil; # release the device while idle + (k, fd) := <-pumpc; + if(k == SINKQUIT) + return; + if(k != SINKRESET) + sinks[k] = fd; + continue; + } + Drain: + for(;;) alt { + (k, fd) := <-pumpc => + if(k == SINKQUIT) + return; + if(k == SINKRESET) { + afd = nil; + sinks[SINKLISTEN] = nil; + sinks[SINKWAKE] = nil; + } else + sinks[k] = fd; + * => + break Drain; + } + if(sinks[SINKLISTEN] == nil && sinks[SINKWAKE] == nil) + continue; + if(afd == nil) { + afd = opencapture(); + if(afd == nil) { + sinks[SINKLISTEN] = nil; + sinks[SINKWAKE] = nil; + continue; + } + } + chunk := capturerate / 10 * 2; # 100ms of s16 mono + if(chunk < 512) + chunk = 512; + buf := array[chunk] of byte; + n := sys->read(afd, buf, len buf); + if(n <= 0) { + afd = nil; + sinks[SINKLISTEN] = nil; + sinks[SINKWAKE] = nil; + continue; + } + if(duplex == "half" && playing) + continue; + for(k := 0; k < 2; k++) + if(sinks[k] != nil && sys->write(sinks[k], buf[0:n], n) < 0) + sinks[k] = nil; # helper died; drop the sink + } +} + +# === Streaming reads (listen / wake) === + +listencmd(): string +{ + if(micmode == "device") + return whisperstreambin + " --stdin --model " + whispermodel + + " --rate " + string capturerate + " --chans 1"; + return whisperstreambin + " --model " + whispermodel + + " --rate 16000 --chans 1"; +} + +wakecmd(): string +{ + if(micmode == "device") + return wakebin + " --stdin --word \"" + wakeword + "\" --threshold " + + wakethreshold + " --rate " + string capturerate; + # startproc wraps the complete host command in single quotes for #C. + # Use double quotes here so a multiword phrase remains one host-shell arg + # without terminating that outer command string. + return wakebin + " --word \"" + wakeword + "\" --threshold " + + wakethreshold; +} + +# Read the next chunk of newline records from a streaming helper, starting +# it on first use. Runs in a spawned proc; the globals it resets on EOF are +# also reset by ctl writes, which is benign — the next read restarts the +# helper either way. +readlisten(): string +{ + if(whisperstreambin == "") + return "error: listen helper not configured"; + standby = 0; # an active reader arms the microphone + listenoff = 0; # a new listen turn re-arms the STT helper + last: ref Hostproc; # last helper to die, for its stderr + # One restart attempt: a stale fd from an exited helper (one-shot + # helpers exit after each utterance) must not eat a read as an error. + for(attempt := 0; attempt < 2; attempt++) { + if(listenproc == nil) { + (p, err) := startproc(listencmd()); + if(p == nil) + return err; + listenproc = p; + if(micmode == "device") + addsink(SINKLISTEN, p); + if(standby || listenoff) { + # `mic off`/`listen off` raced the start; honor it. + killproc(p); + listenproc = nil; + return "error: mic off"; + } + } + (record, ok) := readrecord(listenproc); + if(ok) + return record; + dead := listenproc; + listenproc = nil; + # `mic off`/`listen off` while blocked in the read above kills + # the helper; return instead of restarting it. + if(standby) + return "error: mic off"; + if(listenoff) + return "error: listen off"; + last = dead; + } + return exitreason(last, "error: listen helper exited"); +} + +readwake(): string +{ + if(wakebin == "") + return "error: wake helper not configured"; + standby = 0; # an active reader arms the microphone + last: ref Hostproc; # last helper to die, for its stderr + # One restart attempt, same reason as readlisten: one-shot wake + # helpers exit after each event and must be restarted transparently. + attempt := 0; + for(;;) { + # `mic off` while blocked in the read below kills the helper; + # return instead of restarting it. + if(standby) + return "error: mic off"; + if(wakeproc == nil) { + (p, err) := startproc(wakecmd()); + if(p == nil) + return err; + wakeproc = p; + if(micmode == "device") + addsink(SINKWAKE, p); + if(standby) { + # `mic off` raced the start; honor it. + killproc(p); + wakeproc = nil; + return "error: mic off"; + } + } + (record, ok) := readrecord(wakeproc); + if(ok) { + if(duplex == "half" && playing) { + killproc(wakeproc); + closeproc(wakeproc); + wakeproc = nil; + sys->sleep(100); + continue; + } + return record; + } + last = wakeproc; + wakeproc = nil; + attempt++; + if(attempt >= 2) { + if(!(duplex == "half" && playing)) + break; + # A dead helper (exits with no output — e.g. not + # installed) must not spawn-storm while playback pins + # us in the suppression loop. + sys->sleep(100); + } + } + return exitreason(last, "error: wake helper exited"); +} + +# === TTS (say) === + +# Kokoro contract: text on stdin, s16le mono PCM at the requested rate on +# stdout. Playback is chunked so a cancel takes effect within one chunk. +dosay(text: string): string +{ + if(kokorobin == "") + return "error: kokoro helper not configured"; + text = strip(text); + if(text == "") + return "error: no speakable text"; + cancelreq = 0; + + cmd := kokorobin + " --voice " + voice + " --format pcm --rate " + string audrate; + (p, err) := startproc(cmd); + if(p == nil) + return err; + sayproc = p; + + # Feed text on stdin, close to signal EOF. + tofd := sys->open(p.dir + "/data", Sys->OWRITE); + if(tofd == nil) { + killproc(p); + closeproc(p); + sayproc = nil; + return sys->sprint("error: cannot open %s/data for write: %r", p.dir); + } + b := array of byte (text + "\n"); + sys->write(tofd, b, len b); + tofd = nil; + + afd := openaudioout(audrate); + + total := 0; + buf := array[8192] of byte; + status := ""; + playing = 1; + for(;;) { + n := sys->read(p.datafd, buf, len buf); + if(n <= 0) + break; + if(cancelreq) { + killproc(p); + status = "error: speech canceled"; + break; + } + if(afd == nil) + continue; # drain helper; no audio device + if(sys->write(afd, buf[0:n], n) < 0) { + status = sys->sprint("error: audio write failed: %r"); + killproc(p); + break; + } + total += n; + } + playing = 0; + closeproc(p); + sayproc = nil; + if(status != "") + return status; + if(total == 0) { + if(afd == nil) + return sys->sprint("error: cannot open %s: %r", audiodev); + return "error: kokoro produced no audio"; + } + return sys->sprint("ok: played %d bytes", total); +} + +put16le(buf: array of byte, off, val: int) +{ + buf[off] = byte (val & 16rFF); + buf[off+1] = byte ((val >> 8) & 16rFF); +} + +playnote(fd: ref Sys->FD, freq, ms: int) +{ + nsamp := audrate * ms / 1000; + if(nsamp <= 0) + return; + buf := array[nsamp * 2] of byte; + for(i := 0; i < nsamp; i++) { + v := int (12000.0 * math->sin(2.0 * Math->Pi * + real freq * real i / real audrate)); + put16le(buf, i * 2, v); + } + sys->write(fd, buf, len buf); +} + +playchime(kind: string) +{ + afd := openaudioout(audrate); + if(afd != nil) { + case kind { + "wake" => + playnote(afd, 660, 120); + playnote(afd, 880, 120); + "done" => + playnote(afd, 440, 140); + "on" => + playnote(afd, 523, 90); + playnote(afd, 659, 90); + playnote(afd, 784, 120); + "off" => + playnote(afd, 784, 90); + playnote(afd, 659, 90); + playnote(afd, 523, 120); + } + } + playing = 0; +} + +startchime(kind: string) +{ + kind = strip(kind); + case kind { + "wake" or "done" or "on" or "off" => + playing = 1; + spawn playchime(kind); + * => + sys->fprint(stderr, "speechshim9p: unknown chime: %s\n", kind); + } +} + +asyncsay(donech: chan of array of byte, text: string) +{ + donech <-= array of byte dosay(text); +} + +# === Async read completion (same pattern as speech9p) === + +addasync(tag, fid: int) +{ + asyncpending = (tag, fid) :: asyncpending; +} + +isasync(tag: int): int +{ + for(l := asyncpending; l != nil; l = tl l) { + (t, nil) := hd l; + if(t == tag) + return 1; + } + return 0; +} + +cancelasynctag(tag: int) +{ + newlist: list of (int, int); + for(l := asyncpending; l != nil; l = tl l) { + (t, nil) := hd l; + if(t != tag) + newlist = hd l :: newlist; + } + asyncpending = newlist; +} + +cancelasyncfid(fid: int) +{ + newlist: list of (int, int); + for(l := asyncpending; l != nil; l = tl l) { + (nil, f) := hd l; + if(f != fid) + newlist = hd l :: newlist; + } + asyncpending = newlist; +} + +asynclisten(donec: chan of ref Helperdone, m: ref Tmsg.Read) +{ + donec <-= ref Helperdone(Qlisten, m.fid, m, array of byte readlisten()); +} + +asyncwake(donec: chan of ref Helperdone, m: ref Tmsg.Read) +{ + donec <-= ref Helperdone(Qwake, m.fid, m, array of byte readwake()); +} + +saywait(donec: chan of ref Helperdone, m: ref Tmsg.Read, ch: chan of array of byte) +{ + donec <-= ref Helperdone(Qsay, m.fid, m, <-ch); +} + +asyncdone(srv: ref Styxserver, h: ref Helperdone) +{ + case h.kind { + Qlisten => listenbusy = 0; + Qwake => wakebusy = 0; + } + if(!isasync(h.m.tag)) + return; + cancelasynctag(h.m.tag); + if(h.kind == Qsay) { + fs := getfidstate(h.fid); + fs.sayresp = h.result; + srv.reply(styxservers->readbytes(h.m, h.result)); + return; + } + # Streaming replies (listen/wake) must ignore the fid's read offset: + # consumers hold one fd across many reads, and offset-sliced replies + # would EOF the stream after the first read (INFR-28). Reply with the + # raw record bytes, clamped to the requested count. + srv.reply(ref Rmsg.Read(h.m.tag, clampcount(h.m, h.result))); +} + +clampcount(m: ref Tmsg.Read, data: array of byte): array of byte +{ + if(len data > m.count) + return data[0:m.count]; + return data; +} + +# === Configuration === + +readconfig(): string +{ + result := "kokorobin " + kokorobin + "\n"; + result += "whisperstreambin " + whisperstreambin + "\n"; + result += "wakebin " + wakebin + "\n"; + result += "wakeword " + wakeword + "\n"; + result += "wakethreshold " + wakethreshold + "\n"; + result += "whispermodel " + whispermodel + "\n"; + result += "voice " + voice + "\n"; + result += "rate " + string audrate + "\n"; + result += "audiodev " + audiodev + "\n"; + result += "capturedev " + capturedev + "\n"; + result += "micmode " + micmode + "\n"; + result += "capturerate " + string capturerate + "\n"; + result += "duplex " + duplex + "\n"; + if(standby) + result += "mic off\n"; + else + result += "mic on\n"; + if(listenoff) + result += "listen off\n"; + else + result += "listen on\n"; + return result; +} + +# Capture-path config changed: restart the streaming helpers so they come +# back up against the new device/mode, and make the pump drop its device fd +# and stale sinks. +resetcapture() +{ + killproc(listenproc); + listenproc = nil; + killproc(wakeproc); + wakeproc = nil; + pumpreset(); +} + +applyconfig(cmd: string): string +{ + (n, argv) := sys->tokenize(cmd, " \t\n"); + if(n < 2) + return "error: usage: "; + key := hd argv; + argv = tl argv; + val := ""; + for(; argv != nil; argv = tl argv) { + if(val != "") + val += " "; + val += hd argv; + } + + case key { + "kokorobin" => + kokorobin = val; + "whisperstreambin" => + # Restart the stream with the new helper on next read. + killproc(listenproc); + listenproc = nil; + whisperstreambin = val; + "wakebin" => + killproc(wakeproc); + wakeproc = nil; + wakebin = val; + "wakeword" => + killproc(wakeproc); + wakeproc = nil; + wakeword = val; + "wakethreshold" => + killproc(wakeproc); + wakeproc = nil; + wakethreshold = val; + "whispermodel" or "sttmodel" => + killproc(listenproc); + listenproc = nil; + whispermodel = val; + "voice" => + voice = val; + "rate" => + r := int val; + if(r < 8000 || r > 48000) + return "error: rate must be 8000-48000"; + audrate = r; + "audiodev" => + audiodev = val; + resetcapture(); + "capturedev" => + if(val == "default") + val = ""; + capturedev = val; + resetcapture(); + "micmode" => + if(val != "helper" && val != "device") + return "error: micmode must be helper or device"; + micmode = val; + resetcapture(); + "capturerate" => + r := int val; + if(r < 8000 || r > 48000) + return "error: capturerate must be 8000-48000"; + capturerate = r; + resetcapture(); + "duplex" => + if(val != "full" && val != "half") + return "error: duplex must be full or half"; + duplex = val; + "mic" => + # Voice-mode teardown: `mic off` kills the mic-side helpers + # (and the capture pump's device fd) so the microphone is not + # held open outside a voice session. The next listen/wake read + # re-arms it; `mic on` is accepted for symmetry. + case val { + "off" => + standby = 1; + resetcapture(); + "on" => + standby = 0; + * => + return "error: mic must be on or off"; + } + "listen" => + # Turn-end teardown from voicemode: `listen off` stops only + # the STT helper, so speech between voice turns (ambient talk, + # the assistant's own TTS) cannot queue as stale records that + # replay into the next turn. Wake stays armed; the next listen + # read restarts the STT helper. + case val { + "off" => + listenoff = 1; + killproc(listenproc); + listenproc = nil; + "on" => + listenoff = 0; + * => + return "error: listen must be on or off"; + } + * => + return "error: unknown config key: " + key; + } + return "ok"; +} + +listvoices(): string +{ + if(kokorobin == "") + return "(kokoro helper not configured)\n"; + result := runcmd(kokorobin + " --list-voices 2>/dev/null"); + if(result == "" || hasprefix(result, "error:")) + return "af_bella\n(default; helper unavailable or does not list voices)\n"; + return result; +} + +# === Per-fid state === + +getfidstate(fid: int): ref FidState +{ + for(l := fidstates; l != nil; l = tl l) { + if((hd l).fid == fid) + return hd l; + } + fs := ref FidState(fid, nil, nil); + fidstates = fs :: fidstates; + return fs; +} + +delfidstate(fid: int) +{ + newlist: list of ref FidState; + for(l := fidstates; l != nil; l = tl l) { + if((hd l).fid != fid) + newlist = hd l :: newlist; + } + fidstates = newlist; +} + +# === 9P plumbing === + +navigator(navops: chan of ref Navop) +{ + for(;;) { + navop := <-navops; + if(navop == nil) + return; + pick n := navop { + Stat => + (d, err) := dirgen(int n.path); + n.reply <-= (d, err); + Walk => + walkto(n); + Readdir => + readdir(n, int n.path); + } + } +} + +walkto(n: ref Navop.Walk) +{ + if(int n.path != Qroot) { + n.reply <-= (nil, Enotfound); + return; + } + case n.name { + ".." or "." => + n.path = big Qroot; + "ctl" => + n.path = big Qctl; + "listen" => + n.path = big Qlisten; + "wake" => + n.path = big Qwake; + "say" => + n.path = big Qsay; + "cancel" => + n.path = big Qcancel; + "chime" => + n.path = big Qchime; + "voices" => + n.path = big Qvoices; + * => + n.reply <-= (nil, Enotfound); + return; + } + (d, err) := dirgen(int n.path); + n.reply <-= (d, err); +} + +dirgen(path: int): (ref Sys->Dir, string) +{ + name: string; + perm: int; + case path { + Qroot => + return (dir(Qid(big Qroot, 0, Sys->QTDIR), ".", big 0, 8r555|Sys->DMDIR), nil); + Qctl => + name = "ctl"; + perm = 8r666; + Qlisten => + name = "listen"; + perm = 8r444; + Qwake => + name = "wake"; + perm = 8r444; + Qsay => + name = "say"; + perm = 8r666; + Qcancel => + name = "cancel"; + perm = 8r222; + Qchime => + name = "chime"; + perm = 8r222; + Qvoices => + name = "voices"; + perm = 8r444; + * => + return (nil, Enotfound); + } + return (dir(Qid(big path, 0, Sys->QTFILE), name, big 0, perm), nil); +} + +dir(qid: Sys->Qid, name: string, length: big, perm: int): ref Sys->Dir +{ + d := ref sys->zerodir; + d.qid = qid; + if(perm & Sys->DMDIR) + d.qid.qtype = Sys->QTDIR; + d.mode = perm; + d.name = name; + d.uid = user; + d.gid = user; + d.length = length; + return d; +} + +readdir(n: ref Navop.Readdir, path: int) +{ + if(path != Qroot) { + n.reply <-= (nil, Enotfound); + return; + } + entries := array[] of {Qctl, Qlisten, Qwake, Qsay, Qcancel, Qchime, Qvoices}; + for(i := n.offset; i < len entries && i < n.offset + n.count; i++) { + (d, err) := dirgen(entries[i]); + if(err != nil) { + n.reply <-= (nil, err); + return; + } + n.reply <-= (d, nil); + } + n.reply <-= (nil, nil); +} + +serveloop(tchan: chan of ref Tmsg, srv: ref Styxserver, pidc: chan of int, navops: chan of ref Navop) +{ + pidc <-= sys->pctl(0, nil); + +Serve: + for(;;) { + gm: ref Tmsg; + alt { + gm = <-tchan => + ; + h := <-helperc => + asyncdone(srv, h); + continue; + } + if(gm == nil) + break Serve; + + pick m := gm { + Readerror => + break Serve; + + Flush => + cancelasynctag(m.oldtag); + srv.reply(ref Rmsg.Flush(m.tag)); + + Read => + fid := srv.getfid(m.fid); + if(fid == nil) { + srv.reply(ref Rmsg.Error(m.tag, "bad fid")); + continue; + } + path := int fid.path; + case path { + Qctl => + srv.reply(styxservers->readstr(m, readconfig())); + Qvoices => + srv.reply(styxservers->readstr(m, listvoices())); + Qlisten => + if(listenbusy) + srv.reply(styxservers->readstr(m, "error: listen busy")); + else { + listenbusy = 1; + addasync(m.tag, m.fid); + spawn asynclisten(helperc, m); + } + Qwake => + if(wakebusy) + srv.reply(styxservers->readstr(m, "error: wake busy")); + else { + wakebusy = 1; + addasync(m.tag, m.fid); + spawn asyncwake(helperc, m); + } + Qsay => + fs := getfidstate(m.fid); + if(fs.sayresp == nil && fs.saydone != nil) { + ch := fs.saydone; + fs.saydone = nil; + addasync(m.tag, m.fid); + spawn saywait(helperc, m, ch); + } else if(fs.sayresp != nil) + srv.reply(styxservers->readbytes(m, fs.sayresp)); + else + srv.reply(styxservers->readstr(m, "")); + Qcancel => + srv.reply(styxservers->readstr(m, "")); + * => + srv.default(gm); + } + + Write => + fid := srv.getfid(m.fid); + if(fid == nil) { + srv.reply(ref Rmsg.Error(m.tag, "bad fid")); + continue; + } + path := int fid.path; + case path { + Qctl => + result := applyconfig(string m.data); + srv.reply(ref Rmsg.Write(m.tag, len m.data)); + if(hasprefix(result, "error:")) + sys->fprint(stderr, "speechshim9p: %s\n", result); + Qsay => + fs := getfidstate(m.fid); + fs.sayresp = nil; + fs.saydone = chan of array of byte; + srv.reply(ref Rmsg.Write(m.tag, len m.data)); + spawn asyncsay(fs.saydone, string m.data); + Qcancel => + # Hard cancel: kill the synthesizing helper and let + # the playback loop notice within one chunk. + cancelreq = 1; + killproc(sayproc); + srv.reply(ref Rmsg.Write(m.tag, len m.data)); + Qchime => + startchime(string m.data); + srv.reply(ref Rmsg.Write(m.tag, len m.data)); + * => + srv.reply(ref Rmsg.Error(m.tag, Eperm)); + } + + Clunk => + fid := srv.getfid(m.fid); + if(fid != nil) { + cancelasyncfid(m.fid); + delfidstate(m.fid); + } + srv.default(gm); + + * => + srv.default(gm); + } + } + navops <-= nil; + if(pumprunning) + pumpc <-= (SINKQUIT, nil); # don't outlive the mount +} + +writectl(fd: ref Sys->FD, cmd: string) +{ + data := array of byte cmd; + sys->write(fd, data, len data); +} + +# === Small helpers === + +rf(f: string): string +{ + fd := sys->open(f, Sys->OREAD); + if(fd == nil) + return nil; + buf := array[128] of byte; + n := sys->read(fd, buf, len buf); + if(n <= 0) + return nil; + return string buf[0:n]; +} + +ensuredir(path: string) +{ + sys->create(path, Sys->OREAD, Sys->DMDIR | 8r755); +} + +hasprefix(s, prefix: string): int +{ + return len s >= len prefix && s[0:len prefix] == prefix; +} + +strip(s: string): string +{ + if(s == nil) + return nil; + i := 0; + while(i < len s && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n')) + i++; + j := len s; + while(j > i && (s[j-1] == ' ' || s[j-1] == '\t' || s[j-1] == '\r' || s[j-1] == '\n')) + j--; + if(i >= j) + return ""; + return s[i:j]; +} diff --git a/appl/wm/logon.b b/appl/wm/logon.b index 1cccaa68e..ddb32759b 100644 --- a/appl/wm/logon.b +++ b/appl/wm/logon.b @@ -118,9 +118,14 @@ init(ctxt: ref Draw->Context, nil: list of string) if(smallfont == nil) smallfont = bodyfont; - # If factotum was already started with secstore backing (e.g. headless - # mode with $SECSTORE_PASSWORD), skip the login screen entirely. - if(factotumhaskeys()) { + # Skip the login screen only when the profile explicitly unlocked + # secstore from $SECSTORE_PASSWORD (headless mode) — it sets + # $secstoreautounlock on exactly that path. Factotum merely holding + # keys is NOT evidence of authentication: the profile provisions + # API-key fallbacks (host env vars, /lib/veltro/keys files) into + # factotum before logon runs, and an API key must never bypass the + # password screen. + if(secstoreautounlocked() && factotumhaskeys()) { createsecstoresentinel(); return; } @@ -913,6 +918,22 @@ factotumhaskeys(): int return n > 0; } +# Set by lib/sh/profile only on the $SECSTORE_PASSWORD auto-unlock path. +secstoreautounlocked(): int +{ + fd := sys->open("/env/secstoreautounlock", Sys->OREAD); + if(fd == nil) + return 0; + buf := array[16] of byte; + n := sys->read(fd, buf, len buf); + if(n <= 0) + return 0; + s := string buf[0:n]; + while(len s > 0 && (s[len s - 1] == '\n' || s[len s - 1] == ' ' || s[len s - 1] == 0)) + s = s[0:len s - 1]; + return s == "1"; +} + secstoreacctexists(): int { user := rf("/dev/user"); diff --git a/dis/lucibridge.dis b/dis/lucibridge.dis index f6123c498..6a71a55cb 100644 Binary files a/dis/lucibridge.dis and b/dis/lucibridge.dis differ diff --git a/dis/luciconv.dis b/dis/luciconv.dis index 22a5204a6..e2c5bc0e3 100644 Binary files a/dis/luciconv.dis and b/dis/luciconv.dis differ diff --git a/dis/lucictx.dis b/dis/lucictx.dis index 26abccffd..dfb4c87e8 100644 Binary files a/dis/lucictx.dis and b/dis/lucictx.dis differ diff --git a/dis/lucifer.dis b/dis/lucifer.dis index 5a3c37fb0..ed359b16f 100644 Binary files a/dis/lucifer.dis and b/dis/lucifer.dis differ diff --git a/dis/luciuisrv.dis b/dis/luciuisrv.dis index b9d2072cf..a78ab818b 100644 Binary files a/dis/luciuisrv.dis and b/dis/luciuisrv.dis differ diff --git a/dis/speechtest.dis b/dis/speechtest.dis new file mode 100644 index 000000000..3ab99da6f Binary files /dev/null and b/dis/speechtest.dis differ diff --git a/dis/tests/luciuisrv_test.dis b/dis/tests/luciuisrv_test.dis index f86cbd695..b801a0fbb 100644 Binary files a/dis/tests/luciuisrv_test.dis and b/dis/tests/luciuisrv_test.dis differ diff --git a/dis/tests/speech9p_voice_test.dis b/dis/tests/speech9p_voice_test.dis new file mode 100644 index 000000000..1d7dfbbb8 Binary files /dev/null and b/dis/tests/speech9p_voice_test.dis differ diff --git a/dis/veltro/speech9p.dis b/dis/veltro/speech9p.dis index c2488426f..bdb7a5363 100644 Binary files a/dis/veltro/speech9p.dis and b/dis/veltro/speech9p.dis differ diff --git a/dis/veltro/speechprovider.dis b/dis/veltro/speechprovider.dis new file mode 100644 index 000000000..21ae32ad3 Binary files /dev/null and b/dis/veltro/speechprovider.dis differ diff --git a/dis/veltro/speechshim9p.dis b/dis/veltro/speechshim9p.dis new file mode 100644 index 000000000..aa36d76cf Binary files /dev/null and b/dis/veltro/speechshim9p.dis differ diff --git a/dis/voicemode.dis b/dis/voicemode.dis new file mode 100644 index 000000000..2fe46f053 Binary files /dev/null and b/dis/voicemode.dis differ diff --git a/dis/wm/logon.dis b/dis/wm/logon.dis index a4bc592cc..064326770 100644 Binary files a/dis/wm/logon.dis and b/dis/wm/logon.dis differ diff --git a/docs/SPEECH-ARCHITECTURE.md b/docs/SPEECH-ARCHITECTURE.md index 0af6d04be..2c39d3996 100644 --- a/docs/SPEECH-ARCHITECTURE.md +++ b/docs/SPEECH-ARCHITECTURE.md @@ -54,11 +54,13 @@ flowchart LR | Component | Source | Role | |---------------------------------|-------------------------------------|------| | `speech9p` | `appl/veltro/speech9p.b` | The 9P server. Routes ctl/say/hear/voices to engine backends. | -| `module/speech.m` | `module/speech.m` | Abstract `TTSEngine`/`STTEngine` interface (currently descriptive — speech9p in-lines its three backends rather than loading separate engine modules). | +| `module/speech.m` | `module/speech.m` | Batch result types plus the loadable `SpeechEngine` `.dis` module contract. Legacy `TTSEngine`/`STTEngine` ADTs remain for source compatibility. | +| Provider engine module | `appl/veltro/speechprovider.b` | Production `SpeechEngine` implementation that delegates TTS/STT/voices to a namespace provider. | | `say` tool | `appl/veltro/tools/say.b` | Veltro tool that opens `/n/speech/say` and writes text. | | `hear` tool | `appl/veltro/tools/hear.b` | Veltro tool that writes `start ` to `/n/speech/hear` and reads the transcription back. | -| `lucibridge` auto-speak | `appl/cmd/lucibridge.b:speaktext` | GUI-side path: opens `/n/speech/say` itself when `/voice on` is active. Bypasses the agent tool. | +| `lucibridge` voice output | `appl/cmd/lucibridge.b` | GUI-side path: completion-aware FIFO TTS, sentence-boundary streaming, cancellation, and speech timing. Bypasses the agent tool. | | `nsconstruct` / `tools9p` glue | `appl/veltro/nsconstruct.b`, `tools9p.b:992` | Auto-grants `/n/speech` to agents that have `say` or `hear` registered. | +| `parakeet-stream` adapter | `tools/parakeet_stream.cpp` | Host-side realtime STT helper: stdin s16le PCM → cache-aware streaming Parakeet EOU model → `partial` / `final confidence=…` records. Built by the installer against an upstream clone of [parakeet.cpp](https://github.com/mudler/parakeet.cpp); the **default STT** when it can be built (whisper wrapper is the fallback). See §3.3. | ## 2. Filesystem @@ -107,7 +109,8 @@ write /n/speech/ctl <- pipermodel /opt/piper/models/en_US-lessac-medium.onnx | Key | Values | Notes | |----------------|-------------------------------------------------|-------| -| `engine` | `cmd` · `api` · `local` | `cmd` is the default. `local` is **only** reachable via this verb or auto-detection (the CLI flag rejects it). | +| `engine` | `cmd` · `api` · `local` · `kokoro` · `module` | `cmd` is the default. `module` selects the previously loaded `.dis` engine. | +| `module` | path to a `SpeechEngine` `.dis` module | Loads, configures, and selects a module atomically; failures leave the previous engine active. `-E path` performs the same selection at startup. | | `voice` | engine-specific | API engine silently rewrites `""`, `default`, or `samantha` to `alloy`. | | `lang` | language code, e.g. `en` | Cosmetic for `cmd`; passed through to `api`. | | `rate` | 8000–48000 | Sample rate (Hz) for `/dev/audio` playback. | @@ -119,8 +122,103 @@ write /n/speech/ctl <- pipermodel /opt/piper/models/en_US-lessac-medium.onnx | `apikey` | bearer token | `api` engine. **Stored in process memory in clear.** | | `piperbin` / `pipermodel` | binary path / `.onnx` voice model | `local` engine. | | `whisperbin` / `whispermodel` | binary path / `.bin` GGML model | `local` engine. | +| `ttsengine` | `engine` or `piper` | Selects whether `/n/speech/say` uses the configured speech9p TTS engine or delegates to the provider's `say` file. | +| `provider` | provider mount root | The speech provider mount behind `listen`, `wake`, kokoro-engine `say`, and `cancel` (see the provider contract below). Default `/n/parakeet`; boot points it at `/n/speechshim`. | +| `listenengine` | `whisper` or `parakeet` | Compatibility alias; both values consume the provider mount. | +| `whisperstreambin` / `wakebin` / `kokorobin` / `wakeword` / `wakethreshold` | helper commands and wake tuning | Stored for introspection and forwarded to the provider's `ctl`; `speechshim9p` consumes them. `speech9p` itself runs no helpers. | +| `audiodev` / `capturedev` / `micmode` / `capturerate` | audio routing (see SPEECH-REMOTE-AUDIO.md) | Forwarded to the provider's `ctl` unchanged. In `speechshim9p`: `audiodev` is the playback (and default capture) device path; `capturedev` overrides capture (`default` clears it); `micmode helper\|device` chooses whether the helper CLI grabs the host mic or the shim pumps PCM from the capture device into helper stdin; `capturerate` is the pump sample rate. | +| `duplex` | `full` or `half` | Forwarded to the provider's `ctl`. In `speechshim9p`, `half` suppresses wake/capture delivery while playback or chimes are active. | +| `mic` | `on` or `off` | Forwarded to the provider's `ctl`. In `speechshim9p`, `off` kills the mic-side helpers (and the capture pump's device fd) and fails pending `listen`/`wake` reads with `error: mic off` instead of restarting them; the next read re-arms the microphone. `voicemode` writes `mic off` on voice-mode exit, so the mic is only open during a voice session. | +| `listen` | `on` or `off` | Forwarded to the provider's `ctl`. In `speechshim9p`, `off` kills only the STT helper and fails a pending `listen` read with `error: listen off` instead of restarting it; wake stays armed, and the next `listen` read restarts STT. `voicemode` writes `listen off` at the end of every voice turn (final, error, or timeout), so speech between turns — ambient talk, the assistant's own TTS — cannot queue as stale records that replay into the next turn. | +| `parakeetmount` / `parakeetlisten` / `pipersay` | provider root, STT stream file, and TTS say file | Compatibility aliases for `provider` and its derived `listen`/`say` paths. | + +`/n/speech/listen` is the stable Infernode-facing interface. + +## The speech provider contract + +All streaming voice I/O behind `/n/speech` comes from a single **provider +mount** — a 9P namespace serving this contract: + +| Path | Contract | +|------|----------| +| `/listen` | continuous newline-delimited `partial ...` / `final ...` / `error: ...` records; the provider owns the persistent microphone/STT process | +| `/wake` | read blocks until the wake-word engine fires, then returns one event line (model, score) | +| `/say` | write text to synthesize and play; read the last TTS status | +| `/cancel` | write to hard-cancel active TTS | +| `/chime` | optional write-only local earcons: `wake`, `done`, `on`, `off` | +| `/ctl` | optional provider configuration (helper paths, wake word, voice, ...) | +| `/voices` | optional voice list | + +`speech9p` selects the provider with `echo 'provider /n/x' > /n/speech/ctl` +(`parakeetmount` remains as an alias) and consumes only these files — it runs +no helper binaries itself. Streaming reads ignore the fid offset (a consumer +holds one fd across many reads), and helper-configuration keys written to +`/n/speech/ctl` are forwarded to the provider's `ctl`. + +InferNode boots voice mode in half-duplex by writing `duplex half` after the +default `/n/speechshim` provider is selected. During playback and earcons, the +shim keeps device capture drained but suppresses delivery to STT/wake helpers; +in helper-microphone mode it discards wake events that arrive while playback is +active. This prevents TTS echo from re-triggering voice mode at the cost of +spoken barge-in during assistant speech. Esc barge-in still works, and headset +or echo-controlled setups can restore spoken barge-in with `duplex full`. + +Providers implementing the contract today: + +| Provider | Backing | Lifecycle | +|----------|---------|-----------| +| `speechshim9p` (in-tree, default at `/n/speechshim`) | external helper CLIs — whisper.cpp stream for `listen`, openWakeWord wrapper for `wake`, Kokoro for `say` — driven through `#C`/devcmd | The shim owns the helper processes: streaming helpers are started on first read and restarted transparently when a one-shot helper exits; `cancel` kills the synthesizing process via devcmd `kill`, bounding barge-in silence by one audio chunk. | +| parakeet export (e.g. `/n/parakeet`) | parakeet-cli `--mic --stream` process exported over 9P | The mounted service owns the live microphone process; `speech9p` keeps the listen file open across reads so the stream remains continuous. | +| remote provider (Phase 2) | any of the above mounted over the network | Same contract; namespace composition does the remoting (see SPEECH-REMOTE-AUDIO.md). | + +The real-helper setup path is `tools/install-speech-helpers.sh`. It prepares +the Kokoro, whisper.cpp, and openWakeWord wrapper commands consumed by +`speechshim9p`, prints the `/n/speech/ctl` block, and leaves microphone access +to the interactive Inferno session. See `docs/SPEECH-VOICE-ONLY-PHASE1.md` for +the setup walkthrough and the repo-owned `micmode device` stdin-PCM adapter. + +Parakeet is not a separate speech architecture. It is the first Phase 1 +provider that needs the mounted-service shape because its useful mode is a +long-lived microphone stream, for example: -## 3. The three engines +``` +cd +/examples/cli/parakeet-cli transcribe \ + --model models/parakeet_realtime_eou_120m-v1-f16.gguf \ + --mic --stream --lines +``` + +The mounted service exports that process through a 9P namespace, normally: + +| Path | Contract | +|------|----------| +| `/n/parakeet/listen` | continuous newline-delimited `partial ...`, `final ...`, status, and TTS records from the live microphone stream | +| `/n/parakeet/say` | write text to synthesize and play with Piper; read the last TTS status | +| `/n/parakeet/cancel` | optional write-only cancellation hook; `speech9p` writes `cancel` when `/n/speech/cancel` is written | + +Infernode wiring is then only: + +``` +echo 'listenengine parakeet' > /n/speech/ctl +echo 'ttsengine piper' > /n/speech/ctl +echo 'parakeetmount /n/parakeet' > /n/speech/ctl +``` + +`speech9p` keeps `/n/parakeet/listen` open across reads so it consumes one +mounted stream rather than starting bounded transcription requests. When +`ttsengine piper` is set, `/n/speech/say` delegates to the configured +`pipersay` file, normally `/n/parakeet/say`, so the voice-only path uses +Parakeet STT and Piper TTS from the same mounted process instead of relying on +platform TTS such as macOS `say`. `voicemode` submits only `final ...` records to Lucia; partial records +are available for debug or future live UI work but are not submitted as turns in +Phase 1. + +A Whisper provider could be mounted behind the same stream-file contract later. +Phase 1 keeps the existing Whisper command path because it is already the +default helper mechanism and does not require the Parakeet-specific +`--mic --stream` process lifecycle. + +## 3. Engine backends ```mermaid flowchart TB @@ -157,8 +255,32 @@ while the API and local paths require a working audio device. | `cmd` | `say`/`espeak-ng`/PowerShell via `#C` | `ffmpeg` (macOS) / `arecord` (Linux) → temp WAV → `whisper-cli` | | `api` | POST to `/audio/speech` → PCM → `playpcm` | record via `recordaudio` → POST to `/audio/transcriptions` (multipart) | | `local` | `piper --output-raw` (text on stdin) → PCM → `playpcm` | `ffmpeg`/`arecord` → temp WAV → `whisper-cli` | +| `kokoro` | provider `say` file | provider streaming `listen` file | +| `module` | loaded module returns PCM, or completes playback itself | captured PCM passed to the loaded module | + +### 3.1 Loadable `.dis` engines + +`SpeechEngine` in `module/speech.m` is the runtime module ABI: `init`, `name`, +`caps`, `configure`, `voices`, `synthesize`, and `recognize`. A module advertises +`CAPTTS`, `CAPSTT`, or both. Configuration is re-applied when voice, language, +format, or provider settings change. -### 3.1 Engine selection +```sh +echo 'provider /n/remotespeech' > /n/speech/ctl +echo 'module /dis/veltro/speechprovider.dis' > /n/speech/ctl +cat /n/speech/voices +``` + +The in-tree provider module is deliberately file-oriented: it delegates through +the provider's `say`, `listen`, and `voices` files. Future in-process model +modules can return PCM directly without changing `speech9p`. + +`voicemode` accepts confidence metadata on `partial` and `final` listen +records. Records below its confirmation threshold are held behind a visual and +spoken confirmation instead of being submitted as a user turn. The default is +650 permille; `voicemode -q confidence-permille` selects another value. + +### 3.2 Engine selection ```mermaid flowchart TD @@ -189,6 +311,47 @@ The CLI parser rejecting `-e local` (`speech9p.b:147–155`) is a clear bug — fixed, set `local` via `echo 'engine local' > /n/speech/ctl` after launch, or rely on the Linux auto-promotion in `initplatform`. +**Boot-time selection is owned by the installer.** `tools/install-speech-helpers.sh` +writes its chosen stack — one Inferno-sh ctl line per row — to +`~/.local/share/infernode-speech/speech.ctl.sh`, and `lib/lucifer/boot.sh` +replays that file verbatim when it exists. The file always includes +`engine kokoro`, so an installed system speaks with Kokoro rather than the +`engine cmd` default (the robotic host `say`). Without the file, boot falls +back to hardcoded ctl lines for a legacy helper install (now also including +`engine kokoro`), and with no helpers at all speech9p keeps `engine cmd`. + +### 3.3 Parakeet realtime STT (`parakeet-stream`) + +The default listen helper when the installer can build it. Unlike the +whisper wrapper — which fakes turn-taking with an energy-VAD around a +sliding window — the Parakeet `parakeet_realtime_eou_120m-v1` model is a +cache-aware streaming transducer that emits ``/`` tokens: **the +model decides when the utterance is over**. + +- Source: `tools/parakeet_stream.cpp` (tracked here), compiled by the + installer against a clone of upstream + [parakeet.cpp](https://github.com/mudler/parakeet.cpp) (committed API + only: `ModelLoader`, `StreamingMel`, `StreamingSession`). +- Input: s16le PCM on **stdin only** — the shim's capture pump feeds it + (`micmode device`, `capturerate 16000`), which is what makes remote-mic + topologies pure namespace composition. There is deliberately no + microphone code in the adapter. +- Output: the provider contract's records, flushed per line: + `partial ` … `final confidence=0.9307 `. Confidence is the + mean of the utterance's per-word confidences (NeMo `max_prob`). +- After each `` the adapter resets the streaming session — the model + stops emitting after an EOU otherwise (verified against upstream's own + file-streaming path), and the reset also bounds hypothesis growth over + an hours-long session. +- The streaming EOU model is not yet published as GGUF; the installer + probes `PARAKEET_EOU_MODEL`, its own models dir, and dev checkouts, and + prints conversion instructions when none is found (whisper remains the + fallback in that case). + +Shim configuration is unchanged: the adapter is a drop-in for the +`whisperstreambin` slot because it accepts the same +`--stdin --model M --rate R --chans 1` invocation `listencmd()` builds. + ## 4. Data flow ### 4.1 TTS — write to `/n/speech/say` diff --git a/docs/SPEECH-REMOTE-AUDIO.md b/docs/SPEECH-REMOTE-AUDIO.md index 4eab8b98c..4a5916343 100644 --- a/docs/SPEECH-REMOTE-AUDIO.md +++ b/docs/SPEECH-REMOTE-AUDIO.md @@ -5,73 +5,202 @@ > composition. For the architecture of `speech9p` itself (file tree, engines, > data flow, agent integration), see > [SPEECH-ARCHITECTURE.md](SPEECH-ARCHITECTURE.md). +> +> **Phase 2 status.** The Mac-local voice mode +> ([SPEECH-VOICE-ONLY-PHASE1.md](SPEECH-VOICE-ONLY-PHASE1.md)) is Phase 1. +> Phase 2 starts after that release candidate passes its human exit checks. +> The loadable `SpeechEngine` ABI, provider-backed module, namespace audio +> routing, and `lib/voice/speech-*` launch scripts are already implemented; +> Phase 2 is now the validation, hardening, and distribution milestone rather +> than a green-field remote-audio implementation. +> +> **Security status: development only.** The current examples use anonymous +> `listen -A` / `mount -A` connections and export the host's broad `/dev` +> tree. That does not preserve InferNode's intended narrow, per-process +> namespace capabilities: any client that can reach the listener receives more +> authority than speech needs. A firewall, NAT, or private overlay can reduce +> reachability, but it does not provide capability attenuation inside +> InferNode. Keep these topologies limited to trusted development environments +> until the post-Phase 2 security audit replaces the broad exports with +> authenticated, speech-specific namespaces. + +## Phase 2 Definition and Gates + +Phase 2 comprises: + +1. Validate all three documented topologies with real hosts, including a Mac + terminal and Jetson/other inference host, device permissions, disconnects, + reconnects, latency, microphone capture, and playback. +2. Turn the existing launch scripts into a repeatable remote deployment recipe + with explicit helper/model prerequisites and observable failure states. +3. Add a reproducible, pinned Parakeet EOU conversion helper plus checksum + manifest. Uploading the GGUF remains a separate credential/hosting gate; + clean installs continue to use Whisper until a published model exists. +4. Replace Phase 1's daemon-local one-follow-up latch with server-owned bounded + queue state, visible depth, and queued-turn cancel/replace behavior. +5. Consider native 24000/48000 Hz audio only after topology validation; 22050 + Hz remains the supported default and higher rates are not a release blocker. + +The exact wake phrase/model is intentionally not a Phase 2 acceptance item. + +Automated loopback and mock coverage must precede each topology change. The +implementation stops for human acceptance when real two-host audio or GUI +latency judgement is required; those items are not marked complete from +non-interactive tests. + +Completing the functional Phase 2 gates does not make this transport +release-ready. A separate security-audit gate must verify authentication, +least-authority namespace exports, mount ownership and teardown, control-path +attenuation, and adversarial-client behaviour before the overarching voice +feature can merge into `dev`. + +### Implemented Foundation + +- `speechshim9p` takes playback and capture devices as namespace paths through + `audiodev`, `capturedev`, and `micmode device`. +- `speech9p` forwards routing controls and consumes a single provider mount. +- The loadable `SpeechEngine` `.dis` contract and provider reference module are + available in-tree. +- `/lib/voice/speech-terminal`, `speech-engine`, and `speech-capture` automate + the current export, import, provider, and ctl wiring. + +### Remaining Human Gates + +- Real cross-host audio with two machines and their device/network permissions. +- Jetson or equivalent remote helper/model installation and sustained use. +- Audible queue/backpressure UX and disconnect/recovery behaviour. ## Current Design -`speech9p` presents TTS/STT as a 9P filesystem at `/n/speech`: +`speech9p` presents the stable speech interface at `/n/speech` and consumes a +single **provider mount** (default `/n/speechshim`, served by `speechshim9p`) +for all streaming voice I/O — see the provider contract in +[SPEECH-ARCHITECTURE.md](SPEECH-ARCHITECTURE.md). Two properties make +remoting a pure composition exercise: -``` -/n/speech/ -├── ctl rw engine, voice, lang config -├── say rw write text → synthesized audio plays -├── hear rw write "start" → read transcription back -└── voices r list available voices -``` +1. **The provider is a mount.** `echo 'provider /n/x' > /n/speech/ctl` + points the whole voice pipeline (listen, wake, kokoro say, cancel) at any + namespace serving the contract — local shim, parakeet export, or a mount + from another Infernode instance across the network. +2. **The provider's audio I/O is namespace paths.** `speechshim9p` plays + through `audiodev` (default `/dev/audio`) and, in `micmode device`, + captures s16le PCM from `capturedev` (default: `audiodev`) and pumps it + into the STT/wake helpers' stdin. An imported `/dev/audio` from another + machine drops in with one ctl write — no `bind` required. -The current implementation assumes the user is **physically at the machine running -`speech9p`**. TTS output goes to the local `/dev/audio`; STT records from the local -`/dev/audio`. This is a coherent, reasonable deployment: macOS with `say`, or a Jetson -with espeak/piper/whisper, used as the user's workstation. +The default deployment assumes the user is physically at the machine running +the stack; the topologies below relocate the pieces. --- -## The Plan 9 Extension: Transparent Remote Audio +## The Three Topologies + +### 1. Everything local (default) -Because `speech9p` only ever touches its local namespace, audio I/O can be transparently -remoted by composing namespaces before the server starts — no changes to `speech9p` itself. +What `lib/lucifer/boot.sh` sets up: `speechshim9p` + `speech9p` on the local +machine, provider `/n/speechshim`, helpers (whisper.cpp stream, Kokoro, +openWakeWord) installed on the local host, `micmode helper` so the helper +CLIs grab the local microphone directly. A parakeet export mounted at +`/n/parakeet` is the same topology with a different provider value. -### Architecture +### 2. Remote processing, local microphone and speakers + +The local machine is the I/O terminal; a beefier host (Jetson, second +Infernode instance) runs STT/TTS. Audio is forwarded from the local mic and +played on the local speakers, but everything stays a locally mounted +namespace. ``` -GUI machine (Mac) Headless machine (Jetson) -───────────────── ───────────────────────── -/dev/audio ──── exported via 9P ──► /dev/audio (bound from Mac) - speech9p (uses /dev/audio transparently) - /n/speech ── exported via 9P ──► /n/speech - (mounted on Mac) -Lucifer GUI ──── writes to /n/speech/say ──────────────────────────────────────────► -◄──────────────── audio plays on Mac speakers ◄──────── PCM written to /dev/audio ── +Local terminal (mic + speakers) Remote engine (helpers installed) +─────────────────────────────── ───────────────────────────────── +listen -A 'tcp!*!17010' export /dev ───► mount ... /n/term + speechshim9p & + listen -A 'tcp!*!17019' export /n/speechshim +mount -A 'tcp!!17019' /n/remotespeech ◄──┘ +echo 'provider /n/remotespeech' > /n/speech/ctl +echo 'audiodev /n/term/audio' > /n/speech/ctl # resolved in the REMOTE namespace +echo 'micmode device' > /n/speech/ctl ``` -### Commands - -**Mac — export audio device (add to Lucifer launch script):** +**Local — export the audio device, mount the remote provider:** ```sh listen -A 'tcp!*!17010' export /dev & +mount -A 'tcp!!17019' /n/remotespeech +echo 'provider /n/remotespeech' > /n/speech/ctl ``` -**Jetson — import Mac audio, start speech9p, export it (Jetson launch script):** +**Remote — import the terminal's audio, serve the provider:** ```sh -mount -A 'tcp!!17010' /n/macaudio -bind /n/macaudio/audio /dev/audio -speech9p -e cmd & -listen -A 'tcp!*!17019' export /n/speech & +mount -A 'tcp!!17010' /n/term +speechshim9p & +listen -A 'tcp!*!17019' export /n/speechshim & ``` -**Mac — mount remote speech service:** +**Audio routing (writable from the local side — `speech9p` forwards these +keys to the provider's `ctl`):** ```sh -mount -A 'tcp!!17019' /n/speech +echo 'audiodev /n/term/audio' > /n/speech/ctl +echo 'micmode device' > /n/speech/ctl ``` -Or via the Lucifer catalog: add an entry with `dial=tcp!hephaestus!17019` and click `[+]`. +Now the remote shim synthesizes and recognizes, but reads its PCM from — +and plays it back to — the terminal's audio device over 9P. Note the +`audiodev` value is a path in the *remote* shim's namespace. + +The same topology is automated by two launch scripts: + +```sh +# Remote engine first: +sh /lib/voice/speech-engine tcp!!17010 + +# Local terminal second: +sh /lib/voice/speech-terminal tcp!!17019 +``` + +The terminal script reuses `/lib/voice/listen` (including audio pre-warm and +buffer caps), mounts the exported provider, selects it in `/n/speech/ctl`, and +keeps `duplex half`. The engine script imports the terminal device tree, starts +an isolated `speechshim9p` mount, selects device-fed PCM, and exports the +provider contract. Optional port and mount arguments are documented in each +script's usage header. + +### 3. Remote capture device (e.g. Infernode on an Android phone) + +The phone contributes only its microphone; processing and playback stay +wherever topology 1 or 2 put them. + +**Phone — export the device tree:** +```sh +listen -A 'tcp!*!17010' export /dev & +``` + +**Processing host (local machine in topology 1, remote engine in topology 2) +— import the phone's audio and use it for capture only:** +```sh +mount -A 'tcp!!17010' /n/phone +echo 'capturedev /n/phone/audio' > /n/speech/ctl +echo 'micmode device' > /n/speech/ctl +``` + +`capturedev` overrides capture without touching playback: the wake word and +speech come from the phone's mic while TTS still plays through `audiodev` +(the local speakers, or wherever topology 2 pointed it). Write +`capturedev default` to fall back to `audiodev` again. + +On the processing host, the import and ctl writes are automated as: + +```sh +sh /lib/voice/speech-capture tcp!!17010 +``` ### Why It Works -`speech9p` calls `open("/dev/audio")` and `write()`. Those are ordinary namespace -lookups. After `bind /n/macaudio/audio /dev/audio`, those calls transparently hit the -Mac's audio hardware over 9P. `speech9p` never knows or cares. This is standard Plan 9 -namespace composition — location transparency falls out of the model rather than being -bolted on as a special case. +The shim calls `open()` and `read()`/`write()` on the paths it was given. +Those are ordinary namespace lookups: after a 9P import, they transparently +hit the other machine's audio hardware. This is standard Plan 9 namespace +composition — location transparency falls out of the model rather than +being bolted on as a special case. The provider contract adds the same +property one level up: the entire speech engine is itself just a mount. --- @@ -79,52 +208,52 @@ bolted on as a special case. The final step — mounting the remote speech service — is **already supported** by the catalog `[+]` button (`mountresource()` calls `sys->dial()` + `sys->mount()`). A catalog -entry with the Jetson's address handles it. +entry with the Jetson's address handles it. The audio routing itself is now plain ctl +writes (`audiodev`, `capturedev`, `micmode` — no `bind` step remains), so once the +mounts exist, any shell or agent that can write `/n/speech/ctl` can rewire the audio +path. -The **audio bridge** (the prerequisite) is **not supported** by any current GUI or agent -pathway: +What is still **not supported** by any GUI or agent pathway is the setup on the other +hosts: | Step | Manual? | GUI? | Veltro? | |------|---------|------|---------| -| `listen export /dev` on Mac | yes (launch script) | ✗ | ✗ | -| `mount` Mac audio on Jetson | yes (launch script) | ✗ | ✗ | -| `bind /n/macaudio/audio /dev/audio` | yes | ✗ | ✗ | -| `speech9p &` on Jetson | yes (launch script) | ✗ | ✗ | -| `listen export /n/speech` on Jetson | yes (launch script) | ✗ | ✗ | -| Mount remote speech on Mac | via catalog `[+]` | ✓ | ✗ | +| `listen export /dev` on the mic/speaker host | `speech-terminal` | ✗ | ✗ | +| `mount` terminal/phone audio on the engine host | `speech-engine` / `speech-capture` | ✗ | ✗ | +| `speechshim9p &` + export on the engine host | `speech-engine` | ✗ | ✗ | +| Mount remote provider locally | via catalog `[+]` | ✓ | ✗ | +| `provider` / `audiodev` / `capturedev` / `micmode` ctl writes | yes (one-liners) | ✗ | ✓ (shell tool) | ### What Would Enable Full GUI/Agent Control -1. **`bind` tool** — a Veltro tool that calls `sys->bind(src, dst, flags)` in the *main* - namespace (not the restricted agent namespace). Currently `exec` runs in a restricted - namespace whose changes don't propagate back to Lucifer. +1. **`rcmd` / `ssh` tool** — to start services on the remote machine from Veltro. Without + this, Veltro cannot set up the engine-host side at all. -2. **`rcmd` / `ssh` tool** — to start services on the remote machine from Veltro. Without - this, Veltro cannot set up the Jetson side at all. +2. **Catalog multi-step connect** — extend the catalog entry format to support a sequence + of setup actions (dial, mount, ctl writes, spawn) rather than a single dial+mount. A + "Speech on Jetson" catalog entry could encode the full setup — including the + `provider` and audio-routing ctl writes — and execute it on `[+]`. -3. **Catalog multi-step connect** — extend the catalog entry format to support a sequence - of setup actions (dial, mount, bind, spawn) rather than a single dial+mount. A - "Speech on Jetson" catalog entry could encode the full setup and execute it on `[+]`. - -4. **Mount path for catalog entries** — `mountresource()` currently mounts to - `/tmp/veltro/mnt/`. Speech tools expect `/n/speech`. Either allow catalog entries - to specify a target path, or make speech tools check the catalog mount location. +3. **Mount path for catalog entries** — `mountresource()` currently mounts to + `/tmp/veltro/mnt/`. Since the provider mount point is itself a ctl value + (`provider `), this is a one-write fixup rather than a blocker. ### Recommended Approach (When Implementing) -Option A — **Launch script automation** (low effort, sufficient for now): -Bake the audio bridge into the Jetson's Lucifer launch command alongside `tools9p` and -`lucibridge`. Add the Mac `listen export /dev` to its launch command. The catalog entry -handles the final user-facing mount. +Option A — **Launch script automation** (implemented): +`/lib/voice/speech-terminal`, `/lib/voice/speech-engine`, and +`/lib/voice/speech-capture` perform the exports, mounts, provider startup, and ctl +writes. The scripts intentionally remain explicit operator commands; they do not +invent a remote-control authority or store remote credentials. Option B — **Catalog multi-step connect** (proper GUI solution): Extend `CatalogEntry` with a `setup: list of string` field. Each entry is a command -(`listen`, `mount`, `bind`, `exec`) run in sequence on `[+]`. The catalog file format -gains a `setup=` attribute. `mountresource()` runs the setup sequence before the final -mount. This generalises beyond speech to any multi-step remote service. - -Option C — **`rcmd` tool + `bind` tool** (Veltro-native solution): -Give the agent the tools it needs. `rcmd host cmd` runs a command on a remote Inferno -instance via authenticated 9P exec. `bind src dst` performs `sys->bind()` in the main -namespace. Then Veltro can set up the full pipeline autonomously once it knows the -remote host address. +(`listen`, `mount`, `echo ... > ctl`, `exec`) run in sequence on `[+]`. The catalog +file format gains a `setup=` attribute. `mountresource()` runs the setup sequence +before the final mount. This generalises beyond speech to any multi-step remote service. + +Option C — **`rcmd` tool** (Veltro-native solution): +Give the agent the tool it needs: `rcmd host cmd` runs a command on a remote Inferno +instance via authenticated 9P exec. The local half is already covered — the shell tool +can perform the mounts and ctl writes. Then Veltro can set up the full pipeline +autonomously once it knows the remote host address. diff --git a/docs/SPEECH-VOICE-ONLY-PHASE1.md b/docs/SPEECH-VOICE-ONLY-PHASE1.md new file mode 100644 index 000000000..5b25db157 --- /dev/null +++ b/docs/SPEECH-VOICE-ONLY-PHASE1.md @@ -0,0 +1,936 @@ +# Voice Mode for Lucia: Phase 1 + +Status: Phase 1 release candidate on `dev`. Automated implementation and the +composed voice-to-LLM-to-speech path are covered by blocking tests. Real +microphone/audio quality and physical GUI input remain human gates. Cross-host +acceptance is Phase 2. + +## Phase Boundary and Exit Criteria + +Phase 1 is the usable **single-host Mac voice interaction** milestone. It +includes every supported voice-mode entry/exit surface, live partial drafts, +final/grace/cancel handling, typed-compose preservation, provider-backed +Kokoro TTS, Parakeet streaming STT with a working Whisper fallback, +half-duplex echo safety, spoken approvals/refinements, low-confidence +confirmation, and one visibly capped busy-turn follow-up. It also includes the +installer/boot selection path, LLM-free speech testing, and explicitly selected +local OpenAI-compatible LLM backends. + +Phase 1 is complete only after all automated checks pass and a human confirms +the hardware and physical-interface behavior that CI cannot reproduce: + +1. Through the real emulator microphone path, Parakeet produces usable live + partial/final transcription and Kokoro playback is intelligible, natural + speed, and non-overlapping. +2. Voice chip, compose button, Ctrl+Space, Esc-V, and Option/Alt+V all toggle + the same mode and return cleanly to preserved keyboard input. +3. macOS microphone permission is granted to the actual launch context, the + microphone is released on exit, and Esc stops audible TTS in half-duplex + mode and returns to keyboard input. + +Grace, final deduplication, append/cancel, low-confidence confirmation, spoken +approval/refinement/denial, capped follow-up queuing, explicit LLM-provider +selection, and the full service composition are automated release gates. + +Two-host/Jetson deployment, public Parakeet EOU model distribution, rich queue +management, native 24000/48000 Hz playback, and any particular/custom wake-word +model are Phase 2 or later. See +[SPEECH-REMOTE-AUDIO.md](SPEECH-REMOTE-AUDIO.md). + +## Implementation Status + +Phase 1 structure (1.0–1.6) is implemented. The branch now has: + +- macOS `#A` audio enabled in the emulator config with a CoreAudio-backed + `emu/MacOSX/audio.c` implementation. +- `/dev/audioctl` support for 16000 Hz mono PCM, needed by speech capture. +- A host audio smoke test at `tests/host/audio_macos_test.sh`. +- `speech9p` additions for Kokoro helper config, `sayq`, `cancel`, `listen`, + and `wake` files. Helper binaries are still external and may be absent. +- A unified speech-provider architecture: `speech9p` consumes exactly one + provider mount (contract in docs/SPEECH-ARCHITECTURE.md — listen/wake/say/ + cancel plus optional ctl/voices) and runs no helper binaries itself. The + in-tree `speechshim9p` adapts the external helper CLIs to that contract + and is the boot-time default provider at `/n/speechshim`; a parakeet + export or a remote 9P mount is the same one-line ctl switch. The shim + hard-cancels active TTS by killing the helper process (devcmd `kill`), + so cancellation silence is bounded by one audio chunk. +- Asynchronous serving of `listen`, `wake`, `hear`, and in-flight `say`/`sayq` + result reads in `speech9p`, with Flush/Clunk cancellation of parked reads. + A wake read blocked in a helper no longer freezes the serveloop, so `cancel` + writes, `ctl`, and `sayq` stay live while helpers run. +- `luciuisrv` support for `/n/ui/input-mode` and + `conversation/voiceinput`, so keyboard mode can be paused while + voice-originated turns still have a privileged path into `lucibridge`. + Input-mode changes are broadcast on the global `event` stream. +- `lucibridge` support for `/voice mode on|off`, while preserving existing + `/voice on|off` auto-speak behavior. Keyboard and voice input are read + concurrently, so a mode switch takes effect immediately instead of after + the next message on the previously selected path; typed plain text is + paused (with a notice) while voice mode is active, but typed slash + commands — including `/voice mode off` — still work. +- A resident `voicemode` daemon, pre-spawned at boot in an idle state. It + activates on the `input-mode v` broadcast (with an input-mode poll + fallback when the event stream is unavailable), runs the + WAITING_WAKE → LISTENING → PROCESSING/SPEAKING loop, handles spoken control + intents (stop/cancel, keyboard, approve/deny), and returns to idle on + `input-mode k`. The shipped half-duplex mode suppresses microphone capture + during TTS; explicit full-duplex mode may use wake as spoken barge-in. +- `Esc` exits voice mode: `lucifer`'s kbdproc tracks input-mode from global + events and writes `k` back to `/mnt/ui/input-mode` on Esc, which fans out + to `voicemode` (cancels speech, idles) and `lucibridge` (resumes typing). +- Boot wiring in `lib/lucifer/boot.sh`: `speech9p` starts before + `lucibridge` (so the speech resource registers), and `voicemode` is + pre-spawned idle. Deviation from the original plan: the pre-spawn lives in + `boot.sh` rather than `lucifer.b`, because boot.sh is where the sibling + services (`luciuisrv`, `tools9p`, `lucibridge`) already start. +- `module/speech.m` streaming `Partial` record type with the documented + `partial`/`final`/`error:` wire format for `/n/speech/listen`. +- A repo-owned `whisper-stream-cli --stdin` adapter with energy VAD, partial + snapshots, final records, and aggregate confidence metadata. The installer + deploys it over the batch-only Homebrew `whisper-cli` without reopening the + host microphone for every utterance. +- Completion-aware FIFO TTS in `lucibridge`, sentence-boundary streaming, + first-token/first-audio timings, and cancellation that invalidates queued + speech before the next turn. +- Live transcription drafts at `conversation/draft`, paired with an explicit + `conversation/draft-status`, and rendered as a bordered, visibly unsent user + turn. Typed compose text remains visible but locked until voice mode exits. +- A FIFO `conversation/control` path for spoken cancel, pause, resume, status, + and mid-turn refinements. Tool approvals consume the same voice input path, + fail closed, and remain cancellable. +- Low-confidence STT confirmation with a visual prompt and spoken read-back; + the threshold defaults to 650 permille and is configurable with + `voicemode -q`. +- Namespace-composable cross-host launch scripts under `/lib/voice`, plus a + loadable `SpeechEngine` `.dis` ABI and provider-backed reference module. +- A send **grace window** (default 3 s, `voicemode -g`, 0 = immediate): a + completed utterance remains in the unsent conversation turn with an explicit + countdown before submission. The compact Voice resource shows state only; + saying "cancel" (or Esc) discards the turn, and more speech appends to it and + restarts the window. An explicitly confirmed low-confidence transcript skips + the window. +- Parakeet realtime STT as the preferred default: the InferNode-owned + `tools/parakeet_stream.cpp` adapter streams stdin PCM through the + cache-aware `parakeet_realtime_eou_120m-v1` model, so end-of-utterance is + detected by the model rather than by energy-VAD silence. The installer + builds it when possible and writes the chosen stack to + `speech.ctl.sh` (applied by boot.sh); the whisper wrapper remains the + fallback. Boot also selects `engine kokoro` so assistant speech uses + Kokoro instead of the robotic host `say`. +- One voice affordance: the compose-row button (formerly press-to-dictate + "mic") and Ctrl+Space in the conversation view now toggle the same voice + mode as `Esc-V`, Option/Alt+V, and the Voice chip. The one-shot + dictation-into-compose pathway is removed. +- One busy-turn follow-up may be queued for refinement. Its Voice resource is + visibly `queued`; additional finals are discarded with a visible + `one turn queued` status until the activity returns idle, preventing an + unbounded spoken backlog against a slow agent. +- Tests: `tests/speech_wake_test.b`, `tests/speech_listen_test.b`, + `tests/speech_kokoro_test.b` (fake host helpers via ctl; includes + serveloop-liveness assertions), and `tests/voicemode_test.b` (daemon state + machine against a mock file tree), alongside the earlier + `tests/speech9p_voice_test.b` and `tests/luciuisrv_test.b` coverage. +- A hermetic composed E2E test at `tests/host/speech_e2e_test.sh` runs the real + `luciuisrv`, `voicemode`, `speech9p`, `speechshim9p`, `llmsrv`, and + `lucibridge` services. Deterministic host speech helpers and a loopback + OpenAI-compatible server replace only hardware and external models. It + exercises partial/final transcription and verifies exactly-once final + submission, explicit local provider/model selection with unrelated API keys + removed, the Lucia reply, TTS delivery, the Voice resource, and microphone + release. The test is part of `tools/speech-regress.sh` and therefore blocks + CI. + +Remaining human acceptance: + +- Install the real helper models/binaries, grant macOS microphone permission, + and confirm Parakeet/Kokoro quality through the emulator's actual audio path. +- Exercise each physical GUI/keyboard entry surface once and confirm preserved + keyboard input, microphone release, and audible Esc cancellation. + +The specific/custom wake model is intentionally deferred. Half-duplex is the +shipped echo-safety default; full-duplex spoken barge-in remains an opt-in for +echo-controlled/headset setups. Hard cancellation of arbitrary tools remains +future work, while TTS helper processes are hard-cancelled through +`speechshim9p`. + +This document captures Claude's analysis findings, the decisions confirmed with +the user, and the approved Phase 1 plan for a local, Mac-only voice mode in +Lucia. It was written before implementation so the branch has a stable target +and reviewers can see both the intended architecture and the reasoning behind +it. + +## Goal + +Lucia is currently keyboard-driven. Phase 1 adds a hands-free mode: + +1. Start Lucia. +2. Type `/voice mode on`. +3. Say the configured wake phrase. +4. Speak an utterance. +5. Lucia transcribes it, sends it through the existing conversation path, and + speaks the assistant response. +6. Press `Esc` during speech to cancel TTS and return to keyboard mode. + +Phase 1 runs entirely on one Apple Silicon Mac using the laptop microphone and +speakers. Remote inference, namespace audio routing, and pluggable speech +engine modules were subsequently implemented as additive follow-on work. + +## Existing Starting Point + +The repo already has useful scaffolding: + +- `module/speech.m` defines batch TTS/STT data structures and engine interfaces. +- `appl/veltro/speech9p.b` exposes `/n/speech` with `ctl`, `say`, `hear`, and + `voices`. +- `appl/cmd/lucibridge.b` has `/voice on|off`, but today that only toggles + auto-speak for typed assistant responses. +- `docs/SPEECH-REMOTE-AUDIO.md` describes a later remote-audio direction. + +The current gaps are: + +- macOS has no active `/dev/audio` driver. `emu/MacOSX/emu` still comments out + `audio audio`, and there is no `emu/MacOSX/audio.c`. +- `speech9p` is batch-oriented. It has no streaming listen file, wake-word file, + queued TTS file, or cancellation file. +- STT is not wired into Lucia's conversation input. +- The UI has no voice-mode state machine and no mutual exclusion between typed + input and voice input. +- Current local engines are oriented around `say`, Piper, and batch + `whisper-cli`; Phase 1 wants Kokoro for TTS and streaming whisper.cpp for STT. +- No shipped engine implementation modules exist under `appl/lib/speech*`; the + `cmd`, `api`, and local behavior is implemented inside `speech9p.b`, not as + separately loadable `.dis` engines. +- The existing voice path is response-only: `lucibridge` gets assistant text and + calls `speaktext()`. There is no current voice-in / voice-out loop. +- `docs/SPEECH-REMOTE-AUDIO.md` assumes missing remote plumbing exists. It + names a useful direction, but Phase 1 cannot depend on that path yet. + +## Analysis Findings to Preserve + +Claude evaluated three deployment shapes before the final plan: + +| Shape | Finding | +| --- | --- | +| Mac-only, all-local | Best Phase 1 target. Needs macOS `/dev/audio`, local engines, streaming/VAD, and Lucia voice-mode integration. Fastest to dogfood. | +| Mac as I/O terminal, Jetson as inference host | Good later target. Still needs macOS `/dev/audio`, plus remote audio, 9P export/bind, `rcmd`, catalog connection, and mount-path config. Too much for Phase 1. | +| Headless Jetson with USB headset | Avoids macOS audio by using Linux audio drivers, but does not match the current goal of opening the existing Mac UI and talking to it. | + +The chosen ordering is Mac-local first, then Jetson/remote later. Reasons: + +- The macOS audio driver is a shared prerequisite for both Mac-local and + Mac-I/O-plus-Jetson deployments. +- Mac-local work is dogfoodable without a second machine, network setup, SSH, or + 9P export ceremony. +- It avoids combining three hard problems at once: CoreAudio driver work, new + speech/voice UX, and unfinished remote Plan 9 plumbing. +- Streaming API design, VAD timing, and barge-in semantics are easier to tune on + a single low-latency local loop before adding network latency. + +Audio-layer finding: + +- The audio I/O belongs in the host audio device layer exposed as `/dev/audio`, + not in SDL3. SDL3 may own GUI/event work, but speech capture/playback needs to + be available to Inferno programs and `speech9p` as a normal device. + +Engine-layer finding: + +- A full `.dis` plugin system for `TTSEngine`/`STTEngine` should wait until + Phase 2. Phase 1 should add Kokoro and streaming STT directly in `speech9p.b` + because that matches the current implementation shape and keeps the first + usable path smaller. + +Remote-audio finding: + +- The remote-audio document is directionally useful, but its prerequisites are + not implemented: 9P export of `/dev/audio`, bind tooling, `rcmd`, multi-step + catalog connection, and mount-path configuration. Treat all of that as Phase + 2 plumbing. + +## Confirmed Decisions + +| Decision | Phase 1 choice | +| --- | --- | +| Deployment target | Local Mac only, Apple Silicon first | +| Activation | `/voice mode on` and `/voice mode off` | +| Wake behavior | Use the phrase supplied by the configured wake model | +| Wake model | Model identity is not a Phase 1 acceptance gate | +| Keyboard behavior | Mutually exclusive; typing is paused while voice mode is active | +| Escape hatch | `Esc` always returns to keyboard mode | +| STT model | whisper.cpp `base.en` by default | +| TTS engine | Kokoro, default voice `af_bella` | +| Host helpers | External install; no vendored binaries | +| UI mic button | Deferred to Phase 1.x | +| Jetson / remote audio | Deferred to Phase 2 | + +## Architecture + +Five layers, bottom-up. Phase 2 should replace only layer 2 and add remote +transport; layers 1, 3, 4, and 5 should still apply. + +| Layer | Component | Phase 1 work | +| --- | --- | --- | +| 1 | macOS audio driver | Add `emu/MacOSX/audio.c`; expose `/dev/audio` and `/dev/audioctl` | +| 2 | Host speech engines | Use Kokoro, streaming whisper.cpp, and openWakeWord helpers | +| 3 | `speech9p` | Extend `/n/speech` with streaming, wake, queue, and cancel files | +| 4 | `voicemode` daemon | New state machine that bridges speech events into Lucia input | +| 5 | Lucia UI integration | Add `/voice mode on|off`, status resources, paused typing | + +Data flow: + +```text +Lucia UI + /voice mode on|off + status resources: waiting, listening, processing, speaking + ^ + | writes transcript to /n/ui/activity/{id}/conversation/input +voicemode daemon + IDLE -> WAITING_WAKE -> LISTENING -> PROCESSING -> SPEAKING + ^ + | /n/speech/wake, /n/speech/listen, /n/speech/sayq, /n/speech/cancel +speech9p + ^ + | /dev/audio and /dev/audioctl +macOS CoreAudio driver +``` + +## State Machine + +```text + /voice mode on + | + v + IDLE <------------------------- /voice mode off, Esc + | + v + WAITING_WAKE + typing paused; wake-word engine active + | + | wake word detected + v + LISTENING + streaming STT and VAD + | + | final transcript + v + PROCESSING + transcript injected into Lucia conversation input + | + | assistant response begins + v + SPEAKING + Kokoro TTS playback + | + | playback done + v + WAITING_WAKE + +During SPEAKING, wake-word detection remains active. A wake event writes +`cancel` to `/n/speech/cancel`, cuts off TTS, and transitions to LISTENING. +``` + +Mutual exclusion with typing uses a new single-byte UI file: + +- `/n/ui/input-mode` returns `k` for keyboard mode or `v` for voice mode. +- `voicemode` writes `v` when it enters voice mode and `k` when it exits. +- `lucibridge` checks the file before blocking on conversation input; when it + sees `v`, it sleeps briefly and rechecks instead of consuming typed input. + +## Seamless Voice-Only UX Requirements + +The baseline architecture routes voice through the existing Lucia conversation +and tool infrastructure by injecting the final transcript into +`/n/ui/activity/{id}/conversation/input`. That is the correct foundation: a +voice utterance should become the same kind of user turn as a typed message, so +the existing agent loop, native tool-use protocol, resource updates, and context +zone activity can be reused. + +That routing is not, by itself, the full UX contract. The implementation must +also preserve the visible action feedback that typed/tool-driven Lucia already +has. Voice mode should not become a separate hidden path where the user hears +speech but cannot see which files, tools, resources, or activities are being +used. + +The target user experience must not feel turn-based. The internal LLM/tool loop +may still process discrete turns, but voice mode should present a continuous +conversation and control surface: Lucia keeps listening for interruption, +correction, cancellation, or follow-up intent while work is underway. + +Normal-mode UX: + +- Voice-triggered agent turns must use the same context/resource activity UI as + typed turns. Tool calls should mark the relevant tool resource active, upsert + touched file/path resources, and return them to idle/done/error using the + existing `lucibridge` context update paths. +- Speech tiles are only the audio state: waiting, listening, processing, + speaking. They do not replace the context/resource activity view for actual + work. +- When the user says "hey lucia, do X", the expected normal experience is: + wake fires, Lucia accepts the utterance, the existing conversation/tool loop + starts, and the context zone reflects the same tool/resource activity the user + would see if they had typed "do X". +- Do not force the user into a strict speak-wait-hear-next cycle. After Lucia + has enough confidence in the utterance or command intent, it should begin + acting while still allowing the user to interrupt, refine, or cancel. +- Voice mode should support mid-action follow-ups. Examples: "actually use the + other file", "stop", "show me what changed", "continue", "don't promote that", + or "read the error". These should be routed as control intents against the + active task where possible, not treated as unrelated new chat turns. +- Barge-in is not only for stopping TTS. It is the general mechanism for taking + conversational control back from the system while it is speaking, using tools, + waiting on a long operation, or asking for approval. +- TTS should not block listening. The system should be able to speak while a + lower-volume wake/barge-in listener remains active, with echo handling or + cancellation sufficient to avoid hearing itself as a new command. +- Destructive or sensitive operations still require approval. Voice mode needs + a spoken and visual approval path before write/edit/exec-style operations can + proceed. +- Long-running actions need voice control hooks while work is in progress: + cancel, pause, resume, repeat status, and barge-in should be recognized even + while tools or TTS are active. +- Spoken UI commands need intent mapping rather than assuming literal slash + commands. For example, "bind this folder", "add grep", "show diff", and + "promote this change" should map to the same existing slash/tool operations + with confirmation where ambiguity or risk exists. +- Misheard or ambiguous commands need a correction flow. For risky or + irreversible actions, ask for confirmation using the interpreted command: + "I heard: promote changes under X. Confirm?" +- If speech recognition confidence is high and the command is low risk, avoid + confirmation chatter. If confidence is low or the action is risky, confirm + before acting. +- Assistant speech should be streamed or chunked so the user hears progress + quickly, but tool/action execution should not wait for the whole spoken + response to finish when the next operation is already clear. + +Continuous-loop implications: + +- `voicemode` needs a control channel into the active agent/task loop, not only a + one-shot transcript injection path. At minimum, it must be able to cancel TTS, + cancel or pause an active tool/task where supported, and submit a follow-up + utterance against the current activity. +- The agent loop should expose enough state for voice mode to know whether Lucia + is idle, listening, generating, speaking, waiting for approval, executing a + tool, or blocked on an error. +- Voice mode should preserve a current "interaction focus" so short follow-ups + like "stop", "that one", "open it", or "undo that" apply to the active file, + tool, approval prompt, or task rather than starting from scratch. +- The normal UI should make continuous state legible without debug clutter: + listening, acting, waiting-for-approval, speaking, paused, and error are + enough for the default surface. + +Debug-mode UX: + +- Live partial text is user-facing in the bordered unsent conversation turn so + the speaker can detect omissions before submission. Debug mode may add raw + provider/confidence detail, but hypotheses must never overflow the compact + Voice resource row or overwrite the preserved keyboard compose buffer. +- Action lifecycle visualization should be debug-first: heard -> interpreting -> + acting -> done/error. In normal mode, prefer the existing conversation, + speech-state tile, and context/resource activity signals unless the user asks + for verbose diagnostics. +- Debug mode should expose timing markers for wake detection, VAD finalization, + transcript finalization, first token, first audio, tool start, and tool end so + latency problems can be diagnosed without cluttering the default UI. + +## Implementation Phases + +### Phase 1.0: macOS Audio Driver + +Why first: all low-latency local speech depends on working `/dev/audio` on +macOS. + +Files: + +- New `emu/MacOSX/audio.c`. +- Edit `emu/MacOSX/emu` to enable `audio audio`. +- Edit `emu/MacOSX/mkfile` to link the required Apple audio frameworks. +- Check whether SDL3/headless macOS mkfiles have independent `SYSLIBS` entries. + +Driver contract: + +- Implement the functions declared in `emu/port/audio.h`: + `audio_file_init`, `audio_file_open`, `audio_file_read`, + `audio_file_write`, `audio_ctl_write`, `audio_file_close`, and + `getaudiodev`. +- Reuse `emu/Linux/audio-oss.c` as the closest structural template. Match its + separate input/output locking shape and pause-state handling where applicable. +- Rely on `emu/port/devaudio.c` for the Inferno device registration once the + macOS object is in the build. +- Use CoreAudio / AudioQueue APIs from C. Claude recommended AudioQueue over + higher-level Objective-C APIs because it is C-callable, maps cleanly onto the + existing host driver contract, and should support roughly 10-20 ms audio + latency. +- Consult `emu/port/audio-tbls.c` for existing audio control value tables and + map macOS devices/formats into those semantics rather than inventing a new + control grammar. + +Driver approach: + +- Capture should configure an input AudioQueue at the Inferno-side STT-friendly + default of 16000 Hz mono S16LE when possible. The capture callback feeds a + ring buffer drained by `audio_file_read`. +- Playback should configure an output AudioQueue and a ring buffer fed by + `audio_file_write`. The default playback format should match + `Default_Audio_Format` unless `/dev/audioctl` changes it. +- Use AudioConverter or equivalent conversion when the hardware device rate or + channel count differs from the Inferno-side format. +- Keep the implementation in C if possible; do not introduce Objective-C files + unless the CoreAudio C API proves insufficient. + +Implementation order inside Phase 1.0: + +1. Scaffolding commit: enable the audio build entry, add a compiling stub + `audio.c`, link frameworks, and prove `o.emu` still builds and starts. +2. Playback commit: implement `audio_file_write` and verify raw PCM playback + through `/dev/audio`. +3. Capture commit: implement `audio_file_read` and verify record-then-playback. +4. Control commit: implement `audio_ctl_write` for rate, channels, encoding, + device, and volume where practical. + +Verification: + +- Build the emulator. +- Build command bytecode with native `mk`. +- Record three seconds from `/dev/audio`, then play the raw data back through + `/dev/audio`. +- Add `tests/host/audio_macos_test.sh` once there is a stable host smoke path. + +### Phase 1.1: Kokoro TTS in `speech9p` + +Why now: the current `/n/speech/say` path already exists, and adding a Kokoro +backend is smaller than introducing a full engine plugin system. + +Files: + +- Edit `appl/veltro/speech9p.b`. + +Planned changes: + +- Add `ENGINE_KOKORO` next to existing `cmd`, `api`, and `local` engines. +- Add a `saykokoro` path modeled on the existing local TTS path. +- Stream raw PCM output to `/dev/audio` instead of writing temporary files. +- Extend `ctl` parsing so `engine kokoro` and `voice af_bella` work. +- Keep existing `say`, `hear`, `ctl`, and `voices` behavior compatible. +- Pass Kokoro voice IDs through directly, for example `af_bella` and `am_adam`. + +Host helper: + +- Prefer an external `kokoro-onnx` install plus a thin wrapper command. +- Do not vendor model binaries or Python dependencies into the repo. +- A checked-in thin wrapper such as `bin/kokoro-cli` is acceptable if it only + adapts the external install to the raw PCM contract; avoid C++ bindings in + Phase 1. + +Verification: + +- Start `speech9p`. +- `echo 'engine kokoro' > /n/speech/ctl`. +- `echo 'hello world' > /n/speech/say`. +- Confirm audible speech and first-audio latency under 500 ms on target Mac. + +### Phase 1.2: Streaming STT + +Files: + +- Edit `appl/veltro/speech9p.b`. +- Edit `module/speech.m` additively. + +Planned changes: + +- Add a streaming `Partial` concept with text plus final/non-final state. +- Add `/n/speech/listen`: blocking reads return transcript partials as they + arrive. +- Keep `/n/speech/hear` as the existing batch path. +- Wrap the whisper.cpp streaming binary or an equivalent small helper. +- Treat `/n/speech/listen` as per-fid state. Each open reader should have its + own stream/channel so unrelated clients do not consume each other's partials. +- The wire format should be documented beside the code; Claude's plan used a + simple `Partial` ADT shape with `text` and `isfinal`. + +Why streaming matters: + +- Voice mode should send the LLM request as soon as VAD detects end-of-speech, + not after a full tempfile-based whisper pass completes. +- By the time VAD fires, the final transcript should be at most one chunk away. + +Verification: + +- `cat /n/speech/listen`. +- Speak into the Mac microphone. +- Confirm partials arrive and one final transcript is emitted after + end-of-speech. + +### Phase 1.3: Wake Word and VAD + +Wake word: + +- Use openWakeWord or an equivalent local helper. +- Phase 1 may use a placeholder wake model while preserving `hey lucia` as the + user-facing phrase. +- Add a `wakemodel` ctl key for the model path. +- Claude preferred openWakeWord because it is open source, ONNX-based, + lightweight, and supports custom-trained wake words. +- Porcupine was rejected for Phase 1 because its useful tiers require an + account-bound access key and commercial-use constraints. +- Whisper-based wake detection was rejected because continuously transcribing to + regex-match the wake phrase costs more CPU and has worse latency than a + purpose-built wake-word model. + +VAD: + +- Use whisper.cpp streaming VAD first. +- Treat a separate Silero VAD helper as Phase 1.x only if whisper.cpp VAD is + not good enough. +- Start with a configurable threshold around `vadthold 0.6`; tune from real + microphone tests rather than hardcoding assumptions. + +Files: + +- Edit `appl/veltro/speech9p.b`. + +Planned changes: + +- Add `/n/speech/wake`: reads block until wake-word detection, then return a + line containing model, score, and timestamp. +- Add `/n/speech/cancel` for TTS interruption. +- Add `/n/speech/sayq` for queued or streaming response playback. + +Verification: + +- `cat /n/speech/wake`. +- Say `hey lucia`. +- Confirm a wake event appears. +- Confirm background speech/noise does not trigger at the configured threshold. + +### Phase 1.4: `voicemode` Daemon + +Files: + +- New `appl/cmd/voicemode.b`. +- Edit `appl/cmd/mkfile`. + +Responsibilities: + +- Own the voice-mode state machine. +- Watch `/n/speech/wake`. +- Read `/n/speech/listen`. +- Inject final transcripts into `/n/ui/activity/{id}/conversation/input`. +- Queue assistant response chunks to `/n/speech/sayq`. +- Write `/n/speech/cancel` for barge-in. +- Write `/n/ui/input-mode` to pause and resume typed input. +- Update UI context resources for waiting, listening, processing, and speaking. +- Treat `Esc` or `/voice mode off` as unconditional exit. +- Determine the active activity ID from `/n/ui/active` if available; otherwise + fall back to the most recent activity under `/n/ui/activity`. +- While processing, watch for the next assistant response so response chunks can + be queued to speech as they arrive. + +Open implementation detail: + +- Confirm whether `luciuisrv` already exposes global key events. If not, add a + minimal `/n/ui/keys` file that `voicemode` can read for `Esc`. + +Verification: + +- Run `voicemode` manually. +- Flip voice mode on. +- Confirm transcript injection reaches the existing Lucia conversation path. +- Confirm state resources update. +- Confirm barge-in cancels speech and returns to listening. + +### Phase 1.5: Lucia Integration + +Files: + +- Edit `appl/cmd/lucibridge.b`. +- Edit `appl/cmd/luciuisrv.b`. +- Edit `appl/cmd/lucifer.b`. + +Planned changes: + +- Extend `/voice` parsing: + - `/voice on|off` remains auto-speak for typed interactions. + - `/voice mode on|off` becomes the new hands-free state. +- Add pause-aware conversation input in `lucibridge`. +- Add `/n/ui/input-mode` in `luciuisrv`. +- Add or reuse a key-event stream for `Esc`. +- Pre-spawn `voicemode` in an idle state from `lucifer` to avoid first-use + startup latency. +- Register voice-mode status resources alongside the existing speech resource, + using the same resource-tile pattern already present in Lucia. + +Verification: + +- `/voice mode on` pauses typed input and activates voice resources. +- `/voice mode off` cancels in-flight speech and resumes typing. +- `Esc` resumes typing even while audio or speech helpers are active. +- Existing `/voice on|off` auto-speak behavior still works. + +### Phase 1.6: Tests and Docs + +Tests: + +- New `tests/host/audio_macos_test.sh`: host-side audio smoke test. +- New `tests/speech_kokoro_test.b`: `engine kokoro` TTS smoke path. +- New `tests/speech_listen_test.b`: streaming STT partial/final behavior. +- New `tests/speech_wake_test.b`: wake-word event behavior. +- New `tests/voicemode_test.b`: daemon state-machine behavior with mocked + speech files. + +Docs: + +- Keep this file updated as implementation reality changes. +- Add a short Phase 2 pointer to `docs/SPEECH-REMOTE-AUDIO.md` after Phase 1 + structure is stable. + +## Host Dependencies + +Use the repo installer to prepare the host helpers and print the Inferno ctl +configuration block: + +```sh +tools/install-speech-helpers.sh +``` + +The installer is macOS-first and safe to re-run. It installs Homebrew +`whisper-cpp` when Homebrew is available, creates an isolated venv under +`~/.local/share/infernode-speech/venv`, installs pinned `kokoro-onnx` and +`openwakeword` packages, downloads the Kokoro model/voices and a whisper.cpp +`base.en` model, and generates these provider-contract wrappers: + +```text +~/.local/share/infernode-speech/bin/kokoro-cli +~/.local/share/infernode-speech/bin/whisper-stream-cli +~/.local/share/infernode-speech/bin/openwakeword-cli +``` + +After `/n/speech` is mounted, paste the ctl block printed by the installer. +The helper-mode block has this shape: + +```sh +echo 'kokorobin /Users/me/.local/share/infernode-speech/bin/kokoro-cli' > /n/speech/ctl +echo 'whisperstreambin /Users/me/.local/share/infernode-speech/bin/whisper-stream-cli' > /n/speech/ctl +echo 'wakebin /Users/me/.local/share/infernode-speech/bin/openwakeword-cli' > /n/speech/ctl +echo 'whispermodel /Users/me/.local/share/infernode-speech/models/ggml-base.en.bin' > /n/speech/ctl +echo 'voice af_bella' > /n/speech/ctl +echo 'wakeword hey jarvis' > /n/speech/ctl +echo 'wakethreshold 0.5' > /n/speech/ctl +echo 'duplex half' > /n/speech/ctl +``` + +Then start InferNode, press `Alt+V` (or Esc then `v`) to enter voice mode, and +speak the wake phrase. + +**The spoken wake phrase is currently "hey jarvis"**, because the only +pretrained openWakeWord model available today is `hey_jarvis`. Saying +"hey lucia" will not trigger wake until a custom hey-lucia model is trained +and dropped into `~/.local/share/infernode-speech/models/openwakeword/` +(pass it explicitly with `--model` in the `wakebin` command). The +`whisper-stream-cli` wrapper runs whisper.cpp in VAD mode: each utterance is +transcribed after you stop speaking and emitted as a single `final` record — +so expect turn latency of roughly the utterance length plus transcription +time, and no live partials from this particular wrapper (a parakeet provider +supplies partials). + +Topology 2/3 users can keep the microphone as a namespace device instead of +letting helper CLIs grab the host mic directly: + +```sh +echo 'micmode device' > /n/speech/ctl +``` + +`micmode device` means `speechshim9p` pumps 16 kHz s16le mono PCM into the +configured helper commands' stdin and supplies their `--stdin`, model, phrase, +threshold, rate, and channel arguments. The generated openWakeWord wrapper +consumes that stream directly. Because Homebrew `whisper-stream` only captures a +host device, the generated Whisper wrapper uses the repo-owned stdin adapter: +energy VAD segments the PCM and `whisper-cli` transcribes snapshots into +`partial confidence=N ` and `final confidence=N ` records. This +keeps remote/topology-2/3 capture namespace-backed without reopening a host mic. + +Host smoke coverage lives in `tests/host/speech_helpers_test.sh`. It exercises +TTS/PCM and no-mic wrapper paths only; microphone-dependent wake/STT checks are +left to an interactive TCC-approved session. + +## End-to-End Verification Target + +After all Phase 1 work lands: + +1. Build native prerequisites and emulator. + + ```sh + export ROOT=$PWD + export PATH=$PWD/MacOSX/arm64/bin:$PATH + ./scripts/bootstrap-libs.sh + ./build-macos-sdl3.sh + cd appl/cmd && mk install + ``` + +2. Run host audio smoke test. + + ```sh + ./tests/host/audio_macos_test.sh + ``` + +3. Run speech tests. + + ```sh + ./emu/MacOSX/o.emu -r. /tests/speech_kokoro_test.dis + ./emu/MacOSX/o.emu -r. /tests/speech_listen_test.dis + ./emu/MacOSX/o.emu -r. /tests/speech_wake_test.dis + ./emu/MacOSX/o.emu -r. /tests/voicemode_test.dis + ``` + +4. Run manual voice-mode smoke test. + + ```text + ./run-lucia.sh + /voice mode on + say: [configured wake phrase], what time is it? + expected: transcript appears as user input, assistant responds, response is spoken + press Esc while response is speaking + expected: TTS stops and voice mode returns to keyboard input + enter voice mode again, then press Esc + expected: voice mode exits and typing resumes + ``` + +Acceptance targets: + +- Wake-word detection latency: under 200 ms from utterance end to wake event. +- End-of-speech to final transcript: under 800 ms with whisper.cpp `base.en`. +- Assistant first token to first audio: under 500 ms with Kokoro. +- Esc cancellation to TTS silence: under 200 ms in the shipped half-duplex + mode. Spoken barge-in requires explicit full-duplex opt-in. + +## Testing the Speech Loop Without an LLM + +`tools/speech-test.sh` boots InferNode headless in a speech test mode +that exercises the entire microphone → STT → TTS loop with no LLM, no +GUI, no login, and no API key. Live partial transcripts print to the +terminal as words are spoken, and every non-junk final transcript is +answered by speaking a hard-coded phrase (or the transcript itself with +`-e`). Use it to validate a helper install or an audio topology before +involving a model. + +```sh +tools/speech-test.sh # local helpers, defaults +tools/speech-test.sh -p 'Hello from InferNode' # custom phrase +tools/speech-test.sh -e -n 3 # echo transcripts, exit after 3 turns +``` + +Remote topologies compose the same way as in +[SPEECH-REMOTE-AUDIO.md](SPEECH-REMOTE-AUDIO.md) — mount the remote +export, then point ctl keys at it (mounts are unauthenticated; trusted +networks only): + +```sh +# Remote STT+TTS provider (topology 2): +tools/speech-test.sh --no-helpers \ + -M 'tcp!fast-box!7770 /n/remotespeech' -c 'provider /n/remotespeech' + +# Remote microphone (topology 3, e.g. InferNode on a phone): +tools/speech-test.sh \ + -M 'tcp!phone!7771 /n/phoneaudio' \ + -c 'capturedev /n/phoneaudio/audio' -c 'micmode device' +``` + +The wrapper drives `/dis/speechtest.dis` (`appl/cmd/speechtest.b`), +which bootstraps `speechshim9p` + `speech9p` in its own namespace when +`/n/speech` is not already served — so the same command also works from +a shell inside a booted GUI session, where it reuses the live stack. +The terminal app needs macOS microphone permission (TCC) for local +capture. Unit tests: `tests/speechtest_test.b`. + +### GUI variant + +`tools/speech-test.sh --gui` boots the full lucifer desktop with the +same LLM-free guarantee, via `/lib/lucifer/boot-speechtest.sh` (the +boot-mobile.sh pattern: set variables, `run` the canonical boot.sh). +The login screen is skipped (`skiplogon=1` — no keys are needed since +nothing calls the LLM) and `voicemode` starts in test mode +(`-p phrase`, plus `-e` when given): entering voice mode (Esc-V or a +Voice-chip click), the configured wake phrase, live partials in the Voice +chip, chimes, and half-duplex echo suppression behave as in production, but a +final transcript is posted to the conversation as a "Heard" dialogue +line and answered by speaking the canned phrase instead of becoming an +LLM turn. Spoken control intents ("stop", "keyboard", …) still work. +`voicemode` runs with `-d`; its trace is in `/tmp/voicemode.log` inside +the emu namespace. + +```sh +tools/speech-test.sh --gui # full desktop, canned phrase +tools/speech-test.sh --gui -e # …echo the transcript instead +tools/speech-test.sh --gui -p 'Copy that.' # custom phrase +``` + +`--gui` also exercises the boot-time helper configuration: when +`~/.local/share/infernode-speech/bin` exists (or +`$INFERNODE_SPEECH_HOME` points elsewhere), the launcher passes it +through as `$speechhelperbin` and `boot.sh` applies the installer's ctl +block (kokorobin / whisperstreambin / wakebin / whispermodel / voice / +wakeword / wakethreshold) automatically — the same variable can be set +from a profile to get configured helpers in the normal LLM boot, too. +The headless-only flags (`-n`, `-c`, `-M`, `-d`) are rejected with +`--gui`; remote topologies in the GUI are Phase 2 territory. +Unit tests for the daemon's test mode: `tests/voicemode_test.b` +(`TestMode*` cases). + +### Automated Composed Voice E2E + +`tests/host/speech_e2e_test.sh` is the blocking, hardware-free production-path +test. It starts a loopback OpenAI-compatible endpoint, then runs the real Lucia, +LLM, bridge, voice-mode, speech-provider, and speech-shim services together. +The helper fixture emits deterministic wake plus partial/final records and +captures TTS PCM without requiring microphone permission or installed models. + +The scenario proves that a live/final transcript reaches Lucia exactly once, +the explicitly selected local OpenAI model receives one request, the assistant +reply returns through the conversation, and that reply reaches `speech9p` TTS. +It also verifies the Voice lifecycle resource and microphone release. Run it +directly after building its bytecode, or as part of the normal blocking suite: + +```sh +tools/speech-regress.sh +``` + +## Delivered Files + +| Path | Action | Phase | +| --- | --- | --- | +| `emu/MacOSX/audio.c` | New CoreAudio platform driver | 1.0 | +| `emu/MacOSX/emu` | Enable `audio audio` | 1.0 | +| `emu/MacOSX/mkfile` | Link CoreAudio/AudioToolbox as needed | 1.0 | +| `appl/veltro/speech9p.b` | Add Kokoro, streaming STT, wake, queue, cancel | 1.1-1.3 | +| `module/speech.m` | Add streaming partial type additively | 1.2 | +| `appl/cmd/voicemode.b` | New voice-mode state machine daemon | 1.4 | +| `appl/cmd/mkfile` | Build `voicemode` | 1.4 | +| `appl/cmd/lucibridge.b` | Add `/voice mode on|off` and pause-aware input | 1.5 | +| `appl/cmd/luciuisrv.b` | Add `/n/ui/input-mode` and key-event path | 1.5 | +| `appl/cmd/lucifer.b` | Pre-spawn idle `voicemode` | 1.5 | +| `tests/host/audio_macos_test.sh` | Host audio smoke test | 1.6 | +| `tests/speech_kokoro_test.b` | Kokoro TTS test | 1.6 | +| `tests/speech_listen_test.b` | Streaming STT test | 1.6 | +| `tests/speech_wake_test.b` | Wake-word test | 1.6 | +| `tests/voicemode_test.b` | Voice daemon test | 1.6 | +| `tests/speech_e2e_test.b` | Composed Lucia/LLM/voice service test | 1.6 | +| `tests/host/speech_e2e_test.sh` | Hermetic loopback E2E harness | 1.6 | +| `tests/host/speech_e2e_helper.sh` | Deterministic speech helper fixture | 1.6 | +| `docs/SPEECH-REMOTE-AUDIO.md` | Add Phase 2 pointer after Phase 1 stabilizes | 1.6 | + +## Out of Scope for Phase 1 + +- A particular or custom-trained wake model. +- Jetson-hosted inference. +- Cross-host acceptance and productization of the already-delivered 9P remote + audio launch scripts, routing controls, and loadable engine modules. +- Public distribution of the Parakeet EOU GGUF and a pinned conversion release. +- Server-owned queue depth, queued-turn cancel/replace, and rich queue UI. +- Native 24000/48000 Hz emulator playback. +- Multilingual STT/TTS. +- Voice biometrics or multi-speaker disambiguation. + +## Immediate Next Step + +Run the automated release-candidate suite on `dev`, then complete the three +hardware/physical-interface checks above. If a check fails, add a forward-only +fix commit; do not amend or rebase published candidate history. diff --git a/emu/MacOSX/audio.c b/emu/MacOSX/audio.c new file mode 100644 index 000000000..c39277b85 --- /dev/null +++ b/emu/MacOSX/audio.c @@ -0,0 +1,531 @@ +#include "dat.h" +#include "fns.h" +#include "error.h" +#include "audio.h" +#include +#include +#include + +#define Audio_Mic_Val 1 +#define Audio_Linein_Val 2 + +#define Audio_Speaker_Val 1 +#define Audio_Headphone_Val 2 +#define Audio_Lineout_Val 3 + +#define Audio_Pcm_Val 1 +#define Audio_Ulaw_Val 2 +#define Audio_Alaw_Val 3 + +#include "audio-tbls.c" + +#define Nqueuebuf 3 +#define Defbufsz 8192 +#define Defringsz 262144 + +#define min(a,b) ((a) < (b) ? (a) : (b)) + +typedef struct Ring Ring; +struct Ring { + uchar *data; + int size; + int r; + int w; + int fill; + pthread_mutex_t lk; + pthread_cond_t canread; + pthread_cond_t canwrite; +}; + +static Audio_t av; +static QLock inlock; +static QLock outlock; + +static int inopen; +static int outopen; +static int instarted; +static int outstarted; +static Ring inring; +static Ring outring; +static AudioQueueRef inq; +static AudioQueueRef outq; +static AudioQueueBufferRef inbuf[Nqueuebuf]; +static AudioQueueBufferRef outbuf[Nqueuebuf]; +static AudioStreamBasicDescription infmt; +static AudioStreamBasicDescription outfmt; + +static void closinput(void); +static void closoutput(void); +static void startinput(void); +static void startoutput(void); + +Audio_t* +getaudiodev(void) +{ + return &av; +} + +void +audio_file_init(void) +{ + audio_info_init(&av); +} + +static void +coreerror(char *what, OSStatus s) +{ + char buf[ERRMAX]; + + snprint(buf, sizeof(buf), "%s: %ld", what, (long)s); + error(buf); +} + +static void +ringinit(Ring *r, int n) +{ + if(r->data != nil && r->size == n) + return; + if(r->data != nil) + free(r->data); + r->data = malloc(n); + if(r->data == nil) + error(Enomem); + r->size = n; + r->r = 0; + r->w = 0; + r->fill = 0; + pthread_mutex_init(&r->lk, nil); + pthread_cond_init(&r->canread, nil); + pthread_cond_init(&r->canwrite, nil); +} + +static void +ringreset(Ring *r) +{ + pthread_mutex_lock(&r->lk); + r->r = 0; + r->w = 0; + r->fill = 0; + pthread_cond_broadcast(&r->canread); + pthread_cond_broadcast(&r->canwrite); + pthread_mutex_unlock(&r->lk); +} + +static void +ringfree(Ring *r) +{ + uchar *p; + + if(r->data == nil) + return; + pthread_mutex_lock(&r->lk); + p = r->data; + r->data = nil; + r->size = 0; + r->r = 0; + r->w = 0; + r->fill = 0; + pthread_cond_broadcast(&r->canread); + pthread_cond_broadcast(&r->canwrite); + pthread_mutex_unlock(&r->lk); + free(p); + pthread_cond_destroy(&r->canread); + pthread_cond_destroy(&r->canwrite); + pthread_mutex_destroy(&r->lk); +} + +static int +ringreadnb(Ring *r, uchar *p, int n) +{ + int m, got; + + got = 0; + pthread_mutex_lock(&r->lk); + while(got < n && r->fill > 0){ + m = min(n - got, r->fill); + m = min(m, r->size - r->r); + memcpy(p + got, r->data + r->r, m); + r->r = (r->r + m) % r->size; + r->fill -= m; + got += m; + } + if(got > 0) + pthread_cond_broadcast(&r->canwrite); + pthread_mutex_unlock(&r->lk); + return got; +} + +static int +ringwritenb(Ring *r, uchar *p, int n) +{ + int m, put; + + put = 0; + pthread_mutex_lock(&r->lk); + while(put < n && r->fill < r->size){ + m = min(n - put, r->size - r->fill); + m = min(m, r->size - r->w); + memcpy(r->data + r->w, p + put, m); + r->w = (r->w + m) % r->size; + r->fill += m; + put += m; + } + if(put > 0) + pthread_cond_broadcast(&r->canread); + pthread_mutex_unlock(&r->lk); + return put; +} + +static void +ringwrite(Ring *r, uchar *p, int n) +{ + int m; + + pthread_mutex_lock(&r->lk); + while(n > 0){ + while(r->fill == r->size) + pthread_cond_wait(&r->canwrite, &r->lk); + m = min(n, r->size - r->fill); + m = min(m, r->size - r->w); + memcpy(r->data + r->w, p, m); + r->w = (r->w + m) % r->size; + r->fill += m; + p += m; + n -= m; + pthread_cond_broadcast(&r->canread); + } + pthread_mutex_unlock(&r->lk); +} + +static void +ringread(Ring *r, uchar *p, int n) +{ + int m; + + pthread_mutex_lock(&r->lk); + while(n > 0){ + while(r->fill == 0) + pthread_cond_wait(&r->canread, &r->lk); + m = min(n, r->fill); + m = min(m, r->size - r->r); + memcpy(p, r->data + r->r, m); + r->r = (r->r + m) % r->size; + r->fill -= m; + p += m; + n -= m; + pthread_cond_broadcast(&r->canwrite); + } + pthread_mutex_unlock(&r->lk); +} + +static void +setformat(Audio_d *d, AudioStreamBasicDescription *fmt) +{ + if(d->enc != Audio_Pcm_Val) + error("unsupported macOS audio encoding"); + if(d->bits != 8 && d->bits != 16) + error("unsupported macOS audio sample size"); + if(d->chan != 1 && d->chan != 2) + error("unsupported macOS audio channel count"); + memset(fmt, 0, sizeof(*fmt)); + fmt->mSampleRate = d->rate; + fmt->mFormatID = kAudioFormatLinearPCM; + fmt->mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked; + fmt->mBitsPerChannel = d->bits; + fmt->mChannelsPerFrame = d->chan; + fmt->mFramesPerPacket = 1; + fmt->mBytesPerFrame = (d->bits / Bits_Per_Byte) * d->chan; + fmt->mBytesPerPacket = fmt->mBytesPerFrame; +} + +static int +ringsz(Audio_d *d) +{ + int n; + + n = d->buf * Audio_Max_Buf / Audio_Max_Val; + if(n < Defbufsz) + n = Defbufsz; + if(n < Defringsz) + n = Defringsz; + return n; +} + +static void +outcallback(void *arg, AudioQueueRef q, AudioQueueBufferRef b) +{ + int n; + + USED(arg); + n = ringreadnb(&outring, b->mAudioData, b->mAudioDataBytesCapacity); + if(n < (int)b->mAudioDataBytesCapacity) + memset((uchar*)b->mAudioData + n, 0, b->mAudioDataBytesCapacity - n); + b->mAudioDataByteSize = b->mAudioDataBytesCapacity; + AudioQueueEnqueueBuffer(q, b, 0, nil); +} + +static void +incallback(void *arg, AudioQueueRef q, AudioQueueBufferRef b, + const AudioTimeStamp *start, UInt32 packets, const AudioStreamPacketDescription *desc) +{ + USED(arg); + USED(start); + USED(packets); + USED(desc); + if(b->mAudioDataByteSize > 0) + ringwritenb(&inring, b->mAudioData, b->mAudioDataByteSize); + AudioQueueEnqueueBuffer(q, b, 0, nil); +} + +static void +startoutput(void) +{ + OSStatus s; + int i; + + if(outstarted) + return; + setformat(&av.out, &outfmt); + ringinit(&outring, ringsz(&av.out)); + ringreset(&outring); + s = AudioQueueNewOutput(&outfmt, outcallback, nil, nil, nil, 0, &outq); + if(s != noErr) + coreerror("cannot open CoreAudio output", s); + for(i = 0; i < Nqueuebuf; i++){ + s = AudioQueueAllocateBuffer(outq, Defbufsz, &outbuf[i]); + if(s != noErr) + coreerror("cannot allocate CoreAudio output buffer", s); + memset(outbuf[i]->mAudioData, 0, Defbufsz); + outbuf[i]->mAudioDataByteSize = Defbufsz; + s = AudioQueueEnqueueBuffer(outq, outbuf[i], 0, nil); + if(s != noErr) + coreerror("cannot enqueue CoreAudio output buffer", s); + } + s = AudioQueueStart(outq, nil); + if(s != noErr) + coreerror("cannot start CoreAudio output", s); + outstarted = 1; +} + +static void +startinput(void) +{ + OSStatus s; + int i; + + if(instarted) + return; + setformat(&av.in, &infmt); + ringinit(&inring, ringsz(&av.in)); + ringreset(&inring); + s = AudioQueueNewInput(&infmt, incallback, nil, nil, nil, 0, &inq); + if(s != noErr) + coreerror("cannot open CoreAudio input", s); + for(i = 0; i < Nqueuebuf; i++){ + s = AudioQueueAllocateBuffer(inq, Defbufsz, &inbuf[i]); + if(s != noErr) + coreerror("cannot allocate CoreAudio input buffer", s); + s = AudioQueueEnqueueBuffer(inq, inbuf[i], 0, nil); + if(s != noErr) + coreerror("cannot enqueue CoreAudio input buffer", s); + } + s = AudioQueueStart(inq, nil); + if(s != noErr) + coreerror("cannot start CoreAudio input", s); + instarted = 1; +} + +static void +closoutput(void) +{ + if(outq != nil){ + AudioQueueStop(outq, true); + AudioQueueDispose(outq, true); + outq = nil; + } + outstarted = 0; + ringfree(&outring); +} + +static void +closinput(void) +{ + if(inq != nil){ + AudioQueueStop(inq, true); + AudioQueueDispose(inq, true); + inq = nil; + } + instarted = 0; + ringfree(&inring); +} + +void +audio_file_open(Chan *c, int omode) +{ + USED(c); + switch(omode){ + case OREAD: + qlock(&inlock); + if(waserror()){ + qunlock(&inlock); + nexterror(); + } + if(inopen) + error(Einuse); + inopen = 1; + poperror(); + qunlock(&inlock); + break; + case OWRITE: + qlock(&outlock); + if(waserror()){ + qunlock(&outlock); + nexterror(); + } + if(outopen) + error(Einuse); + outopen = 1; + poperror(); + qunlock(&outlock); + break; + case ORDWR: + qlock(&inlock); + qlock(&outlock); + if(waserror()){ + qunlock(&outlock); + qunlock(&inlock); + nexterror(); + } + if(inopen || outopen) + error(Einuse); + inopen = 1; + outopen = 1; + poperror(); + qunlock(&outlock); + qunlock(&inlock); + break; + default: + error(Ebadarg); + } +} + +void +audio_file_close(Chan *c) +{ + switch(c->mode){ + case OREAD: + qlock(&inlock); + closinput(); + inopen = 0; + qunlock(&inlock); + break; + case OWRITE: + qlock(&outlock); + closoutput(); + outopen = 0; + qunlock(&outlock); + break; + case ORDWR: + qlock(&inlock); + qlock(&outlock); + closinput(); + closoutput(); + inopen = 0; + outopen = 0; + qunlock(&outlock); + qunlock(&inlock); + break; + } +} + +long +audio_file_read(Chan *c, void *va, long count, vlong offset) +{ + long ba; + + USED(c); + USED(offset); + qlock(&inlock); + if(waserror()){ + qunlock(&inlock); + nexterror(); + } + if(!inopen) + error(Eperm); + ba = av.in.bits * av.in.chan / Bits_Per_Byte; + if(ba <= 0 || count % ba) + error(Ebadarg); + startinput(); + ringread(&inring, va, count); + poperror(); + qunlock(&inlock); + return count; +} + +long +audio_file_write(Chan *c, void *va, long count, vlong offset) +{ + long ba; + + USED(c); + USED(offset); + qlock(&outlock); + if(waserror()){ + qunlock(&outlock); + nexterror(); + } + if(!outopen) + error(Eperm); + ba = av.out.bits * av.out.chan / Bits_Per_Byte; + if(ba <= 0 || count % ba) + error(Ebadarg); + startoutput(); + ringwrite(&outring, va, count); + poperror(); + qunlock(&outlock); + return count; +} + +long +audio_ctl_write(Chan *c, void *va, long count, vlong offset) +{ + Audio_t tmpav; + + USED(c); + USED(offset); + tmpav = av; + tmpav.in.flags = 0; + tmpav.out.flags = 0; + if(!audioparse(va, count, &tmpav)) + error(Ebadarg); + + if(!canqlock(&inlock)) + error("device busy"); + if(waserror()){ + qunlock(&inlock); + nexterror(); + } + if(!canqlock(&outlock)) + error("device busy"); + if(waserror()){ + qunlock(&outlock); + nexterror(); + } + if(instarted || outstarted) + error("device busy"); + + if(tmpav.in.flags & AUDIO_MOD_FLAG){ + tmpav.in.flags = 0; + av.in = tmpav.in; + } + if(tmpav.out.flags & AUDIO_MOD_FLAG){ + tmpav.out.flags = 0; + av.out = tmpav.out; + } + + poperror(); + qunlock(&outlock); + poperror(); + qunlock(&inlock); + return count; +} diff --git a/emu/MacOSX/mkfile b/emu/MacOSX/mkfile index 471f534b8..b82742b66 100644 --- a/emu/MacOSX/mkfile +++ b/emu/MacOSX/mkfile @@ -53,6 +53,8 @@ SYSLIBS= \ -lpthread\ -framework CoreFoundation\ -framework IOKit\ + -framework CoreAudio\ + -framework AudioToolbox\ default:V: $O.$CONF diff --git a/emu/MacOSX/mkfile-g b/emu/MacOSX/mkfile-g index b96ac2632..0949543c3 100644 --- a/emu/MacOSX/mkfile-g +++ b/emu/MacOSX/mkfile-g @@ -42,6 +42,8 @@ SYSLIBS= \ -lpthread\ -framework CoreFoundation\ -framework IOKit\ + -framework CoreAudio\ + -framework AudioToolbox\ # -framework CoreFoundation\ # -framework IOKit\ # -framework ApplicationServices\ diff --git a/emu/port/audio-tbls.c b/emu/port/audio-tbls.c index ed84bec62..fc48017ba 100644 --- a/emu/port/audio-tbls.c +++ b/emu/port/audio-tbls.c @@ -33,6 +33,7 @@ svp_t audio_enc_tbl[] = { svp_t audio_rate_tbl[] = { { "8000", 8000 }, /* 8000 samples per second */ { "11025", 11025 }, /* 11025 samples per second */ + { "16000", 16000 }, /* 16000 samples per second */ { "22050", 22050 }, /* 22050 samples per second */ { "44100", 44100 }, /* 44100 samples per second */ {nil}, diff --git a/emu/port/draw-sdl3.c b/emu/port/draw-sdl3.c index 856f3976e..01b365dec 100644 --- a/emu/port/draw-sdl3.c +++ b/emu/port/draw-sdl3.c @@ -1323,6 +1323,9 @@ ios_normalize_rune(Rune r, char *extra, int *extralen) } #endif +static int alt_v_text_pending; +static int alt_v_chord; + /* * Handle SDL_EVENT_TEXT_INPUT. * Decodes UTF-8 text to Unicode codepoints and sends to keyboard queue. @@ -1335,6 +1338,13 @@ handle_text_input(const char *text) Rune r; int n; + /* Alt+V is emitted from KEY_DOWN as ESC,v. SDL may also synthesize + * a printable Option character for the same chord; discard it once. */ + if(alt_v_text_pending) { + alt_v_text_pending = 0; + return; + } + if ((uchar)text[0] < 0x20 && text[0] != '\t') return; @@ -1373,6 +1383,19 @@ handle_key_down(SDL_Event *event) SDL_Keymod mods = event->key.mod; SDL_Keycode kc = event->key.key; + /* Desktop voice-mode chord. Lucifer already treats ESC,v as entry, so + * translate only Alt+V here and leave ordinary Option composition alone. */ + if((mods & SDL_KMOD_ALT) && event->key.scancode == SDL_SCANCODE_V) { + alt_v_text_pending = 1; + alt_v_chord = 1; + gkbdputc(gkbdq, 27); + gkbdputc(gkbdq, 'v'); + return; + } + /* If the previous Alt+V produced no TEXT_INPUT event, do not let its + * suppression leak into the next normal key. */ + alt_v_text_pending = 0; + /* Ctrl+letter or Cmd+letter -> control character (^A=1, ^H=8, etc.) * Cmd (GUI mod) is mapped so macOS Cmd+C/X/V work as copy/cut/paste */ if ((mods & (SDL_KMOD_CTRL | SDL_KMOD_GUI)) && kc >= 'a' && kc <= 'z') @@ -1663,7 +1686,10 @@ sdl3_mainloop(void) case SDL_EVENT_KEY_UP: if (event.key.scancode == SDL_SCANCODE_LALT || event.key.scancode == SDL_SCANCODE_RALT) { - gkbdputc(gkbdq, Latin); + if(alt_v_chord) + alt_v_chord = 0; + else + gkbdputc(gkbdq, Latin); } break; diff --git a/lib/lucifer/boot-speechtest.sh b/lib/lucifer/boot-speechtest.sh new file mode 100644 index 000000000..b8df309b1 --- /dev/null +++ b/lib/lucifer/boot-speechtest.sh @@ -0,0 +1,35 @@ +# GUI speech-test boot — the full lucifer desktop with voicemode in its +# LLM-free test mode: wake ("hey jarvis"), live partials in a bordered +# unsent conversation turn, and every final transcript answered by speaking +# a canned phrase. +# No login, no API key, no LLM traffic — for dogfooding the speech stack +# without per-turn cost. Same pattern as boot-mobile.sh: set variables, +# then hand off to the canonical boot.sh via `run`. +# +# Invoked by tools/speech-test.sh --gui as: +# +# sh -l /lib/lucifer/boot-speechtest.sh <-e|-> +# +# $1 HOST path to the speech-helpers bin dir from +# tools/install-speech-helpers.sh ('-' = leave helpers unconfigured; +# point /n/speech/ctl at a provider manually, e.g. a remote mount) +# $2 '-e' to answer with the transcript itself instead of the phrase +# $3 the canned TTS phrase spoken for every final transcript +# +# voicemode runs with -d in this mode; its trace lands in +# /tmp/voicemode.log inside the emu namespace. + +if {! ~ $1 -} { + speechhelperbin = $1 +} +voicetestargs = ('-d' '-p' $3) +if {~ $2 -e} { + voicetestargs = ('-d' '-e' '-p' $3) +} + +# Test mode needs no secstore keys (nothing calls the LLM), so skip the +# password prompt — same dev-mode semantics as boot-mobile --no-logon. +skiplogon = 1 +echo 'boot-speechtest: LLM-free voice test mode (skiplogon=1)' + +run /lib/lucifer/boot.sh diff --git a/lib/lucifer/boot.sh b/lib/lucifer/boot.sh index a95b9fba8..bfbed17f7 100644 --- a/lib/lucifer/boot.sh +++ b/lib/lucifer/boot.sh @@ -118,9 +118,93 @@ sleep 1 # /tmp truly doesn't exist after this, the mkdir will print to stderr # instead of being silenced. mkdir -p /tmp + +# Speech stack. speechshim9p adapts external host helper CLIs +# (whisper-stream, kokoro, openwakeword) to the speech provider contract +# at /n/speechshim; speech9p serves the stable /n/speech surface and is +# pointed at the shim as its provider. Any other provider serving the +# same contract (a parakeet export, a remote 9P mount) can replace it +# with one ctl write. Helpers are external installs and every path +# soft-fails with an error record, so starting both unconditionally is +# safe. speech9p must come before lucibridge, which registers the speech +# resource tile only if /n/speech is mounted at its startup. +> /tmp/speechshim9p.log +/dis/veltro/speechshim9p >[2] /tmp/speechshim9p.log +> /tmp/speech9p.log +/dis/veltro/speech9p >[2] /tmp/speech9p.log +echo provider /n/speechshim > /n/speech/ctl +echo duplex half > /n/speech/ctl + +# Host speech-helper configuration. The installer writes its chosen stack +# to $prefix/speech.ctl.sh (Kokoro TTS + Parakeet realtime STT when it +# could be built, whisper fallback otherwise) and boot replays that file +# verbatim — the installer is the single source of truth, so new helper +# stacks need no boot.sh change. When $speechhelperbin is preset by +# boot-speechtest.sh, prefer the adjacent installer configuration; fake +# helper bins without that file retain the legacy hardcoded path. +# +# Legacy path: $speechhelperbin names the bin/ dir created by +# tools/install-speech-helpers.sh — a HOST path, because the shim execs +# the helpers through devcmd. Without any of this the shim keeps its +# built-in defaults — bare command names that are not on the host PATH — +# and the wake helper can never exec, which silently makes voice mode +# unable to hear anything at all. +# +# The prefix is a host path, so it is probed through /n/local (the host root). +# The wake phrase is "hey jarvis" — the only pretrained openWakeWord model +# shipped today (see tools/install-speech-helpers.sh). +speechctlfile=() +if {! ~ $#speechhelperbin 0} { + if {ftest -f /n/local^$speechhelperbin^/../speech.ctl.sh} { + speechctlfile=/n/local^$speechhelperbin^/../speech.ctl.sh + } +} +if {~ $#speechhelperbin 0} { + speechprefix=`{echo 'echo ${INFERNODE_SPEECH_HOME:-$HOME/.local/share/infernode-speech}' | os sh >[2] /dev/null} + if {ftest -f /n/local^$speechprefix^/speech.ctl.sh} { + speechctlfile=/n/local^$speechprefix^/speech.ctl.sh + } + if {ftest -d /n/local^$speechprefix^/bin} { + speechhelperbin=$speechprefix^/bin + } +} +if {! ~ $#speechctlfile 0} { + sh $speechctlfile + echo 'boot: speech configured from' $speechctlfile +}{ + if {! ~ $#speechhelperbin 0} { + # engine kokoro routes speech9p's say through the provider's + # Kokoro instead of the robotic host `say` command (engine cmd). + echo engine kokoro > /n/speech/ctl + echo kokorobin $speechhelperbin/kokoro-cli > /n/speech/ctl + echo whisperstreambin $speechhelperbin/whisper-stream-cli > /n/speech/ctl + echo wakebin $speechhelperbin/openwakeword-cli > /n/speech/ctl + echo whispermodel $speechhelperbin/../models/ggml-base.en.bin > /n/speech/ctl + echo voice af_bella > /n/speech/ctl + echo 'wakeword hey jarvis' > /n/speech/ctl + echo wakethreshold 0.5 > /n/speech/ctl + echo 'boot: speech helpers configured from' $speechhelperbin + }{ + echo 'boot: no speech helpers found — voice mode will not hear or speak.' + echo 'boot: run tools/install-speech-helpers.sh, then restart.' + } +} + > /tmp/lucibridge.log lucibridge -a 0 -v -s >[2] /tmp/lucibridge.log & sleep 1 + +# Voice-mode daemon — resident and idle until /mnt/ui/input-mode becomes +# "v" (via /voice mode on, or a spoken control intent). Pre-spawned here +# so entering voice mode has no first-use startup latency. $voicetestargs +# (set by boot-speechtest.sh) puts the daemon in its LLM-free test mode: +# finals are answered with a canned TTS phrase instead of an LLM turn. +> /tmp/voicemode.log +if {! ~ $#voicetestargs 0} { + voicemode $voicetestargs >[2] /tmp/voicemode.log & +}{ + voicemode >[2] /tmp/voicemode.log & +} echo 'create id=tasks type=taskboard label=Tasks' > /mnt/ui/activity/0/presentation/ctl # Plumbing — route file-opens to the presentation view. The stock Inferno diff --git a/lib/sh/profile b/lib/sh/profile index e7765005e..e6796d80f 100644 --- a/lib/sh/profile +++ b/lib/sh/profile @@ -144,6 +144,13 @@ if {! ftest -f /mnt/factotum/proto}{ }{ auth/factotum -S tcp!localhost!5356 -P $secpass } + # Marker for wm/logon: the login screen may be skipped + # ONLY when secstore was genuinely unlocked here from + # $SECSTORE_PASSWORD. Factotum merely holding keys is not + # authentication — the API-key fallbacks below provision + # factotum before logon runs, and an API key must never + # bypass the password screen. + secstoreautounlock=1 }{ auth/factotum } diff --git a/lib/voice/speech-capture b/lib/voice/speech-capture new file mode 100644 index 000000000..69764d78f --- /dev/null +++ b/lib/voice/speech-capture @@ -0,0 +1,27 @@ +# speech-capture — topology 3 processing-host role: import a remote /dev +# tree and use only its audio device for speech capture. +# +# Usage: +# sh /lib/voice/speech-capture tcp!phone!17010 [capture-mount] + +load std + +capture=$1 +capturemnt=$2 +if {~ $#capture 0} { + echo 'usage: voice/speech-capture capture-device-addr [capture-mount]' >[1=2] + exit usage +} +if {~ $#capturemnt 0} { + capturemnt=/n/capture +} + +mkdir -p $capturemnt +if {! mount -A $capture $capturemnt} { + echo 'voice/speech-capture: capture mount failed:' $capture >[1=2] + exit mount +} + +echo 'capturedev '$capturemnt'/audio' > /n/speech/ctl +echo 'micmode device' > /n/speech/ctl +echo 'voice/speech-capture: capture device is' $capturemnt'/audio' diff --git a/lib/voice/speech-engine b/lib/voice/speech-engine new file mode 100644 index 000000000..2e5a43646 --- /dev/null +++ b/lib/voice/speech-engine @@ -0,0 +1,54 @@ +# speech-engine — topology 2 engine role: import the terminal's /dev, +# run speechshim9p against its audio, and export the provider contract. +# +# Usage: +# sh /lib/voice/speech-engine tcp!terminal!17010 [provider-port] [terminal-mount] [provider-mount] + +load std + +terminal=$1 +providerport=$2 +termmnt=$3 +provider=$4 +if {~ $#terminal 0} { + echo 'usage: voice/speech-engine terminal-audio-addr [provider-port] [terminal-mount] [provider-mount]' >[1=2] + exit usage +} +if {~ $#providerport 0} { + providerport=17019 +} +if {~ $#termmnt 0} { + termmnt=/n/term +} +if {~ $#provider 0} { + provider=/n/speechengine +} + +mkdir -p $termmnt +if {! mount -A $terminal $termmnt} { + echo 'voice/speech-engine: terminal audio mount failed:' $terminal >[1=2] + exit mount +} + +mkdir -p $provider +speechshim9p -m $provider & +shim_pid=$apid +sleep 1 +if {! ftest -e $provider/ctl} { + echo 'voice/speech-engine: speechshim9p did not mount at' $provider >[1=2] + if {ftest -e /prog/$shim_pid} { + echo kill >/prog/$shim_pid/ctl + } + exit shim +} + +echo 'audiodev '$termmnt'/audio' > $provider/ctl +echo 'capturedev default' > $provider/ctl +echo 'micmode device' > $provider/ctl +echo 'duplex half' > $provider/ctl + +addr='tcp!*!'$providerport +echo 'voice/speech-engine:' $addr 'exporting' $provider +listen -A $addr { + export $provider +} diff --git a/lib/voice/speech-terminal b/lib/voice/speech-terminal new file mode 100644 index 000000000..3a980e5a3 --- /dev/null +++ b/lib/voice/speech-terminal @@ -0,0 +1,40 @@ +# speech-terminal — topology 2 terminal role: export this machine's audio, +# mount a remote speech provider, and select it for /n/speech. +# +# Usage: +# sh /lib/voice/speech-terminal tcp!engine!17019 [audio-port] [provider-mount] + +load std + +engine=$1 +audioport=$2 +provider=$3 +if {~ $#engine 0} { + echo 'usage: voice/speech-terminal engine-provider-addr [audio-port] [provider-mount]' >[1=2] + exit usage +} +if {~ $#audioport 0} { + audioport=17010 +} +if {~ $#provider 0} { + provider=/n/remotespeech +} + +# voice/listen owns the devaudio bind, buffer caps, pre-warm, and export loop. +sh /lib/voice/listen $audioport & +terminal_pid=$apid +sleep 1 + +mkdir -p $provider +if {! mount -A $engine $provider} { + echo 'voice/speech-terminal: provider mount failed:' $engine >[1=2] + if {ftest -e /prog/$terminal_pid} { + echo kill >/prog/$terminal_pid/ctl + } + exit mount +} + +echo 'provider '$provider > /n/speech/ctl +echo 'duplex half' > /n/speech/ctl +echo 'voice/speech-terminal: audio tcp!*!'$audioport 'provider' $provider +echo ' listener pid=' $terminal_pid diff --git a/module/speech.m b/module/speech.m index 1c8aee977..a9ca9d11d 100644 --- a/module/speech.m +++ b/module/speech.m @@ -30,13 +30,14 @@ # Engine configuration Config: adt { - engine: string; # Engine name: "cmd", "api" + engine: string; # Engine name: "cmd", "api", "local", "kokoro", "module" voice: string; # Voice identifier (engine-specific) lang: string; # Language code (e.g. "en", "es", "fr") apiurl: string; # API endpoint URL (for api engine) apikey: string; # API key (for api engine) cmdtts: string; # Host TTS command (for cmd engine) cmdstt: string; # Host STT command (for cmd engine) + provider: string; # Provider mount for namespace-backed engines infmt: ref AudioFmt; # Input audio format (for STT) outfmt: ref AudioFmt; # Output audio format (from TTS) }; @@ -54,6 +55,23 @@ err: string; # Error message or nil }; + # One record from a streaming STT session (Phase 1.2 addition). + # + # /n/speech/listen serves these as newline-delimited text records: + # + # partial hypothesis; may be revised by later records + # final end-of-speech transcript; closes the utterance + # error: helper failure + # + # Bare text from batch-style helpers is treated as final. voicemode + # consumes the stream; speech9p passes helper records through + # unparsed (see appl/veltro/speech9p.b dolisten and appl/cmd/ + # voicemode.b finaltext). + Partial: adt { + text: string; # Transcript text (partial or final) + isfinal: int; # 0 = hypothesis, 1 = end-of-speech transcript + }; + # TTS engine interface TTSEngine: adt { name: fn(e: self ref TTSEngine): string; @@ -77,3 +95,16 @@ fmtstr: fn(fmt: ref AudioFmt): string; parsefmt: fn(s: string): ref AudioFmt; }; + +# Loadable `.dis` engine contract. Unlike the legacy TTSEngine/STTEngine ADTs +# above, this is a real Limbo module interface and can be loaded dynamically by +# speech9p. One module may expose TTS, STT, or both according to caps(). +SpeechEngine: module { + init: fn(): string; + name: fn(): string; + caps: fn(): int; + configure: fn(cfg: ref Speech->Config): string; + voices: fn(): list of string; + synthesize: fn(text: string): ref Speech->TTSResult; + recognize: fn(audio: array of byte, fmt: ref Speech->AudioFmt): ref Speech->STTResult; +}; diff --git a/tests/host/audio_macos_test.sh b/tests/host/audio_macos_test.sh new file mode 100755 index 000000000..ff9deba22 --- /dev/null +++ b/tests/host/audio_macos_test.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=${ROOT:-$(cd "$(dirname "$0")/../.." && pwd)} +cd "$ROOT" +export ROOT +export PATH="$ROOT/MacOSX/arm64/bin:$PATH" + +EMU=${EMU:-./emu/MacOSX/o.emu} +AUDIODIR=${AUDIO_TEST_TMPDIR:-.omx/tmp} +mkdir -p "$AUDIODIR" + +make_pcm() { + python3 - "$1" "$2" "$3" <<'PY' +import math, struct, sys +path, rate, seconds = sys.argv[1], int(sys.argv[2]), float(sys.argv[3]) +frames = int(rate * seconds) +with open(path, 'wb') as f: + for i in range(frames): + sample = int(12000 * math.sin(2 * math.pi * 440 * i / rate)) + f.write(struct.pack('"$log" 2>&1; then + cat "$log" + if grep -Eq "cannot start CoreAudio (input|output): -66680" "$log"; then + rm -f "$AUDIODIR/audio-capture.pcm" + echo "SKIP: CoreAudio device unavailable in this host session" + fi + return 0 + fi + cat "$log" + if grep -Eq "cannot start CoreAudio (input|output): -66680" "$log"; then + rm -f "$AUDIODIR/audio-capture.pcm" + echo "SKIP: CoreAudio device unavailable in this host session" + return 0 + fi + return 1 +} + +mode=${1:-roundtrip} +case "$mode" in +ctl) + run_inferno "bind -a '#A' /dev; ls /dev/audio /dev/audioctl; cat /dev/audioctl" + ;; +playback) + make_pcm "$AUDIODIR/audio-playback.pcm" 16000 0.25 + run_audio_inferno "bind -a '#A' /dev; echo 'out rate 16000 chans 1 bits 16 enc pcm' > /dev/audioctl; cat /$AUDIODIR/audio-playback.pcm > /dev/audio" + ;; +capture) + rm -f "$AUDIODIR/audio-capture.pcm" + run_audio_inferno "bind -a '#A' /dev; echo 'in rate 16000 chans 1 bits 16 enc pcm' > /dev/audioctl; dd -if /dev/audio -of /$AUDIODIR/audio-capture.pcm -bs 32000 -count 1" + if [ -e "$AUDIODIR/audio-capture.pcm" ] && [ ! -s "$AUDIODIR/audio-capture.pcm" ]; then + # The device opened but delivered no frames. On macOS this is the + # microphone-permission (TCC) posture for non-interactive shells and + # CI: the input AudioQueue starts but never gets buffers. Same skip + # philosophy as the -66680 device-unavailable case above — a real + # capture regression can only be asserted where a mic is usable. + echo "SKIP: no audio captured (microphone unavailable or permission denied in this host session)" + fi + ;; +roundtrip) + "$0" playback + "$0" capture + ;; +*) + echo "usage: $0 [ctl|playback|capture|roundtrip]" >&2 + exit 2 + ;; +esac diff --git a/tests/host/speech_e2e_helper.sh b/tests/host/speech_e2e_helper.sh new file mode 100755 index 000000000..155117fad --- /dev/null +++ b/tests/host/speech_e2e_helper.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# Deterministic host helper used only by speech_e2e_test.sh. + +set -eu + +mode=${1:?mode required} +state=${2:?state directory required} +shift 2 + +wait_for_input() +{ + path=$1 + i=0 + while [ ! -s "$path" ]; do + i=$((i + 1)) + if [ "$i" -ge 1600 ]; then + echo "error: timed out waiting for $(basename "$path")" + exit 1 + fi + sleep 0.05 + done +} + +case "$mode" in +wake|listen) + input=$state/$mode.next + wait_for_input "$input" + cat "$input" + : > "$input" + ;; +say) + cat > "$state/say.last" + { + echo "--- say ---" + cat "$state/say.last" + } >> "$state/say.log" + : > "$state/say.started" + rm -f "$state/say.done" + i=0 + while [ "$i" -lt 12 ]; do + dd if=/dev/zero bs=2048 count=1 2>/dev/null + i=$((i + 1)) + sleep 0.02 + done + : > "$state/say.done" + ;; +*) + echo "speech_e2e_helper: unknown mode: $mode" >&2 + exit 2 + ;; +esac diff --git a/tests/host/speech_e2e_test.sh b/tests/host/speech_e2e_test.sh new file mode 100755 index 000000000..54ab38d6d --- /dev/null +++ b/tests/host/speech_e2e_test.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# Hermetic composed voice-mode test. No microphone, model, key, or network +# service outside loopback is used. + +set -u + +ROOT=${ROOT:-$(cd "$(dirname "$0")/../.." && pwd)} +export ROOT + +case "$(uname -s)" in +Darwin) syshost=MacOSX ;; +Linux) syshost=Linux ;; +*) echo "speech-e2e: unsupported host" >&2; exit 1 ;; +esac + +EMU=${EMU:-$ROOT/emu/$syshost/o.emu} +PYTHON=${PYTHON:-python3} + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +[ -x "$EMU" ] || fail "emulator not built: $EMU" +command -v "$PYTHON" >/dev/null 2>&1 || fail "python3 is required" +[ -f "$ROOT/tests/speech_e2e_test.dis" ] || fail "tests/speech_e2e_test.dis is not built" + +mkdir -p "$ROOT/tmp" +state=$(mktemp -d "$ROOT/tmp/speech-e2e.XXXXXX") || fail "cannot create state directory" +inferno_state=/tmp/$(basename "$state") +stub_pid= +emu_pid= + +cleanup() +{ + [ -n "$emu_pid" ] && kill -9 "$emu_pid" 2>/dev/null || true + [ -n "$stub_pid" ] && kill "$stub_pid" 2>/dev/null || true + [ -n "$stub_pid" ] && wait "$stub_pid" 2>/dev/null || true + rm -rf "$state" +} +trap cleanup EXIT HUP INT TERM + +: > "$state/wake.next" +: > "$state/listen.next" +: > "$state/requests.jsonl" + +"$PYTHON" - "$state" >"$state/stub.log" 2>&1 <<'PY' & +import json +import pathlib +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +state = pathlib.Path(sys.argv[1]) + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, fmt, *args): + return + + def send_json(self, value): + body = json.dumps(value, separators=(",", ":")).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(body) + + def send_sse(self, text): + events = [ + {"id": "chatcmpl-voice-e2e", "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {"content": text}, "finish_reason": None}]}, + {"id": "chatcmpl-voice-e2e", "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12}}, + ] + body = "".join("data: " + json.dumps(event, separators=(",", ":")) + "\n\n" + for event in events) + body += "data: [DONE]\n\n" + encoded = body.encode() + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Content-Length", str(len(encoded))) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(encoded) + + def do_GET(self): + if self.path.endswith("/models"): + self.send_json({"object": "list", "data": [{"id": "ci-voice-e2e", "object": "model"}]}) + return + self.send_error(404) + + def do_POST(self): + if not self.path.endswith("/chat/completions"): + self.send_error(404) + return + size = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(size) + request = json.loads(body) + with (state / "requests.jsonl").open("a", encoding="utf-8") as log: + log.write(json.dumps(request, separators=(",", ":")) + "\n") + if request.get("stream"): + self.send_sse("local LLM working") + return + self.send_json({ + "id": "chatcmpl-voice-e2e", + "object": "chat.completion", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "local LLM working"}, + "finish_reason": "stop", + }], + "usage": {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12}, + }) + + +server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) +(state / "port").write_text(str(server.server_address[1]), encoding="ascii") +server.serve_forever() +PY +stub_pid=$! + +i=0 +while [ ! -s "$state/port" ]; do + i=$((i + 1)) + [ "$i" -lt 100 ] || { + cat "$state/stub.log" >&2 + fail "OpenAI stub did not start" + } + sleep 0.05 +done + +port=$(cat "$state/port") +url=http://127.0.0.1:$port/v1 +log=$state/emulator.log + +env -u ANTHROPIC_API_KEY -u OPENAI_API_KEY \ + "$EMU" -c1 -r"$ROOT" /tests/speech_e2e_test.dis \ + -u "$url" -H "$state" -I "$inferno_state" \ + -X "$ROOT/tests/host/speech_e2e_helper.sh" >"$log" 2>&1 & +emu_pid=$! + +start=$SECONDS +while kill -0 "$emu_pid" 2>/dev/null; do + if grep -q '^PASS$' "$log" 2>/dev/null; then + break + fi + if grep -q -- '^--- FAIL:' "$log" 2>/dev/null; then + break + fi + [ $((SECONDS - start)) -lt 75 ] || break + sleep 0.2 +done + +kill -9 "$emu_pid" 2>/dev/null || true +wait "$emu_pid" 2>/dev/null || true +emu_pid= + +if ! grep -q '^PASS$' "$log" || grep -q -- '^--- FAIL:' "$log"; then + echo "---- speech E2E emulator output ----" >&2 + cat "$log" >&2 + echo "---- OpenAI stub output ----" >&2 + cat "$state/stub.log" >&2 + echo "---- captured OpenAI requests ----" >&2 + cat "$state/requests.jsonl" >&2 + echo "---- speech helper state ----" >&2 + for diagnostic in wake.next listen.next say.last say.log say.started say.done; do + if [ -f "$state/$diagnostic" ]; then + echo "[$diagnostic]" >&2 + cat "$state/$diagnostic" >&2 + fi + done + fail "composed voice-mode scenario failed" +fi + +grep -q '"model":"ci-voice-e2e"' "$state/requests.jsonl" || \ + fail "explicit OpenAI model did not reach the local endpoint" +[ "$(wc -l < "$state/requests.jsonl" | tr -d ' ')" = 1 ] || \ + fail "voice turn did not produce exactly one LLM request" +grep -q 'local LLM working' "$state/say.log" || \ + fail "assistant response was not sent through speech9p" + +echo "PASS: composed voice turn used the explicit local OpenAI endpoint" +echo "PASS: streaming transcript submitted once and response reached TTS" +echo "PASS" diff --git a/tests/host/speech_helpers_test.sh b/tests/host/speech_helpers_test.sh new file mode 100755 index 000000000..41dfeabad --- /dev/null +++ b/tests/host/speech_helpers_test.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +set -euo pipefail + +PREFIX=${INFERNODE_SPEECH_HOME:-"$HOME/.local/share/infernode-speech"} +BIN="$PREFIX/bin" +TMPDIR=${TMPDIR:-/tmp} +WORKDIR=$(mktemp -d "$TMPDIR/infernode-speech-helpers.XXXXXX") +trap 'rm -rf "$WORKDIR"' EXIT + +if [ ! -d "$PREFIX" ]; then + echo "SKIP: speech helper install dir not found: $PREFIX" + exit 77 +fi + +require_exec() { + local path=$1 + if [ ! -x "$path" ]; then + echo "FAIL: missing executable wrapper: $path" >&2 + exit 1 + fi +} + +require_exec "$BIN/kokoro-cli" +require_exec "$BIN/whisper-stream-cli" +require_exec "$BIN/openwakeword-cli" + +"$BIN/kokoro-cli" --list-voices >/dev/null +printf 'hello\n' | timeout 90 "$BIN/kokoro-cli" --voice af_bella --format pcm --rate 24000 >"$WORKDIR/hello.pcm" +if [ ! -s "$WORKDIR/hello.pcm" ]; then + echo "FAIL: kokoro-cli produced no PCM" >&2 + exit 1 +fi + +timeout 10 "$BIN/whisper-stream-cli" --help >/dev/null +timeout 10 "$BIN/openwakeword-cli" --help >/dev/null + +# The native Whisper and openWakeWord backends can enumerate Core Audio in a +# different order. Prove the configured Whisper capture id reaches the native +# binary instead of silently falling back to a hard-coded device. +mkdir -p "$WORKDIR/bin" +cat >"$WORKDIR/bin/whisper-stream" <<'SH' +#!/bin/sh +printf '%s\n' "$@" >"$WHISPER_ARG_LOG" +SH +chmod +x "$WORKDIR/bin/whisper-stream" +printf 'fake model\n' >"$WORKDIR/model.bin" +PATH="$WORKDIR/bin:$PATH" INFERNODE_SPEECH_CAPTURE=2 \ + INFERNODE_SPEECH_WINDOW_MS=5000 \ + WHISPER_ARG_LOG="$WORKDIR/whisper.args" \ + "$BIN/whisper-stream-cli" --model "$WORKDIR/model.bin" >/dev/null +if ! awk 'previous == "--capture" && $0 == "2" { found=1 } { previous=$0 } END { exit !found }' \ + "$WORKDIR/whisper.args"; then + echo "FAIL: whisper-stream-cli did not forward INFERNODE_SPEECH_CAPTURE" >&2 + exit 1 +fi +if ! awk 'previous == "--length" && $0 == "5000" { found=1 } { previous=$0 } END { exit !found }' \ + "$WORKDIR/whisper.args"; then + echo "FAIL: whisper-stream-cli did not forward INFERNODE_SPEECH_WINDOW_MS" >&2 + exit 1 +fi + +# Exercise stdin PCM without microphone permission or a costly real inference. +# The fake whisper-cli writes the same full-JSON shape as whisper.cpp; energy +# VAD, partial/final framing, and confidence extraction remain real. +cat >"$WORKDIR/bin/whisper-cli" <<'SH' +#!/bin/sh +out= +while [ "$#" -gt 0 ]; do + case "$1" in + --output-file) + out=$2 + shift 2 + ;; + *) + shift + ;; + esac +done +cat >"$out.json" <<'JSON' +{"transcription":[{"text":" namespace audio works","tokens":[{"text":" namespace","p":0.81},{"text":" audio","p":0.90},{"text":" works","p":0.99}]}]} +JSON +SH +chmod +x "$WORKDIR/bin/whisper-cli" +python3 -c 'import sys; sys.stdout.buffer.write((b"\xe8\x03" * 8000) + (b"\0\0" * 8000))' | + INFERNODE_WHISPER_CLI="$WORKDIR/bin/whisper-cli" \ + INFERNODE_STT_PARTIAL_MS=200 INFERNODE_STT_SILENCE_MS=200 \ + INFERNODE_STT_RMS_THRESHOLD=100 \ + timeout 20 "$BIN/whisper-stream-cli" --stdin --model "$WORKDIR/model.bin" \ + >"$WORKDIR/whisper-stdin.out" +if ! grep -Eq '^partial confidence=0\.[0-9]+ namespace audio works$' "$WORKDIR/whisper-stdin.out"; then + echo "FAIL: whisper-stream-cli --stdin emitted no confidence-bearing partial" >&2 + cat "$WORKDIR/whisper-stdin.out" >&2 + exit 1 +fi +if ! grep -Eq '^final confidence=0\.[0-9]+ namespace audio works$' "$WORKDIR/whisper-stdin.out"; then + echo "FAIL: whisper-stream-cli --stdin emitted no confidence-bearing final" >&2 + cat "$WORKDIR/whisper-stdin.out" >&2 + exit 1 +fi + +# The wrapper must relay records in real time. A stdio filter in its pipeline +# (tr, sed, grep) block-buffers when writing to a pipe, so tiny transcript +# lines sit in the filter until the helper exits — which a streaming helper +# never does: wake works, listen never delivers, all logs stay empty. Fake a +# whisper-stream that speaks once (with VAD-mode chrome, a timestamp block, +# and a \r) then stays alive well past the deadline; the final must arrive +# while the producer is still running. +cat >"$WORKDIR/bin/whisper-stream" <<'SH' +#!/bin/sh +printf '### Transcription 0 START\n' +printf '[00:00.000 --> 00:02.000] hello from fake whisper\r\n' +sleep 15 +SH +chmod +x "$WORKDIR/bin/whisper-stream" +: >"$WORKDIR/stream.out" +PATH="$WORKDIR/bin:$PATH" "$BIN/whisper-stream-cli" --model "$WORKDIR/model.bin" >"$WORKDIR/stream.out" & +stream_pid=$! +relayed=0 +deadline=$((SECONDS + 6)) +while [ "$SECONDS" -lt "$deadline" ]; do + if grep -q '^final hello from fake whisper$' "$WORKDIR/stream.out"; then + relayed=1 + break + fi + sleep 0.2 +done +kill "$stream_pid" 2>/dev/null || true +wait "$stream_pid" 2>/dev/null || true +if [ "$relayed" -ne 1 ]; then + echo "FAIL: whisper-stream-cli did not relay a record while the helper was still running (stdio buffering in the wrapper pipeline?)" >&2 + cat "$WORKDIR/stream.out" >&2 + exit 1 +fi + +if [ "${INFERNODE_SPEECH_MIC_SMOKE:-0}" = "1" ]; then + # Interactive TCC-approved session: prove the mic-capture helpers start + # and survive a few seconds without crashing (124 = timeout cut them off, + # which is the expected way to end a streaming helper). + micstatus=0 + timeout 5 "$BIN/openwakeword-cli" --word "hey jarvis" --threshold 0.99 >/dev/null || micstatus=$? + case "$micstatus" in + 0|124) ;; + *) + echo "FAIL: openwakeword-cli mic mode exited with $micstatus" >&2 + exit 1 + ;; + esac +else + echo "SKIP: microphone-dependent wake/STT starts not run; set INFERNODE_SPEECH_MIC_SMOKE=1 only in an interactive TCC-approved session" +fi + +printf '\0%.0s' {1..6400} | timeout 45 "$BIN/openwakeword-cli" --stdin --word "hey lucia" --threshold 0.99 >"$WORKDIR/wake.out" || status=$? +status=${status:-0} +case "$status" in +0|124) + ;; +*) + echo "FAIL: openwakeword-cli --stdin exited with $status" >&2 + exit 1 + ;; +esac + +echo "PASS: speech helper wrappers smoke-tested without microphone access" diff --git a/tests/host/speech_installer_download_test.sh b/tests/host/speech_installer_download_test.sh new file mode 100755 index 000000000..bdf9fe0cd --- /dev/null +++ b/tests/host/speech_installer_download_test.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +WORKDIR=$(mktemp -d "${TMPDIR:-/tmp}/infernode-speech-download.XXXXXX") +trap 'rm -rf "$WORKDIR"' EXIT + +mkdir -p "$WORKDIR/bin" +cat >"$WORKDIR/bin/curl" <<'SH' +#!/usr/bin/env bash +set -euo pipefail + +out= +url= +while [ "$#" -gt 0 ]; do + case "$1" in + --output) + out=$2 + shift 2 + ;; + http://*|https://*) + url=$1 + shift + ;; + *) + shift + ;; + esac +done + +printf '%s\n' "$url" >>"$FAKE_CURL_LOG" +case "$url" in +*primary-fails*) exit 22 ;; +*short*) printf 'bad' >"$out" ;; +*) dd if=/dev/zero of="$out" bs=128 count=1 2>/dev/null ;; +esac +SH +chmod +x "$WORKDIR/bin/curl" + +export PATH="$WORKDIR/bin:$PATH" +export FAKE_CURL_LOG="$WORKDIR/curl.log" +export INFERNODE_SPEECH_HOME="$WORKDIR/install" +export WHISPER_MODEL_MIN_BYTES=100 + +# Sourcing exposes the installer helpers without running the installation. +source "$ROOT/tools/install-speech-helpers.sh" + +model="$WORKDIR/models/ggml-base.en.bin" +: >"$FAKE_CURL_LOG" +download_model "$model" 100 \ + "https://example.invalid/primary-fails" \ + "https://example.invalid/fallback" +[ "$(wc -c <"$model")" -ge 100 ] +grep -q 'primary-fails' "$FAKE_CURL_LOG" +grep -q '/fallback' "$FAKE_CURL_LOG" + +# A complete existing model is retained without touching the network. +before=$(cksum "$model") +: >"$FAKE_CURL_LOG" +download_model "$model" 100 "https://example.invalid/primary-fails" +[ "$before" = "$(cksum "$model")" ] +[ ! -s "$FAKE_CURL_LOG" ] + +# A short response is rejected and the next candidate is tried atomically. +rm -f "$model" +: >"$FAKE_CURL_LOG" +download_model "$model" 100 \ + "https://example.invalid/short" \ + "https://example.invalid/fallback" +[ "$(wc -c <"$model")" -ge 100 ] +[ ! -e "$model.tmp" ] + +echo "PASS: speech installer retries, validates, and atomically installs models" diff --git a/tests/luciuisrv_test.b b/tests/luciuisrv_test.b index 335c6e091..ee5361561 100644 --- a/tests/luciuisrv_test.b +++ b/tests/luciuisrv_test.b @@ -355,6 +355,9 @@ testRootDirread(t: ref T) t.assertsne(conv, "error:timeout", "conversation readdir must terminate (INFR-127)"); t.assert(hassubstr(conv, " ctl"), "conversation listing includes ctl"); t.assert(hassubstr(conv, " input"), "conversation listing includes input"); + t.assert(hassubstr(conv, " voiceinput"), "conversation listing includes voiceinput"); + t.assert(hassubstr(conv, " control"), "conversation listing includes control: " + conv); + t.assert(hassubstr(conv, " draft"), "conversation listing includes draft"); } # ============================================================================ @@ -1047,6 +1050,119 @@ testConvInput(t: ref T) fd = nil; } +testVoiceInputMode(t: ref T) +{ + if(actid < 0) { + t.skip("no activity"); + return; + } + + modefile := TESTMNT + "/input-mode"; + t.assertseq(strip(readfile(modefile)), "k", "input-mode defaults to keyboard"); + t.asserteq(writefile(modefile, "v"), 1, "can switch to voice mode"); + t.assertseq(strip(readfile(modefile)), "v", "input-mode reads voice mode"); + t.asserteq(writefile(modefile, "k"), 1, "can switch back to keyboard mode"); + t.assertseq(strip(readfile(modefile)), "k", "input-mode reads keyboard mode"); +} + +testVoiceInput(t: ref T) +{ + if(actid < 0) { + t.skip("no activity"); + return; + } + + voicefile := actbase() + "/conversation/voiceinput"; + (ok, nil) := sys->stat(voicefile); + t.assert(ok >= 0, "conversation/voiceinput should exist"); + t.asserteq(writefile(voicefile, "voice injected turn"), len "voice injected turn", + "voiceinput write should queue voice-originated text"); + t.assertseq(strip(readfile(voicefile)), "voice injected turn", + "voiceinput read should return queued voice-originated text"); +} + +testVoiceInputFIFO(t: ref T) +{ + if(actid < 0) { + t.skip("no activity"); + return; + } + + voicefile := actbase() + "/conversation/voiceinput"; + t.asserteq(writefile(voicefile, "first voice turn"), len "first voice turn", + "first voice write"); + t.asserteq(writefile(voicefile, "second voice turn"), len "second voice turn", + "second voice write"); + t.asserteq(writefile(voicefile, "third voice turn"), len "third voice turn", + "third voice write"); + t.assertseq(strip(readfile(voicefile)), "first voice turn", + "voiceinput preserves FIFO order for first turn"); + t.assertseq(strip(readfile(voicefile)), "second voice turn", + "voiceinput preserves FIFO order for second turn"); + t.assertseq(strip(readfile(voicefile)), "third voice turn", + "voiceinput preserves FIFO order for third turn"); +} + +testConversationControl(t: ref T) +{ + if(actid < 0) { + t.skip("no activity"); + return; + } + control := actbase() + "/conversation/control"; + (ok, nil) := sys->stat(control); + t.assert(ok >= 0, "conversation/control should exist"); + t.asserteq(writefile(control, "pause"), len "pause", "pause control write"); + t.asserteq(writefile(control, "resume"), len "resume", "resume control write"); + t.assertseq(strip(readfile(control)), "pause", "control preserves FIFO first item"); + t.assertseq(strip(readfile(control)), "resume", "control preserves FIFO second item"); +} + +testConversationDraft(t: ref T) +{ + if(actid < 0) { + t.skip("no activity"); + return; + } + + draftfile := actbase() + "/conversation/draft"; + (ok, nil) := sys->stat(draftfile); + t.assert(ok >= 0, "conversation/draft should exist"); + t.asserteq(writefile(draftfile, "half a thou"), len "half a thou", + "first partial replaces the draft"); + t.assertseq(readfile(draftfile), "half a thou", + "draft is readable without becoming a message"); + t.asserteq(writefile(draftfile, "half a thousand"), len "half a thousand", + "revised partial replaces rather than appends"); + t.assertseq(readfile(draftfile), "half a thousand", + "latest hypothesis is the whole draft"); + + # Draft writes are presentation events, distinct from input submission. + writefile(actbase() + "/event", "flush"); + writefile(draftfile, "visible hypothesis"); + ev := readevent(actbase() + "/event"); + t.assertseq(strip(ev), "conversation draft", + "draft replacement notifies the conversation renderer"); + + writefile(draftfile, ""); + t.assertseq(readfile(draftfile), "", "empty write clears the draft"); + + statusfile := actbase() + "/conversation/draft-status"; + (ok, nil) = sys->stat(statusfile); + t.assert(ok >= 0, "conversation/draft-status should exist"); + writefile(actbase() + "/event", "flush"); + t.asserteq(writefile(statusfile, "Sending in 2s - say cancel to stop"), + len "Sending in 2s - say cancel to stop", + "draft status is replaceable presentation state"); + t.assertseq(readfile(statusfile), "Sending in 2s - say cancel to stop", + "draft status is readable without becoming a message"); + ev = readevent(actbase() + "/event"); + t.assertseq(strip(ev), "conversation draft", + "draft status replacement notifies the conversation renderer"); + writefile(statusfile, ""); + t.assertseq(readfile(statusfile), "", "empty write clears draft status"); +} + # ============================================================================ # testConvClear (INFR-131) # @@ -1926,6 +2042,11 @@ init(nil: ref Draw->Context, args: list of string) run("ConvInput", testConvInput); run("ConvCtlBadWrite", testConvCtlBadWrite); run("ConvClear", testConvClear); + run("VoiceInputMode", testVoiceInputMode); + run("VoiceInput", testVoiceInput); + run("VoiceInputFIFO", testVoiceInputFIFO); + run("ConversationControl", testConversationControl); + run("ConversationDraft", testConversationDraft); # Presentation tests run("PresCreate", testPresentationCreate); diff --git a/tests/mkfile b/tests/mkfile index 5686dfdce..7b4dee0bb 100644 --- a/tests/mkfile +++ b/tests/mkfile @@ -78,6 +78,14 @@ TARG=\ sdl3_test.dis\ sort_test.dis\ spawn_test.dis\ + speech9p_voice_test.dis\ + speech_kokoro_test.dis\ + speech_listen_test.dis\ + speech_wake_test.dis\ + speech_e2e_test.dis\ + speechshim_test.dis\ + speechtest_test.dis\ + voicemode_test.dis\ taskparse_test.dis\ ssl_transport_test.dis\ stderr_test.dis\ diff --git a/tests/speech9p_voice_test.b b/tests/speech9p_voice_test.b new file mode 100644 index 000000000..98c574501 --- /dev/null +++ b/tests/speech9p_voice_test.b @@ -0,0 +1,340 @@ +implement Speech9pVoiceTest; + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "testing.m"; + testing: Testing; + T: import testing; + +Speech9pSrv: module { + init: fn(nil: ref Draw->Context, args: list of string); +}; + +Speech9pVoiceTest: module { + init: fn(nil: ref Draw->Context, args: list of string); + _marker: fn(); # prevents joiniface() type conflation with Speech9pSrv +}; + +SRCFILE: con "/tests/speech9p_voice_test.b"; +SRVPATH: con "/dis/veltro/speech9p.dis"; +MNT: con "/tmp/speech9p_voice_test"; +PARAKEETMNT: con "/tmp/parakeet_voice_mount"; + +passed := 0; +failed := 0; +skipped := 0; + +_marker() {} + +run(name: string, testfn: ref fn(t: ref T)) +{ + t := testing->newTsrc(name, SRCFILE); + { + testfn(t); + } exception { + "fail:fatal" => + ; + "fail:skip" => + ; + "*" => + t.failed = 1; + } + if(testing->done(t)) + passed++; + else if(t.skipped) + skipped++; + else + failed++; +} + +strip(s: string): string +{ + if(s == nil) + return nil; + i := 0; + while(i < len s && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n')) + i++; + j := len s; + while(j > i && (s[j-1] == ' ' || s[j-1] == '\t' || s[j-1] == '\r' || s[j-1] == '\n')) + j--; + if(i >= j) + return ""; + return s[i:j]; +} + +writefile(path, data: string): int +{ + fd := sys->open(path, Sys->OWRITE); + if(fd == nil) + return -1; + b := array of byte data; + return sys->write(fd, b, len b); +} + +createfile(path, data: string): int +{ + fd := sys->create(path, Sys->OWRITE, 8r644); + if(fd == nil) + return -1; + b := array of byte data; + return sys->write(fd, b, len b); +} + +readfile(path: string): string +{ + fd := sys->open(path, Sys->OREAD); + if(fd == nil) + return nil; + buf := array[8192] of byte; + n := sys->read(fd, buf, len buf); + if(n < 0) + return nil; + return string buf[0:n]; +} + +writesayread(path, data: string): string +{ + fd := sys->open(path, Sys->ORDWR); + if(fd == nil) + return nil; + b := array of byte data; + if(sys->write(fd, b, len b) < 0) + return nil; + sys->seek(fd, big 0, Sys->SEEKSTART); + buf := array[8192] of byte; + n := sys->read(fd, buf, len buf); + if(n < 0) + return nil; + return string buf[0:n]; +} + +pathexists(path: string): int +{ + (ok, nil) := sys->stat(path); + return ok >= 0; +} + +hassubstr(s, sub: string): int +{ + if(s == nil || sub == nil || len sub > len s) + return 0; + for(i := 0; i <= len s - len sub; i++) + if(s[i:i+len sub] == sub) + return 1; + return 0; +} + +startserver() +{ + sys->create("/tmp", Sys->OREAD, Sys->DMDIR | 8r777); + sys->create(MNT, Sys->OREAD, Sys->DMDIR | 8r755); + sys->create(PARAKEETMNT, Sys->OREAD, Sys->DMDIR | 8r755); + srv := load Speech9pSrv SRVPATH; + if(srv == nil) { + sys->fprint(sys->fildes(2), "cannot load speech9p: %r\n"); + raise "fail:load"; + } + spawn srv->init(nil, "speech9p" :: "-m" :: MNT :: "-e" :: "kokoro" :: "-v" :: "af_bella" :: nil); + sys->sleep(300); +} + +testFiles(t: ref T) +{ + files := array[] of { + "ctl", "say", "sayq", "hear", "listen", "wake", "cancel", "voices" + }; + for(i := 0; i < len files; i++) + t.assert(pathexists(MNT + "/" + files[i]), files[i] + " should exist"); +} + +testConfig(t: ref T) +{ + ctl := readfile(MNT + "/ctl"); + t.assert(ctl != nil, "ctl should be readable"); + t.assert(ctl != nil && len ctl > 0, "ctl should not be empty"); + t.assert(writefile(MNT + "/ctl", "engine kokoro") > 0, "engine kokoro accepted"); + t.assert(writefile(MNT + "/ctl", "voice af_bella") > 0, "voice accepted"); + t.assert(writefile(MNT + "/ctl", "kokorobin /bin/echo") > 0, "kokoro helper accepted"); + t.assert(writefile(MNT + "/ctl", "ttsengine piper") > 0, "tts engine accepted"); + t.assert(writefile(MNT + "/ctl", "listenengine whisper") > 0, "listen engine accepted"); + t.assert(writefile(MNT + "/ctl", "whisperstreambin /bin/echo final test transcript") > 0, + "streaming helper accepted"); + t.assert(writefile(MNT + "/ctl", "wakebin /bin/echo wake score=1.0") > 0, + "wake helper accepted"); + t.assert(writefile(MNT + "/ctl", "wakeword hey lucia") > 0, "wake word accepted"); + t.assert(writefile(MNT + "/ctl", "wakethreshold 0.7") > 0, "wake threshold accepted"); + t.assert(writefile(MNT + "/ctl", "parakeetmount /n/parakeet") > 0, "parakeet mount accepted"); + t.assert(writefile(MNT + "/ctl", "parakeetlisten /n/parakeet/listen") > 0, + "parakeet listen mount accepted"); + t.assert(writefile(MNT + "/ctl", "pipersay /n/parakeet/say") > 0, + "piper say mount accepted"); + ctl = readfile(MNT + "/ctl"); + t.assert(ctl != nil && len ctl > 0, "ctl remains readable after config writes"); + t.assert(hassubstr(ctl, "engine kokoro"), "ctl reports kokoro engine"); + t.assert(hassubstr(ctl, "kokorobin /bin/echo"), "ctl reports kokoro helper"); + t.assert(hassubstr(ctl, "ttsengine piper"), "ctl reports tts engine"); + t.assert(hassubstr(ctl, "listenengine whisper"), "ctl reports listen engine"); + t.assert(hassubstr(ctl, "wakeword hey lucia"), "ctl reports wake word"); + t.assert(hassubstr(ctl, "wakethreshold 0.7"), "ctl reports wake threshold"); + t.assert(hassubstr(ctl, "parakeetmount /n/parakeet"), "ctl reports parakeet mount"); + t.assert(hassubstr(ctl, "parakeetlisten /n/parakeet/listen"), + "ctl reports parakeet listen mount"); + t.assert(hassubstr(ctl, "pipersay /n/parakeet/say"), + "ctl reports piper say mount"); +} + +# speech9p no longer runs helper binaries itself: listen and wake are +# consumed from the configured provider mount (speechshim9p, a parakeet +# export, or — as here — plain files standing in for a provider). +testListenWakeHelpers(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "provider " + PARAKEETMNT) > 0, + "configure provider mount"); + t.assert(createfile(PARAKEETMNT + "/ctl", "") >= 0, + "create fake provider ctl file"); + t.assert(writefile(MNT + "/ctl", "whispermodel /tmp/ggml-test.bin") > 0, + "whisper model accepted"); + providerctl := readfile(PARAKEETMNT + "/ctl"); + t.assert(hassubstr(providerctl, "whispermodel /tmp/ggml-test.bin"), + "whisper model forwarded to provider"); + t.assert(createfile(PARAKEETMNT + "/listen", "final helper transcript\n") > 0, + "create fake provider listen file"); + t.assert(createfile(PARAKEETMNT + "/wake", "wake hey_lucia 0.88\n") > 0, + "create fake provider wake file"); + + listen := readfile(MNT + "/listen"); + t.assert(hassubstr(listen, "final helper transcript"), + "listen returns provider stream output"); + wake := readfile(MNT + "/wake"); + t.assert(hassubstr(wake, "wake hey_lucia"), + "wake returns provider event"); +} + +testParakeetListenMount(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "listenengine parakeet") > 0, + "configure parakeet listen engine"); + t.assert(writefile(MNT + "/ctl", "parakeetmount " + PARAKEETMNT) > 0, + "configure parakeet mount prefix"); + t.assert(writefile(MNT + "/ctl", "parakeetlisten " + PARAKEETMNT + "/listen") > 0, + "configure parakeet listen file"); + t.assert(createfile(PARAKEETMNT + "/listen", "final parakeet transcript\n") > 0, + "create fake mounted parakeet listen file"); + listen := readfile(MNT + "/listen"); + t.assert(hassubstr(listen, "final parakeet transcript"), + "listen returns mounted parakeet stream output"); + + t.assert(writefile(MNT + "/ctl", "listenengine whisper") > 0, + "restore default listen engine for later tests"); +} + +testPiperSayMount(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "ttsengine piper") > 0, + "configure piper tts engine"); + t.assert(writefile(MNT + "/ctl", "parakeetmount " + PARAKEETMNT) > 0, + "configure parakeet mount prefix"); + t.assert(createfile(PARAKEETMNT + "/say", "") >= 0, + "create fake mounted piper say file"); + result := writesayread(MNT + "/say", "mounted piper tts"); + t.assert(hassubstr(result, "mounted piper tts"), + "say returns mounted piper say status"); + written := readfile(PARAKEETMNT + "/say"); + t.assert(hassubstr(written, "mounted piper tts"), + "speech9p delegated say write to mounted piper say file"); + t.assert(writefile(MNT + "/ctl", "ttsengine engine") > 0, + "restore default tts engine for later tests"); +} + +# Match lucibridge's completion-aware client contract: sayq is opened ORDWR, +# written once, rewound, and read for the terminal status of that utterance. +testSayqCompletion(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "ttsengine piper") > 0, + "configure piper tts engine"); + t.assert(writefile(MNT + "/ctl", "parakeetmount " + PARAKEETMNT) > 0, + "configure parakeet mount prefix"); + t.assert(createfile(PARAKEETMNT + "/say", "") >= 0, + "create fake mounted piper say file"); + result := writesayread(MNT + "/sayq", "completion aware speech"); + t.assert(result != nil && len result > 0, + "sayq returns a terminal completion status"); + written := readfile(PARAKEETMNT + "/say"); + t.assert(hassubstr(written, "completion aware speech"), + "sayq delivered text to the fake provider"); + t.assert(writefile(MNT + "/ctl", "ttsengine engine") > 0, + "restore default tts engine"); +} + +testDisEngineModule(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "provider " + PARAKEETMNT) > 0, + "configure module provider mount"); + t.assert(createfile(PARAKEETMNT + "/say", "") >= 0, + "create module provider say file"); + t.assert(createfile(PARAKEETMNT + "/listen", "final module transcript\n") > 0, + "create module provider listen file"); + t.assert(createfile(PARAKEETMNT + "/voices", "module_voice\n") > 0, + "create module provider voices file"); + t.assert(writefile(MNT + "/ctl", + "module /dis/veltro/speechprovider.dis") > 0, + "load provider-backed SpeechEngine module"); + ctl := readfile(MNT + "/ctl"); + t.assert(hassubstr(ctl, "engine module"), + "ctl reports dynamically loaded engine selection"); + t.assert(hassubstr(ctl, "modulename provider"), + "ctl reports loaded module name"); + voices := readfile(MNT + "/voices"); + t.assert(hassubstr(voices, "module_voice"), + "voices are supplied by the loaded module"); + result := writesayread(MNT + "/sayq", "module-backed speech"); + t.assert(hassubstr(result, "ok"), + "module-backed sayq returns terminal status"); + written := readfile(PARAKEETMNT + "/say"); + t.assert(hassubstr(written, "module-backed speech"), + "loaded module delegated speech through its provider namespace"); + t.assert(writefile(MNT + "/ctl", "engine kokoro") > 0, + "restore provider engine for later tests"); +} + +testCancel(t: ref T) +{ + t.assert(writefile(MNT + "/cancel", "cancel") > 0, "cancel write accepted"); + state := strip(readfile(MNT + "/cancel")); + t.assert(state == "cancel pending" || state == "idle", + "cancel state should be readable"); +} + +teardown() +{ + sys->unmount(nil, MNT); +} + +init(nil: ref Draw->Context, args: list of string) +{ + sys = load Sys Sys->PATH; + testing = load Testing Testing->PATH; + if(testing == nil) + raise "fail:load testing"; + testing->init(); + for(a := args; a != nil; a = tl a) + if(hd a == "-v") + testing->verbose(1); + + startserver(); + run("Files", testFiles); + run("Config", testConfig); + run("ListenWakeHelpers", testListenWakeHelpers); + run("ParakeetListenMount", testParakeetListenMount); + run("PiperSayMount", testPiperSayMount); + run("SayqCompletion", testSayqCompletion); + run("DisEngineModule", testDisEngineModule); + run("Cancel", testCancel); + + teardown(); + if(testing->summary(passed, failed, skipped) > 0) + raise "fail:tests failed"; +} diff --git a/tests/speech_e2e_test.b b/tests/speech_e2e_test.b new file mode 100644 index 000000000..77149b75a --- /dev/null +++ b/tests/speech_e2e_test.b @@ -0,0 +1,331 @@ +implement SpeechE2ETest; + +# +# Composed voice-mode integration test. The real Lucia, LLM, speech9p, +# speechshim9p, lucibridge, and voicemode services run together. External +# speech models are replaced by tests/host/speech_e2e_helper.sh, and the +# OpenAI-compatible endpoint is supplied by speech_e2e_test.sh. +# + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "arg.m"; + arg: Arg; + +include "testing.m"; + testing: Testing; + T: import testing; + +Command: module { + init: fn(nil: ref Draw->Context, args: list of string); +}; + +SpeechE2ETest: module { + init: fn(nil: ref Draw->Context, args: list of string); + _marker: fn(); +}; + +SRCFILE: con "/tests/speech_e2e_test.b"; + +passed := 0; +failed := 0; +skipped := 0; + +apiurl: string; +hoststate: string; +infernostate: string; +helper: string; + +_marker() {} + +run(name: string, testfn: ref fn(t: ref T)) +{ + t := testing->newTsrc(name, SRCFILE); + { + testfn(t); + } exception { + "fail:fatal" => + ; + "fail:skip" => + ; + "*" => + t.failed = 1; + } + if(testing->done(t)) + passed++; + else if(t.skipped) + skipped++; + else + failed++; +} + +writefile(path, data: string): int +{ + fd := sys->open(path, Sys->OWRITE); + if(fd == nil) + return -1; + b := array of byte data; + return sys->write(fd, b, len b); +} + +createfile(path: string): int +{ + fd := sys->create(path, Sys->OWRITE, 8r666); + if(fd == nil) + return -1; + return 0; +} + +createwithdata(path, data: string): int +{ + fd := sys->create(path, Sys->OWRITE, 8r644); + if(fd == nil) + return -1; + b := array of byte data; + return sys->write(fd, b, len b); +} + +readfile(path: string): string +{ + fd := sys->open(path, Sys->OREAD); + if(fd == nil) + return nil; + buf := array[16384] of byte; + n := sys->read(fd, buf, len buf); + if(n <= 0) + return nil; + return string buf[0:n]; +} + +contains(s, sub: string): int +{ + if(s == nil || sub == nil || len sub > len s) + return 0; + for(i := 0; i <= len s - len sub; i++) + if(s[i:i + len sub] == sub) + return 1; + return 0; +} + +pathexists(path: string): int +{ + (ok, nil) := sys->stat(path); + return ok >= 0; +} + +waitpath(path: string, ms: int): int +{ + for(waited := 0; waited < ms; waited += 50) { + if(pathexists(path)) + return 1; + sys->sleep(50); + } + return 0; +} + +waitcontains(path, sub: string, ms: int): int +{ + for(waited := 0; waited < ms; waited += 50) { + if(contains(readfile(path), sub)) + return 1; + sys->sleep(50); + } + return 0; +} + +conversationrolecount(role, sub: string): int +{ + n := 0; + for(i := 0; i < 12; i++) { + msg := readfile("/mnt/ui/activity/0/conversation/" + string i); + if(contains(msg, "role=" + role) && contains(msg, sub)) + n++; + } + return n; +} + +waitconversationrole(role, sub: string, ms: int): int +{ + for(waited := 0; waited < ms; waited += 50) { + if(conversationrolecount(role, sub) > 0) + return 1; + sys->sleep(50); + } + return 0; +} + +resourcecontains(sub: string): int +{ + for(i := 0; i < 20; i++) { + resource := readfile("/mnt/ui/activity/0/context/resources/" + string i); + if(resource == nil) + break; + if(contains(resource, sub)) + return 1; + } + return 0; +} + +waitresource(sub: string, ms: int): int +{ + for(waited := 0; waited < ms; waited += 50) { + if(resourcecontains(sub)) + return 1; + sys->sleep(50); + } + return 0; +} + +startmodule(t: ref T, path, name: string, args: list of string) +{ + cmd := load Command path; + if(cmd == nil) + t.fatal("cannot load " + path + ": " + sys->sprint("%r")); + spawn cmd->init(nil, name :: args); +} + +preparemounts() +{ + sys->create("/tmp", Sys->OREAD, Sys->DMDIR | 8r777); + sys->create("/mnt", Sys->OREAD, Sys->DMDIR | 8r755); + sys->create("/n", Sys->OREAD, Sys->DMDIR | 8r755); + sys->create("/mnt/ui", Sys->OREAD, Sys->DMDIR | 8r755); + sys->create("/mnt/llm", Sys->OREAD, Sys->DMDIR | 8r755); + sys->create("/n/speechshim", Sys->OREAD, Sys->DMDIR | 8r755); + sys->create("/n/speech", Sys->OREAD, Sys->DMDIR | 8r755); +} + +startstack(t: ref T) +{ + preparemounts(); + + startmodule(t, "/dis/luciuisrv.dis", "luciuisrv", "-m" :: "/mnt/ui" :: nil); + t.assert(waitpath("/mnt/ui/ctl", 3000), "luciuisrv mounted"); + + startmodule(t, "/dis/veltro/speechshim9p.dis", "speechshim9p", + "-m" :: "/n/speechshim" :: nil); + t.assert(waitpath("/n/speechshim/ctl", 3000), "speechshim9p mounted"); + + cmd := "/bin/sh " + helper; + t.assert(writefile("/n/speechshim/ctl", + "wakebin " + cmd + " wake " + hoststate) > 0, "fake wake helper configured"); + t.assert(writefile("/n/speechshim/ctl", + "whisperstreambin " + cmd + " listen " + hoststate) > 0, + "fake listen helper configured"); + t.assert(writefile("/n/speechshim/ctl", + "kokorobin " + cmd + " say " + hoststate) > 0, "fake TTS helper configured"); + t.assert(createfile(infernostate + "/audio.pcm") >= 0, "fake audio sink created"); + t.assert(writefile("/n/speechshim/ctl", "audiodev " + infernostate + "/audio.pcm") > 0, + "fake audio sink configured"); + t.assert(writefile("/n/speechshim/ctl", "duplex half") > 0, + "half-duplex provider configured"); + + startmodule(t, "/dis/veltro/speech9p.dis", "speech9p", + "-m" :: "/n/speech" :: "-e" :: "kokoro" :: nil); + t.assert(waitpath("/n/speech/ctl", 3000), "speech9p mounted"); + t.assert(writefile("/n/speech/ctl", "provider /n/speechshim") > 0, + "speechshim selected as provider"); + + startmodule(t, "/dis/llmsrv.dis", "llmsrv", + "-m" :: "/mnt/llm" :: "-b" :: "openai" :: "-u" :: apiurl :: + "-M" :: "ci-voice-e2e" :: "-r" :: "low" :: nil); + t.assert(waitpath("/mnt/llm/new", 3000), "llmsrv mounted"); + + # lucibridge deliberately validates deployment configuration before it + # trusts /mnt/llm. Overlay a private OpenAI configuration in this test + # namespace; never read or modify the host user's real configuration. + config := "mode=local\nbackend=openai\nurl=" + apiurl + + "\nmodel=ci-voice-e2e\ntemperature=0.2\n"; + t.assert(createwithdata(infernostate + "/llm.ndb", config) > 0, + "private LLM configuration created"); + t.assert(sys->bind(infernostate + "/llm.ndb", "/lib/ndb/llm", Sys->MREPL) >= 0, + "private LLM configuration bound"); + + t.assert(writefile("/mnt/ui/ctl", "activity create VoiceE2E") > 0, + "Lucia activity created"); + t.assert(waitpath("/mnt/ui/activity/0/conversation/voiceinput", 3000), + "voice input endpoint created"); + + startmodule(t, "/dis/lucibridge.dis", "lucibridge", + "-s" :: "-n" :: "3" :: "-a" :: "0" :: nil); + t.assert(waitresource("label=Voice", 8000), + "lucibridge initialized its LLM session and speech resource"); + + startmodule(t, "/dis/voicemode.dis", "voicemode", + "-g" :: "300" :: "-q" :: "650" :: "-t" :: "5000" :: + "-w" :: "50" :: "-u" :: "/mnt/ui" :: "-s" :: "/n/speech" :: nil); + sys->sleep(300); +} + +testComposedTurn(t: ref T) +{ + startstack(t); + + t.assert(writefile(infernostate + "/wake.next", "wake e2e 0.99\n") > 0, + "wake event scripted"); + t.assert(writefile(infernostate + "/listen.next", + "partial confidence=940 Reply with exactly local LLM working\n" + + "final confidence=940 Reply with exactly: local LLM working.\n") > 0, + "streaming transcript scripted"); + t.assert(writefile("/mnt/ui/input-mode", "v") > 0, "voice mode re-entered"); + + t.assert(waitcontains("/mnt/ui/activity/0/conversation/draft", + "local LLM working", 5000), "live or final transcript reached Lucia draft"); + t.assert(waitconversationrole("human", "local LLM working", 8000), + "final transcript submitted to lucibridge"); + t.assert(waitconversationrole("veltro", "local LLM working", 12000), + "local OpenAI response returned to Lucia"); + t.assert(waitcontains(infernostate + "/say.log", "local LLM working", 8000), + "assistant response reached speech provider"); + t.asserteq(conversationrolecount("human", "local LLM working"), 1, + "final transcript submitted exactly once"); + t.assert(waitresource("label=Voice", 3000), + "Voice lifecycle resource is present"); + + t.assert(writefile("/mnt/ui/input-mode", "k") > 0, "keyboard mode restored"); + t.assert(waitcontains("/n/speechshim/ctl", "mic off", 3000), + "microphone released after composed turn"); +} + +testNeedsWrapper(t: ref T) +{ + t.skip("requires tests/host/speech_e2e_test.sh"); +} + +init(nil: ref Draw->Context, args: list of string) +{ + sys = load Sys Sys->PATH; + testing = load Testing Testing->PATH; + if(testing == nil) { + sys->fprint(sys->fildes(2), "cannot load testing: %r\n"); + raise "fail:load"; + } + testing->init(); + + arg = load Arg Arg->PATH; + if(arg == nil) { + sys->fprint(sys->fildes(2), "cannot load arg: %r\n"); + raise "fail:load"; + } + arg->init(args); + while((o := arg->opt()) != 0) + case o { + 'u' => apiurl = arg->earg(); + 'H' => hoststate = arg->earg(); + 'I' => infernostate = arg->earg(); + 'X' => helper = arg->earg(); + 'v' => testing->verbose(1); + * => ; + } + + if(apiurl == nil || hoststate == nil || infernostate == nil || helper == nil) + run("HostWrapperRequired", testNeedsWrapper); + else + run("ComposedVoiceTurn", testComposedTurn); + + if(testing->summary(passed, failed, skipped) > 0) + raise "fail:tests failed"; +} diff --git a/tests/speech_kokoro_test.b b/tests/speech_kokoro_test.b new file mode 100644 index 000000000..ae1819e01 --- /dev/null +++ b/tests/speech_kokoro_test.b @@ -0,0 +1,227 @@ +implement SpeechKokoroTest; + +# +# Kokoro TTS engine plumbing (Phase 1.1/1.6) through the unified provider +# stack: engine kokoro delegates say/voices to the speechshim9p provider +# mount, which runs the helper. The real kokoro helper is an external host +# install and is never vendored, so this exercises engine selection, voice +# configuration, provider voices listing, and the say path's status +# reporting with a fake helper — the smoke contract that survives on a +# machine with no speech stack installed. +# + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "testing.m"; + testing: Testing; + T: import testing; + +Speech9pSrv: module { + init: fn(nil: ref Draw->Context, args: list of string); +}; + +SpeechKokoroTest: module { + init: fn(nil: ref Draw->Context, args: list of string); + _marker: fn(); # prevents joiniface() type conflation with Speech9pSrv +}; + +SRCFILE: con "/tests/speech_kokoro_test.b"; +SRVPATH: con "/dis/veltro/speech9p.dis"; +SHIMPATH: con "/dis/veltro/speechshim9p.dis"; +MNT: con "/tmp/speech_kokoro_test"; +SHIMMNT: con "/tmp/speech_kokoro_test_shim"; + +passed := 0; +failed := 0; +skipped := 0; + +_marker() {} + +run(name: string, testfn: ref fn(t: ref T)) +{ + t := testing->newTsrc(name, SRCFILE); + { + testfn(t); + } exception { + "fail:fatal" => + ; + "fail:skip" => + ; + "*" => + t.failed = 1; + } + if(testing->done(t)) + passed++; + else if(t.skipped) + skipped++; + else + failed++; +} + +writefile(path, data: string): int +{ + fd := sys->open(path, Sys->OWRITE); + if(fd == nil) + return -1; + b := array of byte data; + return sys->write(fd, b, len b); +} + +readfile(path: string): string +{ + fd := sys->open(path, Sys->OREAD); + if(fd == nil) + return nil; + buf := array[8192] of byte; + n := sys->read(fd, buf, len buf); + if(n < 0) + return nil; + return string buf[0:n]; +} + +writesayread(path, data: string): string +{ + fd := sys->open(path, Sys->ORDWR); + if(fd == nil) + return nil; + b := array of byte data; + if(sys->write(fd, b, len b) < 0) + return nil; + sys->seek(fd, big 0, Sys->SEEKSTART); + buf := array[8192] of byte; + n := sys->read(fd, buf, len buf); + if(n < 0) + return nil; + return string buf[0:n]; +} + +hassubstr(s, sub: string): int +{ + if(s == nil || sub == nil || len sub > len s) + return 0; + for(i := 0; i <= len s - len sub; i++) + if(s[i:i+len sub] == sub) + return 1; + return 0; +} + +startserver() +{ + sys->create("/tmp", Sys->OREAD, Sys->DMDIR | 8r777); + sys->create(MNT, Sys->OREAD, Sys->DMDIR | 8r755); + sys->create(SHIMMNT, Sys->OREAD, Sys->DMDIR | 8r755); + shim := load Speech9pSrv SHIMPATH; + if(shim == nil) { + sys->fprint(sys->fildes(2), "cannot load speechshim9p: %r\n"); + raise "fail:load"; + } + spawn shim->init(nil, "speechshim9p" :: "-m" :: SHIMMNT :: nil); + srv := load Speech9pSrv SRVPATH; + if(srv == nil) { + sys->fprint(sys->fildes(2), "cannot load speech9p: %r\n"); + raise "fail:load"; + } + spawn srv->init(nil, "speech9p" :: "-m" :: MNT :: "-e" :: "kokoro" :: "-v" :: "af_bella" :: nil); + sys->sleep(300); + writefile(MNT + "/ctl", "provider " + SHIMMNT); +} + +testEngineSelection(t: ref T) +{ + ctl := readfile(MNT + "/ctl"); + t.assert(hassubstr(ctl, "engine kokoro"), "ctl reports kokoro engine"); + t.assert(hassubstr(ctl, "voice af_bella"), "ctl reports kokoro default voice"); + t.assert(writefile(MNT + "/ctl", "voice am_adam") > 0, "kokoro voice id passes through"); + ctl = readfile(MNT + "/ctl"); + t.assert(hassubstr(ctl, "voice am_adam"), "ctl reports updated voice"); + t.assert(writefile(MNT + "/ctl", "voice af_bella") > 0, "restore default voice"); +} + +testVoicesListing(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "kokorobin /bin/echo af_bella am_adam") > 0, + "configure fake kokoro helper"); + voices := readfile(MNT + "/voices"); + t.assert(voices != nil && len voices > 0, "voices readable with kokoro engine"); + t.assert(hassubstr(voices, "af_bella"), "voices includes helper output"); +} + +testSayStatus(t: ref T) +{ + # The fake helper emits text instead of PCM; the say path must still + # produce a readable status (playback succeeds or reports an error — + # either way the write-then-read contract holds and nothing hangs). + t.assert(writefile(MNT + "/ctl", "kokorobin /bin/echo") > 0, + "configure fake kokoro helper"); + status := writesayread(MNT + "/say", "hello from kokoro test"); + t.assert(status != nil, "say status readable after kokoro synthesis"); +} + +testEngineSwitchRoundTrip(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "engine cmd") > 0, "switch to cmd engine"); + ctl := readfile(MNT + "/ctl"); + t.assert(hassubstr(ctl, "engine cmd"), "ctl reports cmd engine"); + t.assert(writefile(MNT + "/ctl", "engine kokoro") > 0, "switch back to kokoro"); + ctl = readfile(MNT + "/ctl"); + t.assert(hassubstr(ctl, "engine kokoro"), "ctl reports kokoro engine again"); +} + +killmodule(name: string) +{ + fd := sys->open("/prog", Sys->OREAD); + if(fd == nil) + return; + for(;;) { + (n, dirs) := sys->dirread(fd); + if(n <= 0) + break; + for(i := 0; i < n; i++) { + pid := dirs[i].name; + status := readfile("/prog/" + pid + "/status"); + if(!hassubstr(status, name)) + continue; + ctl := sys->open("/prog/" + pid + "/ctl", Sys->OWRITE); + if(ctl != nil) + sys->fprint(ctl, "killgrp"); + } + } +} + +teardown() +{ + # Unmount both servers so their serveloops see EOF and exit — + # otherwise emu never halts after the tests finish. + sys->unmount(nil, MNT); + sys->sleep(100); + sys->unmount(nil, SHIMMNT); + sys->sleep(100); + # The provider say path can leave the shim's released process group + # referenced after its mount is gone; do not let test cleanup hang emu. + killmodule("Speechshim9p"); +} + +init(nil: ref Draw->Context, args: list of string) +{ + sys = load Sys Sys->PATH; + testing = load Testing Testing->PATH; + if(testing == nil) + raise "fail:load testing"; + testing->init(); + for(a := args; a != nil; a = tl a) + if(hd a == "-v") + testing->verbose(1); + + startserver(); + run("EngineSelection", testEngineSelection); + run("VoicesListing", testVoicesListing); + run("SayStatus", testSayStatus); + run("EngineSwitchRoundTrip", testEngineSwitchRoundTrip); + + teardown(); + if(testing->summary(passed, failed, skipped) > 0) + raise "fail:tests failed"; +} diff --git a/tests/speech_listen_test.b b/tests/speech_listen_test.b new file mode 100644 index 000000000..bed888dfd --- /dev/null +++ b/tests/speech_listen_test.b @@ -0,0 +1,222 @@ +implement SpeechListenTest; + +# +# Streaming STT listen file (Phase 1.2/1.6) through the unified provider +# stack: speech9p consumes the speechshim9p mount, which adapts fake host +# helpers configured via the ctl-forwarded whisperstreambin key. Verifies +# the record wire format is passed through unparsed (partial/final/error +# records are interpreted by voicemode, not speech9p) and that a listen +# read blocked in a slow helper does not freeze either serveloop. +# + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "testing.m"; + testing: Testing; + T: import testing; + +Speech9pSrv: module { + init: fn(nil: ref Draw->Context, args: list of string); +}; + +SpeechListenTest: module { + init: fn(nil: ref Draw->Context, args: list of string); + _marker: fn(); # prevents joiniface() type conflation with Speech9pSrv +}; + +SRCFILE: con "/tests/speech_listen_test.b"; +SRVPATH: con "/dis/veltro/speech9p.dis"; +SHIMPATH: con "/dis/veltro/speechshim9p.dis"; +MNT: con "/tmp/speech_listen_test"; +SHIMMNT: con "/tmp/speech_listen_test_shim"; + +passed := 0; +failed := 0; +skipped := 0; + +_marker() {} + +run(name: string, testfn: ref fn(t: ref T)) +{ + t := testing->newTsrc(name, SRCFILE); + { + testfn(t); + } exception { + "fail:fatal" => + ; + "fail:skip" => + ; + "*" => + t.failed = 1; + } + if(testing->done(t)) + passed++; + else if(t.skipped) + skipped++; + else + failed++; +} + +strip(s: string): string +{ + if(s == nil) + return nil; + i := 0; + while(i < len s && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n')) + i++; + j := len s; + while(j > i && (s[j-1] == ' ' || s[j-1] == '\t' || s[j-1] == '\r' || s[j-1] == '\n')) + j--; + if(i >= j) + return ""; + return s[i:j]; +} + +writefile(path, data: string): int +{ + fd := sys->open(path, Sys->OWRITE); + if(fd == nil) + return -1; + b := array of byte data; + return sys->write(fd, b, len b); +} + +readfile(path: string): string +{ + fd := sys->open(path, Sys->OREAD); + if(fd == nil) + return nil; + buf := array[8192] of byte; + n := sys->read(fd, buf, len buf); + if(n < 0) + return nil; + return string buf[0:n]; +} + +hassubstr(s, sub: string): int +{ + if(s == nil || sub == nil || len sub > len s) + return 0; + for(i := 0; i <= len s - len sub; i++) + if(s[i:i+len sub] == sub) + return 1; + return 0; +} + +reader(path: string, ch: chan of string) +{ + ch <-= readfile(path); +} + +startserver() +{ + sys->create("/tmp", Sys->OREAD, Sys->DMDIR | 8r777); + sys->create(MNT, Sys->OREAD, Sys->DMDIR | 8r755); + sys->create(SHIMMNT, Sys->OREAD, Sys->DMDIR | 8r755); + shim := load Speech9pSrv SHIMPATH; + if(shim == nil) { + sys->fprint(sys->fildes(2), "cannot load speechshim9p: %r\n"); + raise "fail:load"; + } + spawn shim->init(nil, "speechshim9p" :: "-m" :: SHIMMNT :: nil); + srv := load Speech9pSrv SRVPATH; + if(srv == nil) { + sys->fprint(sys->fildes(2), "cannot load speech9p: %r\n"); + raise "fail:load"; + } + spawn srv->init(nil, "speech9p" :: "-m" :: MNT :: "-e" :: "kokoro" :: nil); + sys->sleep(300); + writefile(MNT + "/ctl", "provider " + SHIMMNT); +} + +testFinalRecord(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "listenengine whisper") > 0, + "configure whisper listen engine"); + t.assert(writefile(MNT + "/ctl", "whisperstreambin /bin/echo final listen transcript") > 0, + "configure fake listen helper"); + listen := readfile(MNT + "/listen"); + t.assert(hassubstr(listen, "final listen transcript"), + "listen passes final record through"); +} + +testPartialRecordPassthrough(t: ref T) +{ + # speech9p does not interpret records; a partial hypothesis must reach + # the reader verbatim so voicemode can decide what to do with it. + t.assert(writefile(MNT + "/ctl", "whisperstreambin /bin/echo partial half a thou") > 0, + "configure partial-emitting helper"); + listen := strip(readfile(MNT + "/listen")); + t.assert(hassubstr(listen, "partial half a thou"), + "listen passes partial record through unparsed"); +} + +testListenHelperEmpty(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "whisperstreambin /bin/sh -c \"exit 0\"") > 0, + "configure silent listen helper"); + listen := strip(readfile(MNT + "/listen")); + t.assert(hassubstr(listen, "error:"), "empty helper output becomes an error record"); +} + +# Same serveloop-liveness property as the wake test: a blocked listen read +# must not stall ctl reads or the cancel write barge-in depends on. +testListenDoesNotBlockServer(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", + "whisperstreambin /bin/sh -c \"sleep 2; echo final slow transcript\"") > 0, + "configure slow listen helper"); + + ch := chan of string; + spawn reader(MNT + "/listen", ch); + sys->sleep(200); # let the listen read reach the helper + + t0 := sys->millisec(); + ctl := readfile(MNT + "/ctl"); + t1 := sys->millisec(); + t.assert(ctl != nil && len ctl > 0, "ctl readable while listen helper is busy"); + t.assert(t1 - t0 < 1500, "ctl read not stalled behind the listen helper"); + + t0 = sys->millisec(); + t.assert(writefile(MNT + "/cancel", "cancel") > 0, + "cancel write served while listen helper is busy"); + t1 = sys->millisec(); + t.assert(t1 - t0 < 1500, "cancel write not stalled behind the listen helper"); + + listen := <-ch; + t.assert(hassubstr(listen, "final slow transcript"), + "slow final transcript still delivered"); +} + +teardown() +{ + # Unmount both servers so their serveloops see EOF and exit — + # otherwise emu never halts after the tests finish. + sys->unmount(nil, MNT); + sys->unmount(nil, SHIMMNT); +} + +init(nil: ref Draw->Context, args: list of string) +{ + sys = load Sys Sys->PATH; + testing = load Testing Testing->PATH; + if(testing == nil) + raise "fail:load testing"; + testing->init(); + for(a := args; a != nil; a = tl a) + if(hd a == "-v") + testing->verbose(1); + + startserver(); + run("FinalRecord", testFinalRecord); + run("PartialRecordPassthrough", testPartialRecordPassthrough); + run("ListenHelperEmpty", testListenHelperEmpty); + run("ListenDoesNotBlockServer", testListenDoesNotBlockServer); + + teardown(); + if(testing->summary(passed, failed, skipped) > 0) + raise "fail:tests failed"; +} diff --git a/tests/speech_wake_test.b b/tests/speech_wake_test.b new file mode 100644 index 000000000..dd3205f81 --- /dev/null +++ b/tests/speech_wake_test.b @@ -0,0 +1,228 @@ +implement SpeechWakeTest; + +# +# Wake-word file behavior (Phase 1.3/1.6) through the unified provider +# stack: speech9p consumes /n/speechshim (speechshim9p), which adapts fake +# host helpers configured via the ctl-forwarded wakebin key — no real wake +# model is needed. The central assertion is that a wake read blocked in a +# slow helper does NOT freeze either serveloop: ctl reads and cancel writes +# must still be served while the wake helper runs, because barge-in depends +# on exactly that. +# + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "testing.m"; + testing: Testing; + T: import testing; + +Speech9pSrv: module { + init: fn(nil: ref Draw->Context, args: list of string); +}; + +SpeechWakeTest: module { + init: fn(nil: ref Draw->Context, args: list of string); + _marker: fn(); # prevents joiniface() type conflation with Speech9pSrv +}; + +SRCFILE: con "/tests/speech_wake_test.b"; +SRVPATH: con "/dis/veltro/speech9p.dis"; +SHIMPATH: con "/dis/veltro/speechshim9p.dis"; +MNT: con "/tmp/speech_wake_test"; +SHIMMNT: con "/tmp/speech_wake_test_shim"; + +passed := 0; +failed := 0; +skipped := 0; + +_marker() {} + +run(name: string, testfn: ref fn(t: ref T)) +{ + t := testing->newTsrc(name, SRCFILE); + { + testfn(t); + } exception { + "fail:fatal" => + ; + "fail:skip" => + ; + "*" => + t.failed = 1; + } + if(testing->done(t)) + passed++; + else if(t.skipped) + skipped++; + else + failed++; +} + +strip(s: string): string +{ + if(s == nil) + return nil; + i := 0; + while(i < len s && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n')) + i++; + j := len s; + while(j > i && (s[j-1] == ' ' || s[j-1] == '\t' || s[j-1] == '\r' || s[j-1] == '\n')) + j--; + if(i >= j) + return ""; + return s[i:j]; +} + +writefile(path, data: string): int +{ + fd := sys->open(path, Sys->OWRITE); + if(fd == nil) + return -1; + b := array of byte data; + return sys->write(fd, b, len b); +} + +readfile(path: string): string +{ + fd := sys->open(path, Sys->OREAD); + if(fd == nil) + return nil; + buf := array[8192] of byte; + n := sys->read(fd, buf, len buf); + if(n < 0) + return nil; + return string buf[0:n]; +} + +hassubstr(s, sub: string): int +{ + if(s == nil || sub == nil || len sub > len s) + return 0; + for(i := 0; i <= len s - len sub; i++) + if(s[i:i+len sub] == sub) + return 1; + return 0; +} + +reader(path: string, ch: chan of string) +{ + ch <-= readfile(path); +} + +startserver() +{ + sys->create("/tmp", Sys->OREAD, Sys->DMDIR | 8r777); + sys->create(MNT, Sys->OREAD, Sys->DMDIR | 8r755); + sys->create(SHIMMNT, Sys->OREAD, Sys->DMDIR | 8r755); + shim := load Speech9pSrv SHIMPATH; + if(shim == nil) { + sys->fprint(sys->fildes(2), "cannot load speechshim9p: %r\n"); + raise "fail:load"; + } + spawn shim->init(nil, "speechshim9p" :: "-m" :: SHIMMNT :: nil); + srv := load Speech9pSrv SRVPATH; + if(srv == nil) { + sys->fprint(sys->fildes(2), "cannot load speech9p: %r\n"); + raise "fail:load"; + } + spawn srv->init(nil, "speech9p" :: "-m" :: MNT :: "-e" :: "kokoro" :: nil); + sys->sleep(300); + writefile(MNT + "/ctl", "provider " + SHIMMNT); +} + +testWakeEvent(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "wakebin /bin/echo wake hey_lucia 0.93") > 0, + "configure fake wake helper"); + wake := readfile(MNT + "/wake"); + t.assert(hassubstr(wake, "wake hey_lucia 0.93"), "wake returns helper event"); +} + +testWakeHelperEmpty(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "wakebin /bin/sh -c \"exit 0\"") > 0, + "configure silent wake helper"); + wake := strip(readfile(MNT + "/wake")); + t.assert(hassubstr(wake, "error:"), "empty helper output becomes an error record"); +} + +# The Phase 1 correctness core: while a wake read is blocked in a slow +# helper, the serveloop must keep serving other requests. Before the async +# fix, the ctl read and cancel write below would stall for the full helper +# duration and barge-in was impossible. +testWakeDoesNotBlockServer(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", + "wakebin /bin/sh -c \"sleep 2; echo wake slow-event\"") > 0, + "configure slow wake helper"); + + ch := chan of string; + spawn reader(MNT + "/wake", ch); + sys->sleep(200); # let the wake read reach the helper + + t0 := sys->millisec(); + ctl := readfile(MNT + "/ctl"); + t1 := sys->millisec(); + t.assert(ctl != nil && len ctl > 0, "ctl readable while wake helper is busy"); + t.assert(t1 - t0 < 1500, "ctl read not stalled behind the wake helper"); + + t0 = sys->millisec(); + t.assert(writefile(MNT + "/cancel", "cancel") > 0, + "cancel write served while wake helper is busy"); + t1 = sys->millisec(); + t.assert(t1 - t0 < 1500, "cancel write not stalled behind the wake helper"); + + wake := <-ch; + t.assert(hassubstr(wake, "wake slow-event"), "slow wake event still delivered"); +} + +# A second wake read while one is in flight must fail fast instead of +# queueing behind (or corrupting) the running helper. +testWakeBusy(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", + "wakebin /bin/sh -c \"sleep 2; echo wake busy-event\"") > 0, + "configure slow wake helper"); + + ch := chan of string; + spawn reader(MNT + "/wake", ch); + sys->sleep(200); + second := readfile(MNT + "/wake"); + t.assert(hassubstr(second, "error: wake busy"), + "concurrent wake read reports busy"); + first := <-ch; + t.assert(hassubstr(first, "wake busy-event"), "first wake read gets the event"); +} + +teardown() +{ + # Unmount both servers so their serveloops see EOF and exit — + # otherwise emu never halts after the tests finish. + sys->unmount(nil, MNT); + sys->unmount(nil, SHIMMNT); +} + +init(nil: ref Draw->Context, args: list of string) +{ + sys = load Sys Sys->PATH; + testing = load Testing Testing->PATH; + if(testing == nil) + raise "fail:load testing"; + testing->init(); + for(a := args; a != nil; a = tl a) + if(hd a == "-v") + testing->verbose(1); + + startserver(); + run("WakeEvent", testWakeEvent); + run("WakeHelperEmpty", testWakeHelperEmpty); + run("WakeDoesNotBlockServer", testWakeDoesNotBlockServer); + run("WakeBusy", testWakeBusy); + + teardown(); + if(testing->summary(passed, failed, skipped) > 0) + raise "fail:tests failed"; +} diff --git a/tests/speechshim_test.b b/tests/speechshim_test.b new file mode 100644 index 000000000..e1588c5b5 --- /dev/null +++ b/tests/speechshim_test.b @@ -0,0 +1,504 @@ +implement SpeechshimTest; + +# +# speechshim9p provider contract test. Fake host helpers stand in for the +# external installs. The load-bearing case is CancelKillsSay: cancel must +# kill the synthesizing helper process (devcmd "kill"), so a blocked say +# completes promptly instead of running out the helper — that bound is what +# makes barge-in silence fast with real TTS. +# + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "testing.m"; + testing: Testing; + T: import testing; + +ShimSrv: module { + init: fn(nil: ref Draw->Context, args: list of string); +}; + +SpeechshimTest: module { + init: fn(nil: ref Draw->Context, args: list of string); + _marker: fn(); # prevents joiniface() type conflation with ShimSrv +}; + +SRCFILE: con "/tests/speechshim_test.b"; +SHIMPATH: con "/dis/veltro/speechshim9p.dis"; +MNT: con "/tmp/speechshim_test"; +PCMFILE: con "/tmp/speechshim_test_pcm"; +WAKEPID: con "/tmp/speechshim_suppressed_wake.pid"; + +passed := 0; +failed := 0; +skipped := 0; + +_marker() {} + +run(name: string, testfn: ref fn(t: ref T)) +{ + t := testing->newTsrc(name, SRCFILE); + { + testfn(t); + } exception { + "fail:fatal" => + ; + "fail:skip" => + ; + "*" => + t.failed = 1; + } + if(testing->done(t)) + passed++; + else if(t.skipped) + skipped++; + else + failed++; +} + +writefile(path, data: string): int +{ + fd := sys->open(path, Sys->OWRITE); + if(fd == nil) + return -1; + b := array of byte data; + return sys->write(fd, b, len b); +} + +readfile(path: string): string +{ + fd := sys->open(path, Sys->OREAD); + if(fd == nil) + return nil; + buf := array[8192] of byte; + n := sys->read(fd, buf, len buf); + if(n < 0) + return nil; + return string buf[0:n]; +} + +filesize(path: string): int +{ + (ok, d) := sys->stat(path); + if(ok < 0) + return -1; + return int d.length; +} + +pathexists(path: string): int +{ + (ok, nil) := sys->stat(path); + return ok >= 0; +} + +hassubstr(s, sub: string): int +{ + if(s == nil || sub == nil || len sub > len s) + return 0; + for(i := 0; i <= len s - len sub; i++) + if(s[i:i+len sub] == sub) + return 1; + return 0; +} + +timer(ch: chan of int, ms: int) +{ + sys->sleep(ms); + ch <-= 1; +} + +readproc(path: string, ch: chan of string) +{ + ch <-= readfile(path); +} + +startserver() +{ + sys->create("/tmp", Sys->OREAD, Sys->DMDIR | 8r777); + sys->create(MNT, Sys->OREAD, Sys->DMDIR | 8r755); + srv := load ShimSrv SHIMPATH; + if(srv == nil) { + sys->fprint(sys->fildes(2), "cannot load speechshim9p: %r\n"); + raise "fail:load"; + } + spawn srv->init(nil, "speechshim9p" :: "-m" :: MNT :: nil); + sys->sleep(300); +} + +testFiles(t: ref T) +{ + files := array[] of {"ctl", "listen", "wake", "say", "cancel", "chime", "voices"}; + for(i := 0; i < len files; i++) + t.assert(pathexists(MNT + "/" + files[i]), files[i] + " should exist"); +} + +testConfig(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "wakeword hey lucia") > 0, "wakeword accepted"); + t.assert(writefile(MNT + "/ctl", "wakethreshold 0.7") > 0, "wakethreshold accepted"); + t.assert(writefile(MNT + "/ctl", "voice am_adam") > 0, "voice accepted"); + ctl := readfile(MNT + "/ctl"); + t.assert(hassubstr(ctl, "wakeword hey lucia"), "ctl reports wakeword"); + t.assert(hassubstr(ctl, "wakethreshold 0.7"), "ctl reports wakethreshold"); + t.assert(hassubstr(ctl, "voice am_adam"), "ctl reports voice"); +} + +# A one-shot helper exits after printing its event; the shim must restart +# it on the next read so wake stays armed across events. +testAudioRouting(t: ref T) +{ + ctl := readfile(MNT + "/ctl"); + t.assert(hassubstr(ctl, "audiodev /dev/audio"), "default playback device"); + t.assert(hassubstr(ctl, "micmode helper"), "default capture mode"); + t.assert(hassubstr(ctl, "capturerate 16000"), "default capture rate"); + + t.assert(writefile(MNT + "/ctl", "audiodev /n/phone/audio") > 0, "audiodev accepted"); + t.assert(writefile(MNT + "/ctl", "capturerate 24000") > 0, "capturerate accepted"); + ctl = readfile(MNT + "/ctl"); + t.assert(hassubstr(ctl, "audiodev /n/phone/audio"), "ctl reports audiodev"); + t.assert(hassubstr(ctl, "capturerate 24000"), "ctl reports capturerate"); + + # Invalid values are logged, not applied (ctl writes always succeed). + writefile(MNT + "/ctl", "micmode banana"); + ctl = readfile(MNT + "/ctl"); + t.assert(hassubstr(ctl, "micmode helper"), "invalid micmode not applied"); + + writefile(MNT + "/ctl", "audiodev /dev/audio"); + writefile(MNT + "/ctl", "capturerate 16000"); +} + +testDuplexConfig(t: ref T) +{ + ctl := readfile(MNT + "/ctl"); + t.assert(hassubstr(ctl, "duplex full"), "default duplex is full"); + t.assert(writefile(MNT + "/ctl", "duplex half") > 0, "duplex half accepted"); + ctl = readfile(MNT + "/ctl"); + t.assert(hassubstr(ctl, "duplex half"), "ctl reports duplex half"); + t.assert(writefile(MNT + "/ctl", "duplex full") > 0, "duplex full accepted"); + writefile(MNT + "/ctl", "duplex banana"); + ctl = readfile(MNT + "/ctl"); + t.assert(hassubstr(ctl, "duplex full"), "invalid duplex not applied"); +} + +testChimeAccepted(t: ref T) +{ + fd := sys->create(PCMFILE, Sys->OWRITE, 8r644); + t.assert(fd != nil, "create fake audio device"); + if(fd == nil) + return; + fd = nil; + t.assert(writefile(MNT + "/ctl", "audiodev " + PCMFILE) > 0, "audiodev scratch accepted"); + t.assert(writefile(MNT + "/chime", "wake") > 0, "wake chime write accepted"); + sys->sleep(500); + t.assert(filesize(PCMFILE) > 0, "chime wrote PCM bytes"); + writefile(MNT + "/ctl", "audiodev /dev/audio"); +} + +# micmode device: the shim itself reads PCM from the capture device and +# feeds the listen helper's stdin — the property that makes a 9P-imported +# microphone (remote instance, Android phone) work like the local one. A +# plain file stands in for the device; the fake helper consumes 8 bytes of +# stdin before emitting its record, so the record proves audio actually +# flowed capture-device → pump → helper stdin. +testDeviceCapture(t: ref T) +{ + fd := sys->create(PCMFILE, Sys->OWRITE, 8r644); + t.assert(fd != nil, "create fake capture device"); + if(fd == nil) + return; + b := array of byte "0123456789abcdef"; + sys->write(fd, b, len b); + fd = nil; + + t.assert(writefile(MNT + "/ctl", "capturedev " + PCMFILE) > 0, "capturedev accepted"); + t.assert(writefile(MNT + "/ctl", "micmode device") > 0, "micmode device accepted"); + t.assert(writefile(MNT + "/ctl", + "whisperstreambin /bin/sh -c \"head -c 8 > /dev/null; echo final device audio heard\"") > 0, + "configure stdin-consuming fake listen helper"); + + listen := readfile(MNT + "/listen"); + t.assert(hassubstr(listen, "final device audio heard"), + "helper fed from the capture device produced its record"); + + # The shim, not deployment-specific ctl text, owns the stdin contract. + # This prevents device mode from accidentally reopening the helper's host mic. + t.assert(writefile(MNT + "/ctl", + "whisperstreambin /bin/echo final device argv") > 0, + "configure argv-reporting device listen helper"); + listen = readfile(MNT + "/listen"); + t.assert(hassubstr(listen, "--stdin --model"), + "device listen helper receives stdin and model flags"); + t.assert(hassubstr(listen, "--rate 16000 --chans 1"), + "device listen helper receives capture format"); + + t.assert(writefile(MNT + "/ctl", "wakebin /bin/echo wake device argv") > 0, + "configure argv-reporting device wake helper"); + wake := readfile(MNT + "/wake"); + t.assert(hassubstr(wake, "--stdin --word hey lucia --threshold 0.7 --rate 16000"), + "device wake helper receives stdin, phrase, threshold, and rate"); + + # Restore defaults for the remaining tests. + writefile(MNT + "/ctl", "micmode helper"); + writefile(MNT + "/ctl", "capturedev default"); +} + +testWakeRestarts(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "wakebin /bin/echo wake fake-model 0.91") > 0, + "configure fake wake helper"); + first := readfile(MNT + "/wake"); + t.assert(hassubstr(first, "wake fake-model 0.91"), "first wake event delivered"); + t.assert(hassubstr(first, "--word hey lucia --threshold 0.7"), + "multiword wake phrase and threshold reach helper argv"); + second := readfile(MNT + "/wake"); + t.assert(hassubstr(second, "wake fake-model 0.91"), + "helper restarted for second wake event"); +} + +testListenRecords(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "whisperstreambin /bin/echo final shim transcript") > 0, + "configure fake listen helper"); + listen := readfile(MNT + "/listen"); + t.assert(hassubstr(listen, "final shim transcript"), "listen record delivered"); +} + +# `mic off` (written by voicemode on voice-mode exit) must kill the running +# mic-side helper and complete a pending read with an error instead of +# restarting it — the microphone is only open during a voice session. The +# next read re-arms it without any further ctl write. +testMicOffReleasesHelpers(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", + "whisperstreambin /bin/sh -c \"echo partial armed; sleep 30\"") > 0, + "configure blocking fake listen helper"); + first := readfile(MNT + "/listen"); + t.assert(hassubstr(first, "partial armed"), "helper armed by first read"); + + pendch := chan of string; + spawn readproc(MNT + "/listen", pendch); + sys->sleep(300); # let the read block in the helper + + t0 := sys->millisec(); + t.assert(writefile(MNT + "/ctl", "mic off") > 0, "mic off accepted"); + tmo := chan[1] of int; + spawn timer(tmo, 4000); + got := ""; + alt { + got = <-pendch => + ; + <-tmo => + ; + } + t1 := sys->millisec(); + t.assert(hassubstr(got, "error: mic off"), + "pending listen read completes instead of restarting the helper"); + t.assert(t1 - t0 < 3000, "mic off killed the helper promptly (no 30s run-out)"); + t.assert(hassubstr(readfile(MNT + "/ctl"), "mic off"), "ctl reports mic off"); + + t.assert(writefile(MNT + "/ctl", "whisperstreambin /bin/echo final rearmed") > 0, + "configure fake listen helper for re-arm"); + listen := readfile(MNT + "/listen"); + t.assert(hassubstr(listen, "final rearmed"), "next listen read re-arms the mic"); + t.assert(hassubstr(readfile(MNT + "/ctl"), "mic on"), "ctl reports mic on after re-arm"); +} + +# `listen off` (written by voicemode at the end of each voice turn) must stop +# only the STT helper: a pending listen read completes with an error instead +# of restarting it, wake reads keep working, and the next listen read re-arms +# STT without any further ctl write. This is what keeps between-turn speech +# (ambient talk, the assistant's own TTS) from queuing as stale records that +# replay into the next turn. +testListenOffStopsListenHelper(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", + "whisperstreambin /bin/sh -c \"echo partial turn; sleep 30\"") > 0, + "configure blocking fake listen helper"); + first := readfile(MNT + "/listen"); + t.assert(hassubstr(first, "partial turn"), "helper armed by first read"); + + pendch := chan of string; + spawn readproc(MNT + "/listen", pendch); + sys->sleep(300); # let the read block in the helper + + t0 := sys->millisec(); + t.assert(writefile(MNT + "/ctl", "listen off") > 0, "listen off accepted"); + tmo := chan[1] of int; + spawn timer(tmo, 4000); + got := ""; + alt { + got = <-pendch => + ; + <-tmo => + ; + } + t1 := sys->millisec(); + t.assert(hassubstr(got, "error: listen off"), + "pending listen read completes instead of restarting the helper"); + t.assert(t1 - t0 < 3000, "listen off killed the helper promptly (no 30s run-out)"); + t.assert(hassubstr(readfile(MNT + "/ctl"), "listen off"), "ctl reports listen off"); + + t.assert(writefile(MNT + "/ctl", "wakebin /bin/echo wake still-armed 0.9") > 0, + "configure fake wake helper"); + wake := readfile(MNT + "/wake"); + t.assert(hassubstr(wake, "wake still-armed"), "wake read unaffected by listen off"); + + t.assert(writefile(MNT + "/ctl", "whisperstreambin /bin/echo final listen rearmed") > 0, + "configure fake listen helper for re-arm"); + listen := readfile(MNT + "/listen"); + t.assert(hassubstr(listen, "final listen rearmed"), "next listen read re-arms STT"); + t.assert(hassubstr(readfile(MNT + "/ctl"), "listen on"), "ctl reports listen on after re-arm"); +} + +# A helper that cannot start (not installed, not on PATH) exits immediately +# with its reason on stderr. That reason must reach the client: a bare +# "wake helper exited" gives the user nothing to act on, which is exactly how +# a misconfigured install came to look like "the button does nothing". +testHelperErrorNamesCause(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", + "wakebin infernode-no-such-helper-xyz") > 0, + "configure a wake helper that does not exist"); + err := readfile(MNT + "/wake"); + t.assert(hassubstr(err, "error:"), "missing wake helper reports an error"); + t.assert(hassubstr(err, "not found"), + "error carries the helper's stderr, not just 'helper exited'"); +} + +# Cancel must kill the helper process: with a fake synthesizer that would +# block for 8 seconds, the pending say read has to complete within a couple +# of seconds of the cancel write. +testCancelKillsSay(t: ref T) +{ + t.assert(writefile(MNT + "/ctl", "kokorobin /bin/sh -c \"sleep 8\"") > 0, + "configure blocking fake synthesizer"); + + sayfd := sys->open(MNT + "/say", Sys->ORDWR); + t.assert(sayfd != nil, "say opens"); + if(sayfd == nil) + return; + b := array of byte "hello"; + t.assert(sys->write(sayfd, b, len b) > 0, "say write accepted"); + sys->sleep(500); # let the helper start + + t0 := sys->millisec(); + t.assert(writefile(MNT + "/cancel", "cancel") > 0, + "cancel write served while synthesizing"); + sys->seek(sayfd, big 0, Sys->SEEKSTART); + buf := array[512] of byte; + n := sys->read(sayfd, buf, len buf); + t1 := sys->millisec(); + t.assert(n >= 0, "say status readable after cancel"); + t.assert(t1 - t0 < 4000, "cancel killed the helper (no 8s run-out)"); +} + +testHalfDuplexSwallowsWakeDuringSay(t: ref T) +{ + fd := sys->create(PCMFILE, Sys->OWRITE, 8r644); + t.assert(fd != nil, "create fake audio device"); + if(fd == nil) + return; + fd = nil; + + t.assert(writefile(MNT + "/ctl", "audiodev " + PCMFILE) > 0, "audiodev scratch accepted"); + t.assert(writefile(MNT + "/ctl", "duplex half") > 0, "duplex half accepted"); + t.assert(writefile(MNT + "/ctl", + "kokorobin /bin/sh -c \"printf 0123456789; sleep 2; printf abcdef\"") > 0, + "configure slow fake synthesizer"); + t.assert(writefile(MNT + "/ctl", + "wakebin /bin/sh -c \"rm -f " + WAKEPID + "; echo wake cleanup\"") > 0, + "configure suppressed-helper marker cleanup"); + readfile(MNT + "/wake"); + t.assert(writefile(MNT + "/ctl", + "wakebin /bin/sh -c \"if [ ! -e " + WAKEPID + " ]; then echo $$ > " + + WAKEPID + "; fi; echo wake fake-model 0.92; sleep 30\"") > 0, + "configure long-lived fake wake helper"); + + sayfd := sys->open(MNT + "/say", Sys->ORDWR); + t.assert(sayfd != nil, "say opens"); + if(sayfd == nil) + return; + b := array of byte "hello"; + t.assert(sys->write(sayfd, b, len b) > 0, "say write accepted"); + sys->sleep(300); # let dosay enter its playback loop + + wakech := chan of string; + spawn readproc(MNT + "/wake", wakech); + tmo := chan[1] of int; + spawn timer(tmo, 900); + early := ""; + alt { + early = <-wakech => + ; + <-tmo => + ; + } + t.assert(early == "", "wake read suppressed during half-duplex playback"); + + tmo2 := chan[1] of int; + spawn timer(tmo2, 5000); + got := ""; + alt { + got = <-wakech => + ; + <-tmo2 => + ; + } + t.assert(hassubstr(got, "wake fake-model 0.92"), + "wake read completes after playback"); + + # Replacing wakebin kills the active post-playback helper. Probe the PID + # retained by the first, suppressed helper: it must already be gone. + t.assert(writefile(MNT + "/ctl", + "wakebin /bin/sh -c \"if kill -0 $(cat " + WAKEPID + + ") 2>/dev/null; then echo wake leaked; else echo wake cleaned; fi; rm -f " + + WAKEPID + "\"") > 0, "configure suppressed-helper probe"); + probe := readfile(MNT + "/wake"); + t.assert(hassubstr(probe, "wake cleaned"), + "suppressed wake helper terminated before restart"); + + sys->seek(sayfd, big 0, Sys->SEEKSTART); + buf := array[512] of byte; + sys->read(sayfd, buf, len buf); + writefile(MNT + "/ctl", "duplex full"); + writefile(MNT + "/ctl", "audiodev /dev/audio"); +} + +teardown() +{ + sys->unmount(nil, MNT); +} + +init(nil: ref Draw->Context, args: list of string) +{ + sys = load Sys Sys->PATH; + testing = load Testing Testing->PATH; + if(testing == nil) + raise "fail:load testing"; + testing->init(); + for(a := args; a != nil; a = tl a) + if(hd a == "-v") + testing->verbose(1); + + startserver(); + run("Files", testFiles); + run("Config", testConfig); + run("AudioRouting", testAudioRouting); + run("DuplexConfig", testDuplexConfig); + run("ChimeAccepted", testChimeAccepted); + run("WakeRestarts", testWakeRestarts); + run("ListenRecords", testListenRecords); + run("MicOffReleasesHelpers", testMicOffReleasesHelpers); + run("ListenOffStopsListenHelper", testListenOffStopsListenHelper); + run("DeviceCapture", testDeviceCapture); + run("CancelKillsSay", testCancelKillsSay); + run("HalfDuplexSwallowsWakeDuringSay", testHalfDuplexSwallowsWakeDuringSay); + run("HelperErrorNamesCause", testHelperErrorNamesCause); + + teardown(); + if(testing->summary(passed, failed, skipped) > 0) + raise "fail:tests failed"; +} diff --git a/tests/speechtest_test.b b/tests/speechtest_test.b new file mode 100644 index 000000000..cfe2a7a22 --- /dev/null +++ b/tests/speechtest_test.b @@ -0,0 +1,248 @@ +implement SpeechtestTest; + +# +# Tests for appl/cmd/speechtest.b — the LLM-free STT/TTS test loop. +# +# The speech tree is mocked with plain files (same technique as +# voicemode_test): a plain-file "listen" returns its content on every +# read, "say" is a writable scratch file, and there is no chime file +# (speechtest's chime writes are best-effort). speechtest is run +# without -b, so the mock tree is never mistaken for a missing stack. +# + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "testing.m"; + testing: Testing; + T: import testing; + +SpeechtestTest: module +{ + init: fn(nil: ref Draw->Context, args: list of string); + _marker: fn(); # prevents joiniface() type conflation with SpeechtestCmd +}; + +SpeechtestCmd: module +{ + init: fn(nil: ref Draw->Context, args: list of string); +}; + +_marker() {} + +SRCFILE: con "/tests/speechtest_test.b"; +STPATH: con "/dis/speechtest.dis"; +MOCK: con "/tmp/speechtest_test_speech"; +PHRASE: con "canned reply"; + +passed := 0; +failed := 0; +skipped := 0; + +testpid := -1; + +run(name: string, testfn: ref fn(t: ref T)) +{ + t := testing->newTsrc(name, SRCFILE); + { + testfn(t); + } exception { + "fail:fatal" => + ; + "fail:skip" => + ; + "*" => + t.failed = 1; + } + if(testing->done(t)) + passed++; + else if(t.skipped) + skipped++; + else + failed++; +} + +strip(s: string): string +{ + if(s == nil) + return nil; + i := 0; + while(i < len s && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n')) + i++; + j := len s; + while(j > i && (s[j-1] == ' ' || s[j-1] == '\t' || s[j-1] == '\r' || s[j-1] == '\n')) + j--; + if(i >= j) + return ""; + return s[i:j]; +} + +# Truncating write — mock state changes must not leave residue from +# longer previous contents. +createfile(path, data: string): int +{ + fd := sys->create(path, Sys->OWRITE, 8r644); + if(fd == nil) + return -1; + b := array of byte data; + if(len b == 0) + return 0; + return sys->write(fd, b, len b); +} + +readfile(path: string): string +{ + fd := sys->open(path, Sys->OREAD); + if(fd == nil) + return nil; + buf := array[8192] of byte; + n := sys->read(fd, buf, len buf); + if(n < 0) + return nil; + return string buf[0:n]; +} + +hassubstr(s, sub: string): int +{ + if(s == nil || sub == nil || len sub > len s) + return 0; + for(i := 0; i <= len s - len sub; i++) + if(s[i:i+len sub] == sub) + return 1; + return 0; +} + +waitfor(path, sub: string, timeout: int): int +{ + for(waited := 0; waited < timeout; waited += 100) { + if(hassubstr(readfile(path), sub)) + return 1; + sys->sleep(100); + } + return hassubstr(readfile(path), sub); +} + +mkmock(listenrec: string) +{ + sys->create("/tmp", Sys->OREAD, Sys->DMDIR | 8r777); + sys->create(MOCK, Sys->OREAD, Sys->DMDIR | 8r755); + createfile(MOCK + "/ctl", ""); + createfile(MOCK + "/listen", listenrec); + createfile(MOCK + "/say", ""); +} + +runcmd(pidch: chan of int, extra: list of string) +{ + pidch <-= sys->pctl(Sys->NEWPGRP, nil); + st := load SpeechtestCmd STPATH; + if(st == nil) { + sys->fprint(sys->fildes(2), "cannot load speechtest: %r\n"); + return; + } + st->init(nil, "speechtest" :: "-s" :: MOCK :: "-n" :: "1" :: + "-p" :: PHRASE :: extra); +} + +startcmd(listenrec: string, extra: list of string) +{ + mkmock(listenrec); + pidch := chan of int; + spawn runcmd(pidch, extra); + testpid = <-pidch; +} + +stopcmd() +{ + if(testpid < 0) + return; + fd := sys->open("/prog/" + string testpid + "/ctl", Sys->OWRITE); + if(fd != nil) { + b := array of byte "killgrp"; + sys->write(fd, b, len b); + } + testpid = -1; +} + +runst(name: string, listenrec: string, extra: list of string, testfn: ref fn(t: ref T)) +{ + startcmd(listenrec, extra); + run(name, testfn); + stopcmd(); + sys->sleep(200); +} + +testFinalSpeaksPhrase(t: ref T) +{ + t.assert(waitfor(MOCK + "/say", PHRASE, 5000), + "final transcript should trigger the canned phrase in say"); +} + +testEchoFinal(t: ref T) +{ + t.assert(waitfor(MOCK + "/say", "hello world", 5000), + "-e should speak the transcript itself"); +} + +testPartialDoesNotSpeak(t: ref T) +{ + sys->sleep(600); + t.assertseq(strip(readfile(MOCK + "/say")), "", + "a partial alone must not trigger say"); + createfile(MOCK + "/listen", "final all done\n"); + t.assert(waitfor(MOCK + "/say", PHRASE, 5000), + "the final after partials should trigger say"); +} + +testJunkFinalNotSpoken(t: ref T) +{ + sys->sleep(600); + t.assertseq(strip(readfile(MOCK + "/say")), "", + "a junk final ([BLANK_AUDIO]) must not trigger say"); + createfile(MOCK + "/listen", "final real words\n"); + t.assert(waitfor(MOCK + "/say", PHRASE, 5000), + "a real final after junk should trigger say"); +} + +testErrorDoesNotSpeak(t: ref T) +{ + sys->sleep(600); + t.assertseq(strip(readfile(MOCK + "/say")), "", + "an error record must not trigger say"); +} + +testCtlFileApplied(t: ref T) +{ + t.assert(waitfor(MOCK + "/ctl", "engine kokoro", 5000), + "-C applies the installer-selected speech ctl file before listening"); + t.assert(waitfor(MOCK + "/say", PHRASE, 5000), + "speech test continues after applying the ctl file"); +} + +init(nil: ref Draw->Context, args: list of string) +{ + sys = load Sys Sys->PATH; + testing = load Testing Testing->PATH; + if(testing == nil) { + sys->fprint(sys->fildes(2), "cannot load testing module: %r\n"); + raise "fail:cannot load testing"; + } + testing->init(); + for(a := args; a != nil; a = tl a) + if(hd a == "-v") + testing->verbose(1); + + runst("FinalSpeaksPhrase", "final hello world\n", nil, testFinalSpeaksPhrase); + runst("EchoFinal", "final hello world\n", "-e" :: nil, testEchoFinal); + runst("PartialDoesNotSpeak", "partial hel\n", nil, testPartialDoesNotSpeak); + runst("JunkFinalNotSpoken", "final [BLANK_AUDIO]\n", nil, testJunkFinalNotSpoken); + runst("ErrorDoesNotSpeak", "error: helper missing\n", nil, testErrorDoesNotSpeak); + ctlfile := "/tmp/speechtest_test.ctl.sh"; + createfile(ctlfile, "echo 'engine kokoro' > " + MOCK + "/ctl\n"); + runst("CtlFileApplied", "final configured helper\n", + "-C" :: ctlfile :: nil, testCtlFileApplied); + + if(testing->summary(passed, failed, skipped) > 0) + raise "fail:tests failed"; +} diff --git a/tests/voice_scripts_test.b b/tests/voice_scripts_test.b index 7d8b75af6..39d7b2f19 100644 --- a/tests/voice_scripts_test.b +++ b/tests/voice_scripts_test.b @@ -18,6 +18,8 @@ implement VoiceScriptsTest; # - lib/voice/dial exists, binds devaudio, writes buffer-cap # verbs, calls mount. # - lib/voice/test-tone exists (single-shell loopback recipe). +# - speech-terminal / speech-engine / speech-capture automate the documented +# remote speech namespace topologies without embedding host policy. # include "sys.m"; @@ -131,6 +133,75 @@ testTestToneShape(t: ref T) "voice/test-tone targets the loopback"); } +testSpeechTerminalShape(t: ref T) +{ + s := script_contents(t, "/lib/voice/speech-terminal"); + t.assert(contains(s, "sh /lib/voice/listen"), + "speech-terminal exports local audio through voice/listen"); + t.assert(contains(s, "mount -A $engine $provider"), + "speech-terminal mounts the remote provider"); + t.assert(contains(s, "echo 'provider '$provider > /n/speech/ctl"), + "speech-terminal selects the mounted provider"); + t.assert(contains(s, "echo 'duplex half' > /n/speech/ctl"), + "speech-terminal preserves the half-duplex default"); +} + +testSpeechEngineShape(t: ref T) +{ + s := script_contents(t, "/lib/voice/speech-engine"); + t.assert(contains(s, "mount -A $terminal $termmnt"), + "speech-engine imports terminal audio"); + t.assert(contains(s, "speechshim9p -m $provider"), + "speech-engine starts a provider at an isolated mount"); + t.assert(contains(s, "echo 'audiodev '$termmnt'/audio' > $provider/ctl"), + "speech-engine routes playback and default capture through imported audio"); + t.assert(contains(s, "echo 'micmode device' > $provider/ctl"), + "speech-engine enables namespace-backed PCM capture"); + t.assert(contains(s, "export $provider"), + "speech-engine exports the provider contract"); +} + +testSpeechCaptureShape(t: ref T) +{ + s := script_contents(t, "/lib/voice/speech-capture"); + t.assert(contains(s, "mount -A $capture $capturemnt"), + "speech-capture imports a remote device tree"); + t.assert(contains(s, "echo 'capturedev '$capturemnt'/audio' > /n/speech/ctl"), + "speech-capture changes capture without changing playback"); + t.assert(contains(s, "echo 'micmode device' > /n/speech/ctl"), + "speech-capture enables device-fed helpers"); +} + +testSpeechTestUsesInstalledCtl(t: ref T) +{ + launcher := script_contents(t, "/tools/speech-test.sh"); + t.assert(contains(launcher, "speech.ctl.sh"), + "headless speech test discovers the installer-selected ctl file"); + t.assert(contains(launcher, "-C"), + "headless speech test passes the selected ctl file to speechtest"); + + boot := script_contents(t, "/lib/lucifer/boot.sh"); + t.assert(contains(boot, "$speechhelperbin^/../speech.ctl.sh"), + "GUI speech test prefers the ctl file adjacent to its helper bin"); +} + +testVoiceDraftPresentation(t: ref T) +{ + conv := script_contents(t, "/appl/cmd/luciconv.b"); + t.assert(contains(conv, "draft-status"), + "conversation reads the voice draft status"); + t.assert(contains(conv, "voice-draft"), + "voice hypotheses render as a conversation turn"); + t.assert(contains(conv, "not sent"), + "the pending voice turn is explicitly marked unsent"); + t.assert(contains(conv, "voiceactive() && k != 0"), + "keyboard compose edits are locked while voice owns the turn"); + + boot := script_contents(t, "/appl/cmd/lucifer.b"); + t.assert(contains(boot, "convEvCh <-= ev"), + "global input-mode changes reach the conversation UI"); +} + init(nil: ref Draw->Context, args: list of string) { sys = load Sys Sys->PATH; @@ -149,6 +220,11 @@ init(nil: ref Draw->Context, args: list of string) run("ListenShape", testListenShape); run("DialShape", testDialShape); run("TestToneShape", testTestToneShape); + run("SpeechTerminalShape", testSpeechTerminalShape); + run("SpeechEngineShape", testSpeechEngineShape); + run("SpeechCaptureShape", testSpeechCaptureShape); + run("SpeechTestUsesInstalledCtl", testSpeechTestUsesInstalledCtl); + run("VoiceDraftPresentation", testVoiceDraftPresentation); if(testing->summary(passed, failed, skipped) > 0) raise "fail:tests failed"; diff --git a/tests/voicemode_test.b b/tests/voicemode_test.b new file mode 100644 index 000000000..aa2a3c8e5 --- /dev/null +++ b/tests/voicemode_test.b @@ -0,0 +1,672 @@ +implement VoicemodeTest; + +# +# voicemode daemon state machine (Phase 1.4/1.6), driven against a mock file +# tree instead of live luciuisrv/speech9p. Plain files give always-ready +# reads; the daemon's poll fallback (no /event file in the mock ui) and its +# pacing sleeps make that workable. Covers: idle-until-voice-mode, partial +# records not injected, final transcript injection through conversation/ +# voiceinput, spoken "keyboard" control intent, idle return on +# input-mode "k" with a "mic off" ctl write releasing the microphone, and +# LLM-free test mode (-p/-e: finals bypass voiceinput and answer with a +# canned say instead). +# + +include "sys.m"; + sys: Sys; + +include "draw.m"; + +include "testing.m"; + testing: Testing; + T: import testing; + +VoicemodeDaemon: module { + init: fn(nil: ref Draw->Context, args: list of string); +}; + +VoicemodeTest: module { + init: fn(nil: ref Draw->Context, args: list of string); + _marker: fn(); # prevents joiniface() type conflation with VoicemodeDaemon +}; + +SRCFILE: con "/tests/voicemode_test.b"; +VMPATH: con "/dis/voicemode.dis"; +MOCKUI: con "/tmp/voicemode_test_ui"; +MOCKSPEECH: con "/tmp/voicemode_test_speech"; + +passed := 0; +failed := 0; +skipped := 0; + +daemonpid := -1; +daemonargs: list of string; + +_marker() {} + +run(name: string, testfn: ref fn(t: ref T)) +{ + t := testing->newTsrc(name, SRCFILE); + { + testfn(t); + } exception { + "fail:fatal" => + ; + "fail:skip" => + ; + "*" => + t.failed = 1; + } + if(testing->done(t)) + passed++; + else if(t.skipped) + skipped++; + else + failed++; +} + +runvm(name: string, extra: list of string, testfn: ref fn(t: ref T)) +{ + startdaemon(extra); + run(name, testfn); + stopdaemon(); + sys->sleep(200); +} + +strip(s: string): string +{ + if(s == nil) + return nil; + i := 0; + while(i < len s && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n')) + i++; + j := len s; + while(j > i && (s[j-1] == ' ' || s[j-1] == '\t' || s[j-1] == '\r' || s[j-1] == '\n')) + j--; + if(i >= j) + return ""; + return s[i:j]; +} + +# Truncating write — mock state changes must not leave residue from longer +# previous contents (a stale tail would corrupt the next parsed record). +createfile(path, data: string): int +{ + fd := sys->create(path, Sys->OWRITE, 8r644); + if(fd == nil) + return -1; + b := array of byte data; + if(len b == 0) + return 0; + return sys->write(fd, b, len b); +} + +readfile(path: string): string +{ + fd := sys->open(path, Sys->OREAD); + if(fd == nil) + return nil; + buf := array[8192] of byte; + n := sys->read(fd, buf, len buf); + if(n < 0) + return nil; + return string buf[0:n]; +} + +hassubstr(s, sub: string): int +{ + if(s == nil || sub == nil || len sub > len s) + return 0; + for(i := 0; i <= len s - len sub; i++) + if(s[i:i+len sub] == sub) + return 1; + return 0; +} + +countsubstr(s, sub: string): int +{ + if(s == nil || sub == nil || len sub == 0 || len sub > len s) + return 0; + n := 0; + for(i := 0; i <= len s - len sub; i++) + if(s[i:i+len sub] == sub) + n++; + return n; +} + +# Poll until path's content contains sub, or timeout (ms). Returns 1 on hit. +waitfor(path, sub: string, timeout: int): int +{ + for(waited := 0; waited < timeout; waited += 100) { + if(hassubstr(readfile(path), sub)) + return 1; + sys->sleep(100); + } + return hassubstr(readfile(path), sub); +} + +waitnotfor(path, sub: string, timeout: int): int +{ + for(waited := 0; waited < timeout; waited += 100) { + if(hassubstr(readfile(path), sub)) + return 0; + sys->sleep(100); + } + return !hassubstr(readfile(path), sub); +} + +mkmock() +{ + sys->create("/tmp", Sys->OREAD, Sys->DMDIR | 8r777); + sys->create(MOCKUI, Sys->OREAD, Sys->DMDIR | 8r755); + sys->create(MOCKUI + "/activity", Sys->OREAD, Sys->DMDIR | 8r755); + sys->create(MOCKUI + "/activity/0", Sys->OREAD, Sys->DMDIR | 8r755); + sys->create(MOCKUI + "/activity/0/context", Sys->OREAD, Sys->DMDIR | 8r755); + sys->create(MOCKUI + "/activity/0/conversation", Sys->OREAD, Sys->DMDIR | 8r755); + createfile(MOCKUI + "/input-mode", "k"); + createfile(MOCKUI + "/activity/current", "0"); + createfile(MOCKUI + "/activity/0/context/ctl", ""); + createfile(MOCKUI + "/activity/0/status", "working"); + createfile(MOCKUI + "/activity/0/conversation/ctl", ""); + createfile(MOCKUI + "/activity/0/conversation/voiceinput", ""); + createfile(MOCKUI + "/activity/0/conversation/control", ""); + createfile(MOCKUI + "/activity/0/conversation/draft", ""); + createfile(MOCKUI + "/activity/0/conversation/draft-status", ""); + createfile(MOCKSPEECH + "/wake", "wake hey_lucia 0.9\n"); + createfile(MOCKSPEECH + "/listen", "partial warming up\n"); + createfile(MOCKSPEECH + "/cancel", ""); + createfile(MOCKSPEECH + "/say", ""); + createfile(MOCKSPEECH + "/ctl", ""); +} + +rundaemon(pidch: chan of int) +{ + pidch <-= sys->pctl(Sys->NEWPGRP, nil); + vm := load VoicemodeDaemon VMPATH; + if(vm == nil) { + sys->fprint(sys->fildes(2), "cannot load voicemode: %r\n"); + return; + } + # -g 0 keeps the legacy immediate-submit semantics for the pre-grace + # tests; a later -g in daemonargs overrides it (Arg keeps the last). + # The mock file tree re-delivers the same final on every read, which + # in grace mode reads as "more speech" and restarts the window + # forever — grace tests manage the listen file explicitly instead. + vm->init(nil, "voicemode" :: "-u" :: MOCKUI :: "-s" :: MOCKSPEECH :: + "-g" :: "0" :: daemonargs); +} + +startdaemon(extra: list of string) +{ + daemonargs = extra; + sys->create(MOCKSPEECH, Sys->OREAD, Sys->DMDIR | 8r755); + mkmock(); + pidch := chan of int; + spawn rundaemon(pidch); + daemonpid = <-pidch; + sys->sleep(300); +} + +stopdaemon() +{ + if(daemonpid < 0) + return; + fd := sys->open("/prog/" + string daemonpid + "/ctl", Sys->OWRITE); + if(fd != nil) { + b := array of byte "killgrp"; + sys->write(fd, b, len b); + } + daemonpid = -1; +} + +testIdleUntilVoiceMode(t: ref T) +{ + # input-mode is "k": the pre-spawned daemon must not touch the speech + # or ui files. + sys->sleep(700); + ctx := readfile(MOCKUI + "/activity/0/context/ctl"); + t.assert(!hassubstr(ctx, "via=voice-mode"), "daemon idle while input-mode is k"); + vi := readfile(MOCKUI + "/activity/0/conversation/voiceinput"); + t.assert(!hassubstr(vi, "warming"), "no injection while idle"); +} + +testActivatesOnVoiceMode(t: ref T) +{ + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "via=voice-mode", 5000), + "daemon activates and reports voice status after input-mode v"); + # Wake fires (mock is always ready) which must cut any active TTS. + t.assert(waitfor(MOCKSPEECH + "/cancel", "cancel", 3000), + "wake event writes speech cancel (barge-in path)"); +} + +testPartialNotInjected(t: ref T) +{ + # listen still returns a partial record; the daemon must keep waiting + # rather than submitting the hypothesis as a user turn. + sys->sleep(1000); + vi := readfile(MOCKUI + "/activity/0/conversation/voiceinput"); + t.assert(!hassubstr(vi, "warming"), "partial record is not injected"); +} + +testPartialThenFinalInjected(t: ref T) +{ + createfile(MOCKSPEECH + "/listen", "partial confidence=0.7000 half\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=listening", 5000), + "partial record keeps daemon in listening state"); + sys->sleep(300); + createfile(MOCKSPEECH + "/listen", + "partial confidence=0.7000 half\nfinal confidence=0.9500 full transcript\n"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/voiceinput", + "full transcript", 5000), + "final after partial is injected"); +} + +testPartialUpdatesPendingDraft(t: ref T) +{ + createfile(MOCKSPEECH + "/listen", "partial half a thou\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", + "label=Voice type=audio status=listening", 5000), + "voice resource reports listening without duplicating transcript text"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/draft", + "half a thou", 3000), + "partial transcript replaces the visible conversation draft"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/draft-status", + "Listening", 3000), + "partial transcript has an explicit pending-bubble status"); + createfile(MOCKSPEECH + "/listen", "final full transcript\n"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", + "label=Voice: queued type=audio status=queued", 5000), + "busy final visibly reports the queued follow-up"); + t.assertseq(readfile(MOCKUI + "/activity/0/conversation/draft"), "", + "final transcript clears the draft before submission"); + t.assertseq(readfile(MOCKUI + "/activity/0/conversation/draft-status"), "", + "final transcript clears the pending-bubble status"); +} + +testFinalInjected(t: ref T) +{ + createfile(MOCKSPEECH + "/ctl", ""); + createfile(MOCKSPEECH + "/listen", "final hello from voice\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/voiceinput", + "hello from voice", 5000), + "final transcript injected into conversation/voiceinput"); + t.assert(waitfor(MOCKSPEECH + "/ctl", "listen off", 3000), + "completed turn stops the STT helper (listen off ctl write)"); +} + +testListenTimeoutReturnsToWaiting(t: ref T) +{ + createfile(MOCKSPEECH + "/ctl", ""); + createfile(MOCKSPEECH + "/listen", ""); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=listening", 5000), + "daemon enters listening with empty listen stream"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=waiting", 3000), + "listen timeout returns status to waiting"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "label=Voice: no speech heard", 3000), + "timeout is visibly distinct from a completed empty turn"); + t.assertseq(readfile(MOCKUI + "/activity/0/conversation/draft"), "", + "listen timeout clears any draft hypothesis"); + t.assert(waitfor(MOCKSPEECH + "/ctl", "listen off", 3000), + "timeout stops the STT helper (listen off ctl write)"); +} + +testWakeDebounce(t: ref T) +{ + createfile(MOCKSPEECH + "/listen", "final debounce turn\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/voiceinput", + "debounce turn", 5000), + "first wake reaches listen and injects final"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=queued", 3000), + "completed busy turn remains visibly queued until the agent accepts it"); + sys->sleep(300); + t.assert(!hassubstr(readfile(MOCKUI + "/activity/0/context/ctl"), "status=listening"), + "immediate second wake is debounced instead of starting listen"); +} + +testJunkFinalNotInjected(t: ref T) +{ + createfile(MOCKSPEECH + "/listen", "final [BLANK_AUDIO]\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitnotfor(MOCKUI + "/activity/0/conversation/voiceinput", + "BLANK_AUDIO", 1000), + "bracketed silence marker is not injected"); + createfile(MOCKSPEECH + "/listen", "final Thank you.\n"); + t.assert(waitnotfor(MOCKUI + "/activity/0/conversation/voiceinput", + "Thank you", 1200), + "whisper silence hallucination is not injected"); +} + +testControlIntentPunctuation(t: ref T) +{ + createfile(MOCKSPEECH + "/listen", "partial waiting\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=listening", 5000), + "daemon is listening before punctuated control intent"); + createfile(MOCKSPEECH + "/cancel", ""); + createfile(MOCKSPEECH + "/listen", "final Stop.\n"); + t.assert(waitfor(MOCKSPEECH + "/cancel", "cancel", 5000), + "punctuated stop intent cancels speech"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/control", "cancel", 3000), + "punctuated stop also cancels the active agent turn"); + vi := readfile(MOCKUI + "/activity/0/conversation/voiceinput"); + t.assert(!hassubstr(vi, "Stop"), "punctuated control intent is not injected"); +} + +testPauseResumeControls(t: ref T) +{ + createfile(MOCKSPEECH + "/listen", "final pause\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/control", "pause", 5000), + "spoken pause reaches active-turn control"); + createfile(MOCKUI + "/activity/0/status", "paused"); + createfile(MOCKSPEECH + "/listen", "final continue\n"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/control", "resume", 5000), + "spoken continue resumes only a paused activity"); +} + +testYesRequiresBlocked(t: ref T) +{ + createfile(MOCKSPEECH + "/listen", "final yes\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/voiceinput", "yes", 5000), + "bare yes is injected when no approval is blocked"); + createfile(MOCKUI + "/input-mode", "k"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=idle", 5000), + "daemon exits before blocked approval case"); + createfile(MOCKUI + "/activity/0/status", "blocked"); + createfile(MOCKUI + "/activity/0/conversation/voiceinput", ""); + createfile(MOCKUI + "/activity/0/context/ctl", ""); + createfile(MOCKSPEECH + "/listen", "final yes\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/voiceinput", "Allow", 5000), + "bare yes maps to Allow only while activity is blocked"); +} + +testNoDeniesBlockedApproval(t: ref T) +{ + createfile(MOCKUI + "/activity/0/status", "blocked"); + createfile(MOCKSPEECH + "/listen", "final no\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/voiceinput", "Deny", 5000), + "bare no maps to Deny while activity is blocked"); +} + +testStatusSpeaksActivityState(t: ref T) +{ + createfile(MOCKUI + "/activity/0/status", "working"); + createfile(MOCKSPEECH + "/listen", "final status\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKSPEECH + "/say", "Current activity is working.", 5000), + "spoken status reads the current activity state through TTS"); +} + +testLowConfidenceRequiresConfirmation(t: ref T) +{ + createfile(MOCKSPEECH + "/listen", + "final confidence=0.4200 remove the old report\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/ctl", + "title=Confirm speech", 5000), + "low-confidence final asks for visual confirmation"); + vi := readfile(MOCKUI + "/activity/0/conversation/voiceinput"); + t.assert(!hassubstr(vi, "remove the old report"), + "low-confidence final is not submitted before confirmation"); + createfile(MOCKSPEECH + "/listen", "final confidence=0.9900 yes\n"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/voiceinput", + "remove the old report", 5000), + "high-confidence yes submits the interpreted utterance"); +} + +testLowConfidenceNoRequestsRepeat(t: ref T) +{ + createfile(MOCKSPEECH + "/listen", + "final confidence=0.4200 delete the release branch\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/ctl", + "title=Confirm speech", 5000), + "low-confidence final asks for confirmation before a rejection"); + createfile(MOCKSPEECH + "/listen", "final confidence=0.9900 no\n"); + t.assert(waitfor(MOCKSPEECH + "/say", "Okay. Please say it again.", 5000), + "spoken no discards the interpretation and asks for a repeat"); + vi := readfile(MOCKUI + "/activity/0/conversation/voiceinput"); + t.assert(!hassubstr(vi, "delete the release branch"), + "rejected low-confidence interpretation is never submitted"); +} + +testHelperErrorSurfacedOnce(t: ref T) +{ + createfile(MOCKSPEECH + "/wake", "error: wake helper missing\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/ctl", "title=Voice", 6000), + "third consecutive helper error posts a voice notice"); + msg := readfile(MOCKUI + "/activity/0/conversation/ctl"); + t.assert(countsubstr(msg, "title=Voice") == 1, + "voice helper notice is written once in the mock conversation ctl"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=error", 1000), + "helper error surfaces in context status"); +} + +testSpokenKeyboardIntent(t: ref T) +{ + # "keyboard" is a control intent: it returns input to keyboard mode + # instead of becoming a chat turn. + createfile(MOCKSPEECH + "/listen", "final keyboard\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/input-mode", "k", 5000), + "spoken keyboard intent flips input-mode back to k"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=idle", 5000), + "voice status returns to idle after exit"); + vi := readfile(MOCKUI + "/activity/0/conversation/voiceinput"); + t.assert(!hassubstr(vi, "keyboard"), "control intent was not injected as a turn"); +} + +testTestModeSpeaksPhrase(t: ref T) +{ + # -p puts the daemon in LLM-free test mode: the final transcript is + # shown as a "Heard" dialogue line and answered by saying the canned + # phrase; conversation/voiceinput (the LLM path) is never written. + createfile(MOCKSPEECH + "/listen", "final hello there\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKSPEECH + "/say", "canned reply", 5000), + "test-mode final answers with the canned phrase in say"); + t.assert(hassubstr(readfile(MOCKUI + "/activity/0/conversation/ctl"), + "title=Heard text=hello there"), + "test-mode final posts the transcript as a Heard dialogue line"); + vi := readfile(MOCKUI + "/activity/0/conversation/voiceinput"); + t.assert(!hassubstr(vi, "hello there"), + "test-mode final is not injected as an LLM turn"); +} + +testTestModeEchoesTranscript(t: ref T) +{ + createfile(MOCKSPEECH + "/listen", "final echo me back\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKSPEECH + "/say", "echo me back", 5000), + "-e answers with the transcript itself"); + vi := readfile(MOCKUI + "/activity/0/conversation/voiceinput"); + t.assert(!hassubstr(vi, "echo me back"), + "-e final is not injected as an LLM turn"); +} + +testTestModeControlIntentStillWorks(t: ref T) +{ + # Control intents must keep acting on the session in test mode + # instead of being spoken back. + createfile(MOCKSPEECH + "/listen", "final keyboard\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/input-mode", "k", 5000), + "spoken keyboard intent exits voice mode in test mode"); + say := readfile(MOCKSPEECH + "/say"); + t.assert(!hassubstr(say, "canned reply"), + "control intent does not trigger the canned phrase"); +} + +testReentryAndInputModeExit(t: ref T) +{ + # Re-enter voice mode, then exit via an external input-mode write (the + # path Esc in lucifer and "/voice mode off" in lucibridge use). + createfile(MOCKSPEECH + "/listen", "partial nothing yet\n"); + createfile(MOCKUI + "/activity/0/context/ctl", ""); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "via=voice-mode", 5000), + "daemon re-activates on second input-mode v"); + createfile(MOCKSPEECH + "/cancel", ""); + createfile(MOCKUI + "/input-mode", "k"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=idle", 5000), + "external input-mode k returns daemon to idle"); + t.assert(waitfor(MOCKSPEECH + "/cancel", "cancel", 3000), + "exit cancels any in-flight speech"); + t.assert(waitfor(MOCKSPEECH + "/ctl", "mic off", 3000), + "exit releases the microphone (mic off ctl write)"); +} + +# Grace window (-g): a good final is announced (chip status=sending, text in +# the compose draft) and only injected after the window elapses. The mock's +# re-delivery of the same final keeps restarting the window (it reads as +# appended speech), so each test rewrites the listen file to advance. +testGraceSubmitsAfterWindow(t: ref T) +{ + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=listening", 5000), + "listening after wake"); + createfile(MOCKSPEECH + "/listen", "final confidence=0.9500 graceful hello\n"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=sending", 5000), + "grace window announced on the chip"); + # Stop the re-delivery; the window now runs out undisturbed. + createfile(MOCKSPEECH + "/listen", ""); + vi := readfile(MOCKUI + "/activity/0/conversation/voiceinput"); + t.assert(!hassubstr(vi, "graceful hello"), "not injected inside the window"); + draft := readfile(MOCKUI + "/activity/0/conversation/draft"); + t.assert(hassubstr(draft, "graceful hello"), "pending transcript in the draft"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/draft-status", + "Sending in", 3000), "pending transcript has a visible countdown"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/voiceinput", + "graceful hello", 5000), "injected after the grace window"); +} + +testGraceCancelDiscards(t: ref T) +{ + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=listening", 5000), + "listening after wake"); + createfile(MOCKSPEECH + "/listen", "final confidence=0.9500 do not send this\n"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=sending", 5000), + "grace window announced on the chip"); + # The re-delivered final keeps the window open until this lands. + createfile(MOCKSPEECH + "/listen", "final confidence=0.9500 cancel\n"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=waiting", 5000), + "spoken cancel returns to waiting"); + createfile(MOCKSPEECH + "/listen", ""); + sys->sleep(900); # longer than the grace window + vi := readfile(MOCKUI + "/activity/0/conversation/voiceinput"); + t.assert(!hassubstr(vi, "do not send this"), + "cancelled transcript is never injected"); +} + +testGraceAppendMergesTurn(t: ref T) +{ + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=listening", 5000), + "listening after wake"); + createfile(MOCKSPEECH + "/listen", "final confidence=0.9500 first part\n"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=sending", 5000), + "grace window announced on the chip"); + createfile(MOCKSPEECH + "/listen", "final confidence=0.9500 second part\n"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/draft", "second part", 5000), + "appended speech lands in the draft"); + createfile(MOCKSPEECH + "/listen", ""); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/voiceinput", + "second part", 5000), "merged turn injected after the window"); + vi := readfile(MOCKUI + "/activity/0/conversation/voiceinput"); + t.assert(hassubstr(vi, "first part"), "merged turn keeps the first utterance"); +} + +testBusyFollowupQueueIsCapped(t: ref T) +{ + # The mock activity starts "working". One spoken follow-up may request + # refinement and enter voiceinput; further speech must not build an + # unbounded queue while that activity remains busy. + createfile(MOCKSPEECH + "/listen", "final first queued turn\n"); + createfile(MOCKUI + "/input-mode", "v"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/voiceinput", + "first queued turn", 5000), "first busy follow-up is queued"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", "status=queued", 3000), + "queued follow-up is visible on the voice resource"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/control", "refine", 3000), + "first busy follow-up requests refinement"); + + createfile(MOCKSPEECH + "/listen", ""); + createfile(MOCKUI + "/activity/0/conversation/voiceinput", ""); + createfile(MOCKSPEECH + "/listen", "final second queued turn\n"); + t.assert(waitfor(MOCKUI + "/activity/0/context/ctl", + "one turn queued", 5000), "additional busy follow-up reports the cap"); + t.assert(waitnotfor(MOCKUI + "/activity/0/conversation/voiceinput", + "second queued turn", 1200), "additional busy follow-up is discarded"); + + # Once the activity becomes idle, the latch clears and a later turn is + # accepted normally. + createfile(MOCKSPEECH + "/listen", ""); + createfile(MOCKUI + "/activity/0/status", "idle"); + createfile(MOCKSPEECH + "/listen", "final after idle\n"); + t.assert(waitfor(MOCKUI + "/activity/0/conversation/voiceinput", + "after idle", 5000), "idle activity clears the queued-turn cap"); +} + +init(nil: ref Draw->Context, args: list of string) +{ + sys = load Sys Sys->PATH; + testing = load Testing Testing->PATH; + if(testing == nil) + raise "fail:load testing"; + testing->init(); + for(a := args; a != nil; a = tl a) + if(hd a == "-v") + testing->verbose(1); + + runvm("IdleUntilVoiceMode", nil, testIdleUntilVoiceMode); + runvm("ActivatesOnVoiceMode", nil, testActivatesOnVoiceMode); + runvm("PartialNotInjected", nil, testPartialNotInjected); + runvm("PartialThenFinalInjected", nil, testPartialThenFinalInjected); + runvm("PartialUpdatesPendingDraft", nil, testPartialUpdatesPendingDraft); + runvm("FinalInjected", nil, testFinalInjected); + runvm("ListenTimeoutReturnsToWaiting", "-t" :: "500" :: "-w" :: "2000" :: nil, + testListenTimeoutReturnsToWaiting); + runvm("WakeDebounce", "-w" :: "1000" :: nil, testWakeDebounce); + runvm("JunkFinalNotInjected", "-w" :: "200" :: nil, testJunkFinalNotInjected); + runvm("ControlIntentPunctuation", nil, testControlIntentPunctuation); + runvm("PauseResumeControls", "-w" :: "200" :: nil, testPauseResumeControls); + runvm("YesRequiresBlocked", "-w" :: "200" :: nil, testYesRequiresBlocked); + runvm("NoDeniesBlockedApproval", "-w" :: "200" :: nil, + testNoDeniesBlockedApproval); + runvm("StatusSpeaksActivityState", "-w" :: "200" :: nil, + testStatusSpeaksActivityState); + runvm("LowConfidenceRequiresConfirmation", "-w" :: "200" :: nil, + testLowConfidenceRequiresConfirmation); + runvm("LowConfidenceNoRequestsRepeat", "-w" :: "200" :: nil, + testLowConfidenceNoRequestsRepeat); + runvm("HelperErrorSurfacedOnce", nil, testHelperErrorSurfacedOnce); + runvm("SpokenKeyboardIntent", nil, testSpokenKeyboardIntent); + runvm("TestModeSpeaksPhrase", "-p" :: "canned reply" :: nil, + testTestModeSpeaksPhrase); + runvm("TestModeEchoesTranscript", "-e" :: nil, testTestModeEchoesTranscript); + runvm("TestModeControlIntentStillWorks", "-p" :: "canned reply" :: nil, + testTestModeControlIntentStillWorks); + runvm("ReentryAndInputModeExit", nil, testReentryAndInputModeExit); + runvm("GraceSubmitsAfterWindow", "-g" :: "600" :: "-w" :: "200" :: nil, + testGraceSubmitsAfterWindow); + runvm("GraceCancelDiscards", "-g" :: "600" :: "-w" :: "200" :: nil, + testGraceCancelDiscards); + runvm("GraceAppendMergesTurn", "-g" :: "600" :: "-w" :: "200" :: nil, + testGraceAppendMergesTurn); + runvm("BusyFollowupQueueIsCapped", "-w" :: "200" :: nil, + testBusyFollowupQueueIsCapped); + + if(testing->summary(passed, failed, skipped) > 0) + raise "fail:tests failed"; +} diff --git a/tools/install-speech-helpers.sh b/tools/install-speech-helpers.sh new file mode 100755 index 000000000..633488fbb --- /dev/null +++ b/tools/install-speech-helpers.sh @@ -0,0 +1,706 @@ +#!/usr/bin/env bash +set -euo pipefail + +PREFIX=${INFERNODE_SPEECH_HOME:-"$HOME/.local/share/infernode-speech"} +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +VENV="$PREFIX/venv" +BIN="$PREFIX/bin" +LIBEXEC="$PREFIX/libexec" +MODELS="$PREFIX/models" +KOKORO_DIR="$MODELS/kokoro" +OPENWAKEWORD_DIR="$MODELS/openwakeword" +WHISPER_MODEL="$MODELS/ggml-base.en.bin" + +KOKORO_ONNX_VERSION=${KOKORO_ONNX_VERSION:-0.4.7} +OPENWAKEWORD_VERSION=${OPENWAKEWORD_VERSION:-0.6.0} +KOKORO_MODEL_URL=${KOKORO_MODEL_URL:-https://github.com/thewh1teagle/kokoro-onnx/releases/download/model-files-v1.0/kokoro-v1.0.onnx} +KOKORO_VOICES_URL=${KOKORO_VOICES_URL:-https://github.com/thewh1teagle/kokoro-onnx/releases/download/model-files-v1.0/voices-v1.0.bin} +WHISPER_MODEL_URL=${WHISPER_MODEL_URL:-https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin} +WHISPER_MODEL_FALLBACK_URL=${WHISPER_MODEL_FALLBACK_URL:-} +if [ -z "$WHISPER_MODEL_FALLBACK_URL" ]; then + case "$WHISPER_MODEL_URL" in + *\?*) WHISPER_MODEL_FALLBACK_URL="" ;; + *) WHISPER_MODEL_FALLBACK_URL="${WHISPER_MODEL_URL}?download=true" ;; + esac +fi +WHISPER_MODEL_MIN_BYTES=${WHISPER_MODEL_MIN_BYTES:-100000000} + +# Parakeet realtime STT (preferred over whisper when it can be built). +# PARAKEET_SRC may point at an existing parakeet.cpp checkout; otherwise the +# upstream repo is cloned under the install prefix. The streaming EOU model +# is not yet published as GGUF, so we also probe dev checkouts and accept an +# explicit PARAKEET_EOU_MODEL path (see find_parakeet_eou_model). +PARAKEET_DIR="$MODELS/parakeet" +PARAKEET_SRC=${PARAKEET_SRC:-"$PREFIX/src/parakeet.cpp"} +PARAKEET_REPO_URL=${PARAKEET_REPO_URL:-https://github.com/mudler/parakeet.cpp.git} +PARAKEET_EOU_MODEL=${PARAKEET_EOU_MODEL:-} +PARAKEET_EOU_URL=${PARAKEET_EOU_URL:-https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/parakeet_realtime_eou_120m-v1-q8_0.gguf} + +# Set by install_parakeet on success; selects the ctl configuration. +PARAKEET_OK=0 +PARAKEET_MODEL_PATH="" + +log() { + printf '%s\n' "$*" +} + +download_once() { + local url=$1 + local dest=$2 + local tmp="$dest.tmp" + + if [ -s "$dest" ]; then + log "exists: $dest" + return 0 + fi + mkdir -p "$(dirname "$dest")" + log "download: $url" + rm -f "$tmp" + if ! curl -L --fail --retry 3 --retry-all-errors --retry-delay 1 \ + --output "$tmp" "$url"; then + rm -f "$tmp" + return 1 + fi + if [ ! -s "$tmp" ]; then + log "warn: empty download from $url" + rm -f "$tmp" + return 1 + fi + mv "$tmp" "$dest" +} + +# Download a large model atomically from the first working URL. Existing +# complete files are never replaced; short HTML/error responses are rejected +# before they can become the configured model. +download_model() { + local dest=$1 + local minbytes=$2 + shift 2 + + if [ -s "$dest" ] && [ "$(wc -c <"$dest")" -ge "$minbytes" ]; then + log "exists: $dest" + return 0 + fi + + local url tmp="$dest.tmp" + mkdir -p "$(dirname "$dest")" + for url in "$@"; do + [ -n "$url" ] || continue + log "download: $url" + rm -f "$tmp" + if ! curl -L --fail --retry 3 --retry-all-errors --retry-delay 1 \ + --output "$tmp" "$url"; then + rm -f "$tmp" + continue + fi + if [ ! -s "$tmp" ] || [ "$(wc -c <"$tmp")" -lt "$minbytes" ]; then + log "warn: rejected short model download from $url" + rm -f "$tmp" + continue + fi + mv "$tmp" "$dest" + return 0 + done + rm -f "$tmp" + return 1 +} + +install_whisper_cpp() { + if ! command -v brew >/dev/null 2>&1; then + log "skip: Homebrew not found; install whisper-cpp separately if needed" + return 0 + fi + if brew list --formula whisper-cpp >/dev/null 2>&1; then + log "exists: Homebrew whisper-cpp" + return 0 + fi + log "install: brew install whisper-cpp" + brew install whisper-cpp +} + +install_python_deps() { + if [ ! -x "$VENV/bin/python" ]; then + log "create: $VENV" + python3 -m venv "$VENV" + fi + "$VENV/bin/python" -m pip install --upgrade pip + "$VENV/bin/python" -m pip install \ + "kokoro-onnx==$KOKORO_ONNX_VERSION" \ + "openwakeword==$OPENWAKEWORD_VERSION" \ + "soundfile>=0.13,<0.14" \ + "sounddevice>=0.5,<0.6" \ + "numpy>=2,<3" +} + +download_openwakeword_models() { + mkdir -p "$OPENWAKEWORD_DIR" + INFERNODE_OPENWAKEWORD_DIR="$OPENWAKEWORD_DIR" "$VENV/bin/python" <<'PY' +import os +import pathlib +import shutil + +target = pathlib.Path(os.environ["INFERNODE_OPENWAKEWORD_DIR"]) +target.mkdir(parents=True, exist_ok=True) + +try: + import openwakeword + import openwakeword.utils + + openwakeword.utils.download_models() + src = pathlib.Path(openwakeword.__file__).resolve().parent / "resources" / "models" + copied = [] + for path in src.glob("*jarvis*"): + if path.is_file(): + out = target / path.name + shutil.copy2(path, out) + copied.append(str(out)) + if copied: + print("copied: " + ", ".join(copied)) + else: + print("notice: openWakeWord downloaded models, but no jarvis model was found to copy") +except Exception as exc: + print("notice: openWakeWord model download skipped: %s" % exc) +PY +} + +write_wrappers() { + mkdir -p "$BIN" "$LIBEXEC" + + cat >"$BIN/kokoro-cli" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +PREFIX=${INFERNODE_SPEECH_HOME:-"$HOME/.local/share/infernode-speech"} +exec "$PREFIX/venv/bin/python" "$PREFIX/libexec/kokoro_cli.py" "$@" +SH + + cat >"$LIBEXEC/kokoro_cli.py" <<'PY' +#!/usr/bin/env python3 +import argparse +import os +import struct +import sys + +import numpy as np + +DEFAULT_VOICES = [ + "af_bella", "af_sarah", "am_adam", "am_michael", + "bf_emma", "bf_isabella", "bm_george", "bm_lewis", +] + + +def resample(samples, src_rate, dst_rate): + if src_rate == dst_rate or len(samples) == 0: + return samples + duration = len(samples) / float(src_rate) + out_len = max(1, int(duration * dst_rate)) + x_old = np.linspace(0.0, duration, num=len(samples), endpoint=False) + x_new = np.linspace(0.0, duration, num=out_len, endpoint=False) + return np.interp(x_new, x_old, samples).astype(np.float32) + + +def main(): + parser = argparse.ArgumentParser(description="InferNode Kokoro stdout-PCM wrapper") + parser.add_argument("--voice", default="af_bella") + parser.add_argument("--format", choices=["pcm"], default="pcm") + parser.add_argument("--rate", type=int, default=24000) + parser.add_argument("--list-voices", action="store_true") + args = parser.parse_args() + + if args.list_voices: + print("\n".join(DEFAULT_VOICES)) + return 0 + + text = sys.stdin.read().strip() + if not text: + return 0 + + prefix = os.environ.get( + "INFERNODE_SPEECH_HOME", + os.path.expanduser("~/.local/share/infernode-speech"), + ) + model = os.path.join(prefix, "models", "kokoro", "kokoro-v1.0.onnx") + voices = os.path.join(prefix, "models", "kokoro", "voices-v1.0.bin") + + from kokoro_onnx import Kokoro + + kokoro = Kokoro(model, voices) + samples, sample_rate = kokoro.create(text, voice=args.voice, speed=1.0, lang="en-us") + samples = np.asarray(samples, dtype=np.float32) + samples = resample(samples, int(sample_rate), args.rate) + samples = np.clip(samples, -1.0, 1.0) + pcm = (samples * 32767.0).astype(""$BIN/whisper-stream-cli" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +PREFIX=${INFERNODE_SPEECH_HOME:-"$HOME/.local/share/infernode-speech"} +exec "$PREFIX/libexec/whisper_stream_cli.sh" "$@" +SH + + cat >"$LIBEXEC/whisper_stream_cli.sh" <<'SH' +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +usage: whisper-stream-cli [--model PATH] [--rate HZ] [--chans N] [--capture ID] [--length MS] [--stdin] + +Wrap whisper.cpp's whisper-stream helper in VAD mode (--step 0): each +utterance is transcribed after you stop speaking and emitted as one record: + final + +VAD mode is what makes a voice turn complete — sliding-window step mode +only ever yields interim hypotheses, which the voice-mode daemon treats +as partials and never injects. + +The --stdin mode reads s16le PCM and uses the repo-owned energy-VAD adapter +around whisper-cli. Records include the aggregate token confidence: + partial confidence=0.8123 + final confidence=0.9234 +EOF +} + +model="" +rate=16000 +chans=1 +capture=${INFERNODE_SPEECH_CAPTURE:--1} +length=${INFERNODE_SPEECH_WINDOW_MS:-5000} +stdin_mode=0 +while [ "$#" -gt 0 ]; do + case "$1" in + --model) + model=${2:-} + shift 2 + ;; + --rate) + rate=${2:-16000} + shift 2 + ;; + --chans) + chans=${2:-1} + shift 2 + ;; + --capture) + capture=${2:--1} + shift 2 + ;; + --length) + length=${2:-5000} + shift 2 + ;; + --stdin) + stdin_mode=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "error: unknown argument: $1" + exit 2 + ;; + esac +done + +if [ -z "$model" ]; then + model=${INFERNODE_SPEECH_HOME:-"$HOME/.local/share/infernode-speech"}/models/ggml-base.en.bin +fi + +if [ "$stdin_mode" -eq 1 ]; then + PREFIX=${INFERNODE_SPEECH_HOME:-"$HOME/.local/share/infernode-speech"} + exec "$PREFIX/venv/bin/python" "$PREFIX/libexec/whisper_stdin_cli.py" \ + --stdin --model "$model" --rate "$rate" --chans "$chans" --length "$length" +fi + +find_whisper_stream() { + if command -v whisper-stream >/dev/null 2>&1; then + command -v whisper-stream + return 0 + fi + if command -v brew >/dev/null 2>&1; then + local prefix + prefix=$(brew --prefix whisper-cpp 2>/dev/null || true) + if [ -n "$prefix" ] && [ -x "$prefix/bin/whisper-stream" ]; then + printf '%s\n' "$prefix/bin/whisper-stream" + return 0 + fi + fi + return 1 +} + +bin=$(find_whisper_stream || true) +if [ -z "$bin" ]; then + echo "error: whisper-stream binary not found; install Homebrew whisper-cpp" + exit 0 +fi +if [ ! -s "$model" ]; then + echo "error: whisper model not found: $model" + exit 0 +fi + +# VAD mode: whisper-stream waits for end-of-utterance, then prints the +# transcribed segment. Filter its chrome — "### Transcription" separators, +# ANSI escapes, [timestamp] blocks — and emit each utterance as a final. +# +# No stdio filter (tr, sed, grep) may sit in this pipeline: writing to a +# pipe they block-buffer, and transcript lines are tiny, so records would +# sit in the filter's buffer until the helper exits — which it never does. +# \r is stripped per line in bash instead. stderr is not discarded: the +# shim keeps a bounded tail of it, the only diagnostic when whisper dies. +esc=$(printf '\033') +"$bin" --model "$model" --capture "$capture" --step 0 --length "$length" --keep 200 --vad-thold 0.6 | +while IFS= read -r line; do + line=${line//$'\r'/} + case "$line" in + '#'*) continue ;; + esac + text=$(printf '%s' "$line" | sed -E "s/${esc}\[[0-9;]*[A-Za-z]//g; s/\[[^]]*\]//g; s/^[[:space:]]+//; s/[[:space:]]+\$//") + [ -n "$text" ] || continue + echo "final $text" +done +SH + + cat >"$BIN/openwakeword-cli" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +PREFIX=${INFERNODE_SPEECH_HOME:-"$HOME/.local/share/infernode-speech"} +exec "$PREFIX/venv/bin/python" "$PREFIX/libexec/openwakeword_cli.py" "$@" +SH + + cat >"$LIBEXEC/openwakeword_cli.py" <<'PY' +#!/usr/bin/env python3 +import argparse +import os +import signal +import sys +import time + +import numpy as np + + +def parse_args(): + parser = argparse.ArgumentParser(description="InferNode openWakeWord wrapper") + parser.add_argument("--word", default="hey jarvis") + parser.add_argument("--threshold", type=float, default=0.5) + parser.add_argument("--stdin", action="store_true", help="read 16 kHz s16le mono PCM from stdin") + parser.add_argument("--model", default="", help="explicit model file or openWakeWord model name") + parser.add_argument("--rate", type=int, default=16000) + return parser.parse_args() + + +def model_for(word, explicit): + if explicit: + return explicit + prefix = os.environ.get( + "INFERNODE_SPEECH_HOME", + os.path.expanduser("~/.local/share/infernode-speech"), + ) + candidates = [ + os.path.join(prefix, "models", "openwakeword", "hey_jarvis_v0.1.onnx"), + os.path.join(prefix, "models", "openwakeword", "hey_jarvis_v0.1.tflite"), + ] + for path in candidates: + if os.path.exists(path): + return path + normalized = word.lower().replace("_", " ").strip() + if normalized in ("hey lucia", "lucia"): + return "hey jarvis" + return normalized or "hey jarvis" + + +def load_model(selected): + from openwakeword.model import Model + + framework = "onnx" if selected.endswith(".onnx") else "tflite" + return Model(wakeword_models=[selected], inference_framework=framework) + + +def handle_frame(model, frame, threshold): + scores = model.predict(frame) + best_name = "" + best_score = 0.0 + for name, score in scores.items(): + value = float(score) + if value > best_score: + best_name = name + best_score = value + if best_score >= threshold: + print("wake %s %.4f" % (best_name or "wake", best_score), flush=True) + + +def run_stdin(args, model): + chunk_bytes = 1280 * 2 + while True: + data = sys.stdin.buffer.read(chunk_bytes) + if not data: + # stdin EOF: the capture pump closed our stdin (device gone, + # config change). Exit so the shim's restart logic owns recovery. + return 0 + usable = len(data) - (len(data) % 2) + if usable <= 0: + continue + frame = np.frombuffer(data[:usable], dtype=np.int16) + handle_frame(model, frame, args.threshold) + + +def run_microphone(args, model): + try: + import sounddevice as sd + except Exception: + print("error: microphone capture requires sounddevice; use micmode device with --stdin", flush=True) + return 0 + + with sd.RawInputStream(samplerate=args.rate, channels=1, dtype="int16", blocksize=1280) as stream: + while True: + data, _overflowed = stream.read(1280) + frame = np.frombuffer(data, dtype=np.int16) + handle_frame(model, frame, args.threshold) + + +def main(): + signal.signal(signal.SIGTERM, lambda _signum, _frame: sys.exit(0)) + args = parse_args() + selected = model_for(args.word, args.model) + model = load_model(selected) + if args.stdin: + return run_stdin(args, model) + return run_microphone(args, model) + + +if __name__ == "__main__": + raise SystemExit(main()) +PY + + install -m 755 "$SCRIPT_DIR/whisper_stdin_cli.py" "$LIBEXEC/whisper_stdin_cli.py" + + chmod +x "$BIN/kokoro-cli" "$BIN/whisper-stream-cli" "$BIN/openwakeword-cli" + chmod +x "$LIBEXEC/kokoro_cli.py" "$LIBEXEC/whisper_stream_cli.sh" "$LIBEXEC/openwakeword_cli.py" +} + +# Locate (or fetch) the cache-aware streaming EOU GGUF. Echoes the path on +# stdout, or nothing when unavailable. The model is not yet in the published +# mudler/parakeet-cpp-gguf set, so the download is attempted last and is +# allowed to fail. +find_parakeet_eou_model() { + if [ -n "$PARAKEET_EOU_MODEL" ] && [ -s "$PARAKEET_EOU_MODEL" ]; then + echo "$PARAKEET_EOU_MODEL" + return 0 + fi + local m + for m in "$PARAKEET_DIR"/parakeet_realtime_eou_120m*.gguf; do + [ -s "$m" ] && { echo "$m"; return 0; } + done + # Dev convenience: copy a locally converted model out of a checkout. + for m in "$PARAKEET_SRC"/models/parakeet_realtime_eou_120m*.gguf \ + "$HOME"/Projects/parakeet.cpp/models/parakeet_realtime_eou_120m*.gguf; do + if [ -s "$m" ]; then + mkdir -p "$PARAKEET_DIR" + cp "$m" "$PARAKEET_DIR/" + echo "$PARAKEET_DIR/$(basename "$m")" + return 0 + fi + done + local dest="$PARAKEET_DIR/$(basename "$PARAKEET_EOU_URL")" + if download_once "$PARAKEET_EOU_URL" "$dest" 2>/dev/null && [ -s "$dest" ]; then + echo "$dest" + return 0 + fi + rm -f "$dest.tmp" + return 0 +} + +# Build parakeet-stream: InferNode's realtime STT adapter (the tracked +# source tools/parakeet_stream.cpp) compiled against an upstream clone of +# parakeet.cpp. Native EOU turn-taking, faster and more accurate than the +# whisper base.en VAD wrapper. Soft-fails at every step — whisper remains +# the fallback stack. +install_parakeet() { + local jobs=4 + if ! command -v cmake >/dev/null 2>&1 || ! command -v git >/dev/null 2>&1; then + log "skip: parakeet needs cmake + git (whisper stack remains the default)" + return 0 + fi + if [ ! -f "$PARAKEET_SRC/CMakeLists.txt" ]; then + log "clone: $PARAKEET_REPO_URL" + if ! git clone --depth 1 --recurse-submodules --shallow-submodules \ + "$PARAKEET_REPO_URL" "$PARAKEET_SRC"; then + log "skip: parakeet clone failed (whisper stack remains the default)" + return 0 + fi + fi + + local build="$PARAKEET_SRC/build-infernode" + local metal_flag="" + if [ "$(uname -s)" = "Darwin" ] && [ "$(uname -m)" = "arm64" ]; then + metal_flag="-DPARAKEET_GGML_METAL=ON" + fi + if [ ! -f "$build/CMakeCache.txt" ]; then + log "cmake: configuring parakeet.cpp" + if ! cmake -B "$build" -S "$PARAKEET_SRC" -DPARAKEET_SHARED=ON \ + -DPARAKEET_BUILD_CLI=OFF -DGGML_NATIVE=OFF $metal_flag \ + -DCMAKE_BUILD_TYPE=Release >/dev/null; then + log "skip: parakeet cmake configure failed" + return 0 + fi + fi + log "build: libparakeet (~1 min)" + if ! cmake --build "$build" -j "$jobs" >/dev/null; then + log "skip: parakeet build failed" + return 0 + fi + + local libdir="" f + for f in "$build"/libparakeet.dylib "$build"/libparakeet.so \ + "$build"/src/libparakeet.dylib "$build"/src/libparakeet.so; do + [ -e "$f" ] && { libdir=$(dirname "$f"); break; } + done + if [ -z "$libdir" ]; then + log "skip: libparakeet not found under $build" + return 0 + fi + local ggmldir="$build/third_party/ggml/src" + + log "compile: parakeet-stream adapter" + if ! c++ -std=c++17 -O2 "$SCRIPT_DIR/parakeet_stream.cpp" \ + -I"$PARAKEET_SRC/src" -I"$PARAKEET_SRC/include" \ + -I"$PARAKEET_SRC/third_party/ggml/include" \ + -L"$libdir" -L"$ggmldir" -lparakeet -lggml-base \ + -Wl,-rpath,"$libdir" -Wl,-rpath,"$ggmldir" \ + -o "$BIN/parakeet-stream"; then + log "skip: parakeet-stream compile failed" + return 0 + fi + + local model + model=$(find_parakeet_eou_model) + if [ -z "$model" ]; then + log "notice: parakeet-stream built, but no streaming EOU model found." + log " Convert nvidia/parakeet_realtime_eou_120m-v1 with parakeet.cpp's" + log " scripts/convert_parakeet_to_gguf.py into $PARAKEET_DIR/," + log " or set PARAKEET_EOU_MODEL=/path/to/model.gguf and re-run." + log " Falling back to the whisper stack until then." + return 0 + fi + + # Smoke: model must load and the adapter must exit cleanly on EOF. + log "smoke: parakeet-stream loads $model" + if ! dd if=/dev/zero bs=32000 count=1 2>/dev/null | \ + "$BIN/parakeet-stream" --stdin --model "$model" --rate 16000 >/dev/null; then + log "skip: parakeet-stream smoke test failed (whisper stack remains)" + return 0 + fi + + PARAKEET_OK=1 + PARAKEET_MODEL_PATH="$model" + log "ok: parakeet realtime STT installed" +} + +# The boot-time speech configuration, one Inferno-sh command per line. +# lib/lucifer/boot.sh runs this file verbatim when it exists, making the +# installer the single source of truth for which helper stack is active. +write_speech_ctl() { + local ctl="$PREFIX/speech.ctl.sh" + { + echo "# Written by tools/install-speech-helpers.sh — applied by boot.sh." + echo "# Regenerate by re-running the installer; hand-edits survive until then." + echo "echo 'engine kokoro' > /n/speech/ctl" + echo "echo 'kokorobin $BIN/kokoro-cli' > /n/speech/ctl" + echo "echo 'wakebin $BIN/openwakeword-cli' > /n/speech/ctl" + echo "echo 'voice af_bella' > /n/speech/ctl" + echo "echo 'wakeword hey jarvis' > /n/speech/ctl" + echo "echo 'wakethreshold 0.5' > /n/speech/ctl" + echo "echo 'duplex half' > /n/speech/ctl" + if [ "$PARAKEET_OK" = 1 ]; then + echo "echo 'whisperstreambin $BIN/parakeet-stream' > /n/speech/ctl" + echo "echo 'whispermodel $PARAKEET_MODEL_PATH' > /n/speech/ctl" + echo "echo 'micmode device' > /n/speech/ctl" + echo "echo 'capturerate 16000' > /n/speech/ctl" + else + echo "echo 'whisperstreambin $BIN/whisper-stream-cli' > /n/speech/ctl" + echo "echo 'whispermodel $WHISPER_MODEL' > /n/speech/ctl" + echo "echo 'micmode helper' > /n/speech/ctl" + fi + } > "$ctl" + log "wrote: $ctl" +} + +print_ctl_block() { + cat < /n/speech/ctl +echo 'whispermodel $WHISPER_MODEL' > /n/speech/ctl +echo 'micmode helper' > /n/speech/ctl +EOF + else + cat < +// final confidence=0.9123 +// +// A `final` fires on each model-emitted end-of-utterance (/) +// event — the model itself decides when a turn is over, replacing the +// energy-VAD heuristic the whisper wrapper needs. `confidence=` is the +// mean of the utterance's per-word confidences (NeMo max_prob, min- +// aggregated per word by parakeet.cpp); it is omitted when no words were +// finalized. Exits 0 on stdin EOF after flushing the tail. +// +// Built by tools/install-speech-helpers.sh against a clone of +// https://github.com/mudler/parakeet.cpp — only committed upstream API is +// used (ModelLoader, StreamingMel, StreamingSession). The chunk windowing +// below mirrors the schedule of parakeet.cpp's test_streaming_encoder: +// chunk 0 = chunk_size_first frames, no overlap; later chunks = +// pre_encode_cache_size overlap + chunk_size frames; keep_all_outputs on +// the final (flush) chunk only. +// +// Flags mirror the whisper-stream wrapper so the shim can exec either +// binary from the same ctl configuration. Unknown flags are ignored with +// a warning rather than rejected, so shim-side additions never hard-break +// an installed adapter. + +#include +#include +#include +#include +#include +#include +#include + +#include "model.hpp" +#include "mel.hpp" +#include "streaming.hpp" +#include "ggml_graph.hpp" // pk::set_num_threads + +#include "ggml.h" // ggml_log_set, to keep helper stderr readable + +namespace { + +std::string trim_copy(const std::string& s) { + size_t b = s.find_first_not_of(" \t\r\n"); + if (b == std::string::npos) return ""; + size_t e = s.find_last_not_of(" \t\r\n"); + return s.substr(b, e - b + 1); +} + +// Column-append newly-ready mel frames onto the row-major [n_mels, T] buffer. +void append_mel_frames(std::vector& mel_buf, int n_mels, int& mel_T, + const std::vector& frames, int n_new) { + if (n_new <= 0) return; + const int old_T = mel_T; + const int new_T = old_T + n_new; + std::vector out((size_t)n_mels * new_T); + for (int m = 0; m < n_mels; ++m) { + for (int t = 0; t < old_T; ++t) + out[(size_t)m * new_T + t] = mel_buf[(size_t)m * old_T + t]; + for (int t = 0; t < n_new; ++t) + out[(size_t)m * new_T + (old_T + t)] = frames[(size_t)m * n_new + t]; + } + mel_buf.swap(out); + mel_T = new_T; +} + +std::vector mel_window(const std::vector& mel, int n_mels, int T, + int lo, int hi) { + const int len = hi - lo; + std::vector w((size_t)n_mels * len); + for (int m = 0; m < n_mels; ++m) + for (int t = 0; t < len; ++t) + w[(size_t)m * len + t] = mel[(size_t)m * T + (lo + t)]; + return w; +} + +// Feed every complete chunk window available in mel_buf to the session. +// A window that reaches the buffer end is held back unless `flush` — its +// frames may still grow — except on flush, where it goes in with is_last. +void feed_ready_chunks(pk::StreamingSession& sess, + const std::vector& mel_buf, + int n_mels, int mel_T, int& fed_idx, + bool& first_chunk, bool flush) { + const int chunk0 = sess.chunk_size_first(); + const int chunk_main = sess.chunk_size(); + const int pre_cache = sess.pre_encode_cache_size(); + while (fed_idx < mel_T) { + const int chunk_size = first_chunk ? chunk0 : chunk_main; + const int hi = std::min(fed_idx + chunk_size, mel_T); + if (hi - fed_idx <= 0) break; + const bool reaches_end = (hi >= mel_T); + if (!flush && reaches_end) break; + const int lo = first_chunk ? fed_idx : std::max(0, fed_idx - pre_cache); + std::vector win = mel_window(mel_buf, n_mels, mel_T, lo, hi); + const bool is_last = flush && reaches_end; + sess.feed_mel_chunk(win, hi - lo, is_last); + fed_idx += chunk_size; + first_chunk = false; + if (is_last) break; + } +} + +struct RecordEmitter { + pk::StreamingSession& sess; + size_t finalized_chars = 0; + std::string last_partial; + std::vector word_confs; + + explicit RecordEmitter(pk::StreamingSession& s) : sess(s) {} + + void collect_words() { + for (const pk::Word& w : sess.drain_words()) + word_confs.push_back(w.conf); + } + + void emit_final_if_any() { + const std::string& text = sess.text(); + if (text.size() > finalized_chars) { + const std::string utter = trim_copy(text.substr(finalized_chars)); + if (!utter.empty()) { + if (word_confs.empty()) { + std::printf("final %s\n", utter.c_str()); + } else { + double sum = 0.0; + for (float c : word_confs) sum += c; + std::printf("final confidence=%.4f %s\n", + sum / word_confs.size(), utter.c_str()); + } + std::fflush(stdout); + } + finalized_chars = text.size(); + } + word_confs.clear(); + last_partial.clear(); + } + + // Returns true when an end-of-utterance final was emitted, so the + // caller can reset the stream for the next turn. + bool step(bool flush) { + collect_words(); + std::vector evs = sess.drain_events(); + if (!evs.empty() || flush) { + emit_final_if_any(); + if (!evs.empty()) + return true; + } + if (flush) + return false; + const std::string cur = trim_copy(sess.text().substr(finalized_chars)); + if (!cur.empty() && cur != last_partial) { + std::printf("partial %s\n", cur.c_str()); + std::fflush(stdout); + last_partial = cur; + } + return false; + } +}; + +// Read exactly n bytes unless EOF interrupts; returns bytes read. +size_t read_full(void* buf, size_t n) { + size_t got = 0; + while (got < n) { + size_t r = std::fread((char*)buf + got, 1, n - got, stdin); + if (r == 0) break; + got += r; + } + return got; +} + +} // namespace + +int main(int argc, char** argv) { + // The shim tails this helper's stderr into its log; ggml's Metal + // pipeline-compile chatter and [parakeet] load logs would bury real + // errors there. PARAKEET_STREAM_DEBUG=1 restores them. + if (std::getenv("PARAKEET_STREAM_DEBUG") == nullptr) { + setenv("PARAKEET_LOG", "0", 1); + ggml_log_set([](enum ggml_log_level, const char*, void*) {}, nullptr); + } + + std::string model_path; + int rate = 16000; + int chans = 1; + bool use_stdin = false; + + for (int i = 1; i < argc; ++i) { + if (std::strcmp(argv[i], "--model") == 0 && i + 1 < argc) { + model_path = argv[++i]; + } else if (std::strcmp(argv[i], "--rate") == 0 && i + 1 < argc) { + rate = std::atoi(argv[++i]); + } else if (std::strcmp(argv[i], "--chans") == 0 && i + 1 < argc) { + chans = std::atoi(argv[++i]); + } else if (std::strcmp(argv[i], "--stdin") == 0) { + use_stdin = true; + } else if (std::strcmp(argv[i], "--threads") == 0 && i + 1 < argc) { + pk::set_num_threads(std::atoi(argv[++i])); + } else if (i + 1 < argc && argv[i][0] == '-' && argv[i + 1][0] != '-') { + std::fprintf(stderr, "parakeet-stream: ignoring %s %s\n", + argv[i], argv[i + 1]); + ++i; + } else { + std::fprintf(stderr, "parakeet-stream: ignoring %s\n", argv[i]); + } + } + if (model_path.empty()) { + std::fprintf(stderr, + "usage: parakeet-stream --stdin --model " + "[--rate HZ] [--chans N] [--threads N]\n"); + return 2; + } + if (!use_stdin) { + // The adapter deliberately has no microphone code: audio capture + // belongs to the shim's capture pump (`micmode device`), which is + // what makes remote-mic topologies pure namespace composition. + std::fprintf(stderr, + "error: parakeet-stream only supports --stdin PCM input; " + "set 'micmode device' on the speech shim\n"); + return 2; + } + if (rate <= 0 || chans <= 0) { + std::fprintf(stderr, "error: bad --rate/--chans\n"); + return 2; + } + + pk::ModelLoader ml; + if (!ml.load(model_path)) { + std::fprintf(stderr, "error: cannot load model %s\n", model_path.c_str()); + return 1; + } + if (!ml.config().streaming.present) { + std::fprintf(stderr, + "error: %s is not a cache-aware streaming model " + "(need parakeet_realtime_eou_120m-v1)\n", model_path.c_str()); + return 1; + } + + pk::StreamingMel mel(ml); + pk::StreamingSession sess(ml); + const int n_mels = mel.n_mels(); + std::vector mel_buf; + int mel_T = 0; + int fed_idx = 0; + bool first_chunk = true; + RecordEmitter emitter(sess); + + // 100ms of input per iteration keeps partial latency low without + // burning a graph launch per tiny read. + const int block_samples = rate / 10; + std::vector raw((size_t)block_samples * chans); + std::vector mono; + mono.reserve(block_samples); + // Linear-resampler carry between blocks (position in input samples). + double resample_pos = 0.0; + float prev_sample = 0.0f; + bool have_prev = false; + std::vector pcm16k; + + for (;;) { + size_t want = raw.size() * sizeof(int16_t); + size_t got = read_full(raw.data(), want); + size_t n_in = got / sizeof(int16_t) / chans; + if (n_in == 0) break; + + mono.clear(); + for (size_t i = 0; i < n_in; ++i) { + int acc = 0; + for (int c = 0; c < chans; ++c) acc += raw[i * chans + c]; + mono.push_back((float)(acc / chans) / 32768.0f); + } + + const std::vector* feed = &mono; + if (rate != 16000) { + // Streaming linear resample, carrying one sample across blocks. + pcm16k.clear(); + const double step = (double)rate / 16000.0; + while (true) { + double pos = resample_pos; + long idx = (long)pos; + if (idx >= (long)mono.size() - (have_prev ? 0 : 1)) break; + float s0, s1; + if (idx < 0) { s0 = have_prev ? prev_sample : mono[0]; s1 = mono[0]; } + else { s0 = mono[(size_t)idx]; s1 = (size_t)idx + 1 < mono.size() ? mono[(size_t)idx + 1] : mono[(size_t)idx]; } + double frac = pos - idx; + pcm16k.push_back((float)(s0 + (s1 - s0) * frac)); + resample_pos += step; + if ((long)resample_pos >= (long)mono.size()) break; + } + resample_pos -= (double)mono.size(); + prev_sample = mono.back(); + have_prev = true; + feed = &pcm16k; + } + if (feed->empty()) continue; + + int n_new = 0; + std::vector frames = mel.feed(feed->data(), (int)feed->size(), n_new); + append_mel_frames(mel_buf, n_mels, mel_T, frames, n_new); + feed_ready_chunks(sess, mel_buf, n_mels, mel_T, fed_idx, first_chunk, false); + + // After the model emits , the streaming session stops + // producing text (matching NeMo's reset-per-turn realtime recipe; + // verified against upstream's own file --stream path). Restart the + // whole stream at each utterance boundary — that is also what + // bounds the session's hypothesis growth over an hours-long voice + // session. The few mel tail samples dropped land in post-turn + // silence. + if (emitter.step(false)) { + sess.reset(); + mel.reset(); + mel_buf.clear(); + mel_T = 0; + fed_idx = 0; + first_chunk = true; + emitter.finalized_chars = 0; + emitter.last_partial.clear(); + emitter.word_confs.clear(); + } + } + + // EOF: flush the mel tail and the decoder, then emit any trailing text + // as a final so a capture teardown never swallows a spoken turn. + int n_tail = 0; + std::vector tail = mel.finalize(n_tail); + append_mel_frames(mel_buf, n_mels, mel_T, tail, n_tail); + feed_ready_chunks(sess, mel_buf, n_mels, mel_T, fed_idx, first_chunk, true); + sess.finalize(); + emitter.step(true); + + // ggml-metal's static destructors abort in __cxa_finalize (observed on + // macOS arm64 Metal builds); every record is already flushed, so skip + // static teardown instead of crashing a clean shutdown into SIGABRT. + std::fflush(stdout); + std::_Exit(0); +} diff --git a/tools/speech-regress.sh b/tools/speech-regress.sh new file mode 100755 index 000000000..6e9100177 --- /dev/null +++ b/tools/speech-regress.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# +# speech-regress.sh — one-command regression suite for the voice/speech stack. +# +# Builds and runs the targeted Limbo suites for the Lucia voice bridge/UI, +# voicemode, speech9p, speechshim9p, and the speech tooling inside the +# emulator, then the host-side installer/helper/audio smoke tests. This is +# deliberately NOT +# ./run-tests.sh: +# it touches only the speech/voice suites, so it is cheap enough to run +# after every change to appl/cmd/voicemode.b, appl/veltro/speech*.b, +# module/speech.m, tools/install-speech-helpers.sh, or the boot wiring. +# +# Pass criteria come from the testing framework's output (a final PASS +# summary and no "--- FAIL:" lines), not only the exit status. +# +# usage: tools/speech-regress.sh [-v] +# -v stream each suite's output as it runs (failures always print) + +set -u + +verbose=0 +[ "${1:-}" = "-v" ] && verbose=1 + +cd "$(dirname "$0")/.." || exit 1 +export ROOT=$PWD + +case "$(uname -s)" in +Darwin) SYSHOST=MacOSX ;; +Linux) SYSHOST=Linux ;; +*) echo "speech-regress: unsupported host: $(uname -s)" >&2; exit 1 ;; +esac +objtype=$(uname -m) +case "$objtype" in +arm64|aarch64) objtype=arm64 ;; +x86_64|amd64) objtype=amd64 ;; +esac +export PATH=$ROOT/$SYSHOST/$objtype/bin:$PATH + +EMU=${EMU:-$ROOT/emu/$SYSHOST/o.emu} +if [ ! -x "$EMU" ]; then + echo "speech-regress: emulator not built: $EMU" >&2 + exit 1 +fi + +# Emu suites, cheapest protocol tests first. The Lucia bridge/UI suites are +# included because voice controls, approval, live drafts, and TTS lifecycle +# now cross those boundaries. +SUITES=" +speechshim_test +speech9p_voice_test +speech_wake_test +speech_listen_test +voicemode_test +lucibridge_test +lucibridge_approval_test +luciuisrv_test +speechtest_test +voice_scripts_test +speech_kokoro_test +" + +# Built here but run through its host wrapper below because it needs a +# loopback OpenAI-compatible server and deterministic host speech helpers. +BUILD_ONLY_SUITES="speech_e2e_test" + +# Rebuild only the suites we run. Without the native mk (fresh clone, +# tools not bootstrapped) fall back to whatever bytecode is already there. +if command -v mk >/dev/null 2>&1; then + echo "== building test bytecode" + buildlog=$(mktemp "${TMPDIR:-/tmp}/speech-regress-build.XXXXXX") || { + echo "speech-regress: could not create build log" >&2 + exit 1 + } + for t in $SUITES $BUILD_ONLY_SUITES; do + if ! (cd tests && mk "$t.dis") >>"$buildlog" 2>&1; then + echo "speech-regress: build failed for $t:" >&2 + cat "$buildlog" >&2 + rm -f "$buildlog" + exit 1 + fi + done + rm -f "$buildlog" +else + echo "speech-regress: native mk not on PATH; running existing bytecode" >&2 +fi + +logdir=$(mktemp -d "${TMPDIR:-/tmp}/speech-regress.XXXXXX") || { + echo "speech-regress: could not create log directory" >&2 + exit 1 +} +trap 'rm -rf "$logdir"' EXIT + +failed="" +npass=0 +nskip=0 + +run_suite() { + local name=$1 limit=$2 + local log=$logdir/$name.log status ok=0 + + printf '== %s ' "$name" + if [ "$verbose" = 1 ]; then + echo + timeout "$limit" "$EMU" -r. "/tests/$name.dis" 2>&1 | tee "$log" + status=${PIPESTATUS[0]} + else + timeout "$limit" "$EMU" -r. "/tests/$name.dis" >"$log" 2>&1 + status=$? + fi + + # Inferno's emulator does not have one portable success exit status after + # the test program finishes: it may exit normally, be reaped by timeout(1), + # or terminate itself with SIGKILL (137 on Linux). The testing framework's + # final PASS marker, with no failure marker, is the authoritative verdict. + if grep -q '^PASS$' "$log" && ! grep -q -- '--- FAIL:' "$log"; then + ok=1 + fi + + if [ "$ok" = 1 ]; then + echo "PASS" + npass=$((npass + 1)) + else + echo "FAIL (exit $status)" + failed="$failed $name" + if [ "$verbose" != 1 ]; then + echo "---- tail of $name output ----" + tail -30 "$log" + echo "------------------------------" + fi + fi +} + +for t in $SUITES; do + limit=240 + [ "$t" = speech_kokoro_test ] && limit=120 + run_suite "$t" "$limit" +done + +run_host_test() { + local name=$1 log status=0 skips + log=$logdir/$name.log + + printf '== %s ' "$name" + bash "tests/host/$name" >"$log" 2>&1 || status=$? + case "$status" in + 0) + skips=$(grep '^SKIP:' "$log" 2>/dev/null | tr '\n' ';' || true) + if [ -n "$skips" ]; then + echo "PASS (partial: ${skips%;})" + else + echo "PASS" + fi + npass=$((npass + 1)) + ;; + 77) + echo "SKIP ($(grep '^SKIP:' "$log" | head -1))" + nskip=$((nskip + 1)) + ;; + *) + echo "FAIL (exit $status)" + failed="$failed $name" + echo "---- tail of $name output ----" + tail -30 "$log" + echo "----------------------------------" + ;; + esac +} + +# This is the blocking composed path: real Lucia/LLM/speech services with +# deterministic loopback fixtures replacing only microphones and models. +run_host_test speech_e2e_test.sh + +# The download test is hermetic: it sources the installer with a fake curl. +run_host_test speech_installer_download_test.sh + +# The helper test returns 77 when no helper install exists. Its deterministic +# stdin-PCM coverage still runs without microphone permission when installed. +run_host_test speech_helpers_test.sh + +# CoreAudio coverage is meaningful only on macOS. It reports partial skips +# when the current session lacks an audio device or TCC microphone permission. +if [ "$(uname -s)" = Darwin ]; then + run_host_test audio_macos_test.sh +fi + +echo +if [ -n "$failed" ]; then + echo "speech-regress: FAIL:$failed" + exit 1 +fi +if [ "$nskip" -gt 0 ]; then + echo "speech-regress: $npass passed, $nskip skipped" +else + echo "speech-regress: all $npass suites passed" +fi diff --git a/tools/speech-test.sh b/tools/speech-test.sh new file mode 100755 index 000000000..4a11c88e8 --- /dev/null +++ b/tools/speech-test.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# +# speech-test.sh — boot InferNode in speech test mode: microphone -> STT +# (live partials) -> a hard-coded TTS phrase for every final transcript. +# No LLM, no login, no API key. +# +# Headless (default): thin wrapper around /dis/speechtest.dis +# (appl/cmd/speechtest.b), which bootstraps speechshim9p + speech9p +# itself and prints partials/finals to this terminal. +# +# GUI (--gui): boots the full lucifer desktop via +# /lib/lucifer/boot-speechtest.sh with voicemode in LLM-free test mode — +# say "hey jarvis", speak, watch the bordered unsent turn update, hear +# the canned phrase. Esc-V / Voice-chip click toggle voice mode as usual. +# +# Usage: +# tools/speech-test.sh # headless, defaults +# tools/speech-test.sh --gui # full desktop, no LLM +# tools/speech-test.sh -p 'Hello from InferNode' # custom phrase +# tools/speech-test.sh -e # echo the transcript back +# tools/speech-test.sh -n 3 # exit after 3 turns (headless) +# +# Remote topologies (headless only; see docs/SPEECH-REMOTE-AUDIO.md; +# mounts are unauthenticated — trusted networks only): +# # remote STT+TTS provider: +# tools/speech-test.sh --no-helpers \ +# -M 'tcp!fast-box!7770 /n/remotespeech' -c 'provider /n/remotespeech' +# # remote microphone (e.g. InferNode on a phone exporting /dev/audio): +# tools/speech-test.sh \ +# -M 'tcp!phone!7771 /n/phoneaudio' \ +# -c 'capturedev /n/phoneaudio/audio' -c 'micmode device' +# +# The terminal app needs macOS microphone permission (TCC) for local +# capture; approve the prompt on first run. Ctrl-C exits. +# +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +HELPERS="${INFERNODE_SPEECH_HOME:-$HOME/.local/share/infernode-speech}" + +case "$(uname -s)" in +Darwin) EMU="$ROOT/emu/MacOSX/o.emu" ;; +Linux) EMU="$ROOT/emu/Linux/o.emu" ;; +*) echo "error: unsupported platform $(uname -s)" >&2; exit 1 ;; +esac +if [ ! -x "$EMU" ]; then + echo "error: emulator not built: $EMU" >&2 + exit 1 +fi + +args=(-b) +usehelpers=1 +gui=0 +phrase='Speech test complete. I heard you.' +echoflag=- +headlessonly="" +while [ $# -gt 0 ]; do + case "$1" in + -g|--gui) gui=1; shift ;; + -p|--phrase) phrase="$2"; args+=(-p "$2"); shift 2 ;; + -n|--turns) headlessonly="$headlessonly -n"; args+=(-n "$2"); shift 2 ;; + -c|--ctl) headlessonly="$headlessonly -c"; args+=(-c "$2"); shift 2 ;; + -M|--mount) headlessonly="$headlessonly -M"; args+=(-M "$2"); shift 2 ;; + -e|--echo) echoflag=-e; args+=(-e); shift ;; + -d|--debug) headlessonly="$headlessonly -d"; args+=(-d); shift ;; + --no-helpers) usehelpers=0; shift ;; + -h|--help) sed -n '2,35p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "error: unknown option $1 (see -h)" >&2; exit 2 ;; + esac +done + +helperbin=- +configfile=- +if [ "$usehelpers" = 1 ]; then + if [ -d "$HELPERS/bin" ]; then + helperbin="$HELPERS/bin" + if [ -f "$HELPERS/speech.ctl.sh" ]; then + configfile="/n/local$HELPERS/speech.ctl.sh" + fi + else + echo "note: $HELPERS/bin not found — run tools/install-speech-helpers.sh," >&2 + echo " or pass --no-helpers with -c/-M lines for a remote provider" >&2 + fi +fi + +if [ "$gui" = 1 ]; then + if [ -n "$headlessonly" ]; then + echo "error: headless-only option(s):$headlessonly — not supported with --gui" >&2 + exit 2 + fi + exec "$EMU" -c1 -pheap=1024m -pmain=1024m -pimage=1024m "-r$ROOT" \ + sh -l /lib/lucifer/boot-speechtest.sh "$helperbin" "$echoflag" "$phrase" +fi + +if [ "$configfile" != - ]; then + args=(-b -C "$configfile" "${args[@]:1}") +elif [ "$helperbin" != - ]; then + args=(-b -H "$helperbin" "${args[@]:1}") +fi + +exec "$EMU" -c1 "-r$ROOT" /dis/speechtest.dis "${args[@]}" diff --git a/tools/whisper_stdin_cli.py b/tools/whisper_stdin_cli.py new file mode 100644 index 000000000..2185d4614 --- /dev/null +++ b/tools/whisper_stdin_cli.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Stream raw PCM from stdin into whisper.cpp with lightweight energy VAD. + +The helper deliberately owns only the stdin topology. Direct microphone +capture stays with whisper-stream, while this process turns a namespace-backed +s16le stream into newline-delimited partial/final records for speechshim9p. +""" + +import argparse +import array +import collections +import json +import math +import os +import shutil +import subprocess +import sys +import tempfile +import wave + + +def parse_args(): + parser = argparse.ArgumentParser( + description="InferNode whisper.cpp stdin-PCM streaming adapter" + ) + parser.add_argument("--model", required=True) + parser.add_argument("--rate", type=int, default=16000) + parser.add_argument("--chans", type=int, default=1) + parser.add_argument("--length", type=int, default=15000, + help="maximum utterance length in milliseconds") + parser.add_argument("--stdin", action="store_true") + return parser.parse_args() + + +def whisper_cli(): + explicit = os.environ.get("INFERNODE_WHISPER_CLI", "") + if explicit: + return explicit + found = shutil.which("whisper-cli") + if found: + return found + brew = shutil.which("brew") + if brew: + try: + prefix = subprocess.check_output( + [brew, "--prefix", "whisper-cpp"], text=True, + stderr=subprocess.DEVNULL, + ).strip() + candidate = os.path.join(prefix, "bin", "whisper-cli") + if os.access(candidate, os.X_OK): + return candidate + except (OSError, subprocess.SubprocessError): + pass + return "" + + +def rms(frame): + samples = array.array("h") + samples.frombytes(frame[: len(frame) - (len(frame) % 2)]) + if sys.byteorder != "little": + samples.byteswap() + if not samples: + return 0.0 + return math.sqrt(sum(sample * sample for sample in samples) / len(samples)) + + +def clean_text(text): + return " ".join(text.replace("\r", " ").replace("\n", " ").split()) + + +def transcribe(binary, model, rate, pcm): + with tempfile.TemporaryDirectory(prefix="infernode-whisper-") as work: + audio = os.path.join(work, "utterance.wav") + output = os.path.join(work, "transcript") + with wave.open(audio, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(rate) + wav.writeframes(pcm) + + command = [ + binary, "--model", model, "--file", audio, + "--language", "en", "--no-timestamps", "--no-prints", + "--output-json-full", "--output-file", output, + ] + try: + result = subprocess.run( + command, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + text=True, timeout=120, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise RuntimeError(str(exc)) from exc + if result.returncode != 0: + detail = clean_text(result.stderr)[-300:] + raise RuntimeError(detail or "whisper-cli exited with status %d" % result.returncode) + + try: + with open(output + ".json", encoding="utf-8") as stream: + data = json.load(stream) + except (OSError, ValueError) as exc: + raise RuntimeError("whisper-cli produced no valid JSON result") from exc + + segments = data.get("transcription", []) + text = clean_text(" ".join(str(segment.get("text", "")) for segment in segments)) + probabilities = [] + for segment in segments: + for token in segment.get("tokens", []): + value = token.get("p") + if isinstance(value, (int, float)) and value > 0: + probabilities.append(max(1.0e-6, min(1.0, float(value)))) + confidence = 0.0 + if probabilities: + confidence = math.exp(sum(math.log(value) for value in probabilities) / len(probabilities)) + return text, confidence + + +def emit(kind, text, confidence): + if text: + print("%s confidence=%.4f %s" % (kind, confidence, text), flush=True) + + +def main(): + args = parse_args() + if args.chans != 1: + print("error: whisper stdin PCM requires one channel", flush=True) + return 0 + if args.rate < 8000 or args.rate > 48000: + print("error: whisper stdin PCM rate must be 8000-48000", flush=True) + return 0 + if not os.path.isfile(args.model): + print("error: whisper model not found: %s" % args.model, flush=True) + return 0 + binary = whisper_cli() + if not binary: + print("error: whisper-cli binary not found; install whisper-cpp", flush=True) + return 0 + + frame_ms = int(os.environ.get("INFERNODE_STT_FRAME_MS", "20")) + silence_ms = int(os.environ.get("INFERNODE_STT_SILENCE_MS", "700")) + partial_ms = int(os.environ.get("INFERNODE_STT_PARTIAL_MS", "1500")) + threshold = float(os.environ.get("INFERNODE_STT_RMS_THRESHOLD", "350")) + start_ms = int(os.environ.get("INFERNODE_STT_START_MS", "60")) + preroll_ms = int(os.environ.get("INFERNODE_STT_PREROLL_MS", "300")) + frame_bytes = max(2, args.rate * 2 * frame_ms // 1000) + frame_bytes -= frame_bytes % 2 + start_frames = max(1, start_ms // frame_ms) + preroll = collections.deque(maxlen=max(1, preroll_ms // frame_ms)) + + active = [] + voiced_run = 0 + trailing_silence = 0 + elapsed = 0 + next_partial = partial_ms + last_partial = "" + pending = b"" + + def recognize(kind): + nonlocal last_partial + try: + text, confidence = transcribe(binary, args.model, args.rate, b"".join(active)) + except RuntimeError as exc: + print("error: whisper stdin transcription failed: %s" % clean_text(str(exc)), flush=True) + return + if kind == "partial": + if not text or text == last_partial: + return + last_partial = text + emit(kind, text, confidence) + + while True: + chunk = sys.stdin.buffer.read(frame_bytes - len(pending)) + if not chunk: + if active: + recognize("final") + return 0 + pending += chunk + if len(pending) < frame_bytes: + continue + frame, pending = pending[:frame_bytes], pending[frame_bytes:] + voiced = rms(frame) >= threshold + + if not active: + preroll.append(frame) + voiced_run = voiced_run + 1 if voiced else 0 + if voiced_run < start_frames: + continue + active = list(preroll) + elapsed = len(active) * frame_ms + trailing_silence = 0 + next_partial = max(partial_ms, elapsed + frame_ms) + continue + + active.append(frame) + elapsed += frame_ms + trailing_silence = 0 if voiced else trailing_silence + frame_ms + + if partial_ms > 0 and elapsed >= next_partial and trailing_silence < silence_ms: + recognize("partial") + next_partial += partial_ms + + if trailing_silence >= silence_ms or elapsed >= args.length: + recognize("final") + active = [] + preroll.clear() + voiced_run = 0 + trailing_silence = 0 + elapsed = 0 + last_partial = "" + + +if __name__ == "__main__": + raise SystemExit(main())