Tap twice and start speaking. The first words land at your cursor after about - a second and a half, and the rest keep up as you go — a word or two - every four tenths of a second. When you stop, there is almost nothing left to arrive.
+ a second and a half, and the rest keep up as you go — in lumps of about + a second and a half, punctuation and all.| step | seconds |
|---|---|
| first words at your cursor | 1.5 |
| a new word or two, thereafter | 0.43 |
| still to arrive when you stop talking | 9 of 195 characters |
| a live pass, under 30s of speech | 1.5 |
| a live pass, past 30s of speech | 2.4 |
| first transcription after boot, cold | 13.3 |
| microphone open, first ever | 9.9 |
| speech lost at the start of a clip | 0.04 |
M4 Pro, macOS 26, one 10.5 second sentence. Two models do it: a small one
- answers the live pass while you are still talking, and large-v3-turbo writes the
- transcript you keep. The words are typed as key events rather than pasted, which is the
- difference between a lump every second and a word every four tenths. Measure your own before
+
M4 Pro, macOS 26. One model does both jobs —
+ large-v3-turbo answers the live pass and writes the transcript you keep —
+ because the model typing while you talk is the model that decides your punctuation. The words
+ are typed as key events rather than pasted: 2.6ms, and no clipboard. Measure your own before
believing any of these, including ours.
ctypes. No Xcode, no signing, no notarization.Your voice is transcribed by your own machine and the recording is discarded the moment that is done, before anything else happens to it. The microphone closes with - the key, and it cannot be left open.
+ the key — and by itself after fifteen seconds of quiet, so it cannot be left open.The transcript is never inspected, filtered, or written down. The words that arrive while you talk are typed straight in and never touch your clipboard at all; the final transcript goes through it and puts your previous contents back. Nothing here makes a network request except diff --git a/murmurflow/cli.py b/murmurflow/cli.py index 3c4347d..a45bf3e 100644 --- a/murmurflow/cli.py +++ b/murmurflow/cli.py @@ -32,24 +32,14 @@ def _out(line: str = "") -> None: def _setup(name: str = "") -> int: - """Download the speech models into ``~/.murmurflow/models/``. + """Download the speech model into ``~/.murmurflow/models/``. - TWO of them, and only because they do different jobs. The big one writes the transcript you - keep, and it is the single biggest lever on accuracy — bigger models are the difference between - proper nouns transcribed and proper nouns guessed at. The small one answers the live pass while - you are still talking, where 0.4s matters far more than the last few percent of accuracy (see - :data:`whisper.PARTIAL_PREFERENCE`). Naming one explicitly downloads only that one. + ONE model, and it does both jobs: it writes the transcript you keep AND answers the live pass + while you are still talking. It was two — a small model typed live, the big one wrote the final + — until the live pass began typing PUNCTUATION rather than only words. The marks it chooses are + the marks you keep, and there the two models are not close. Naming one downloads that one. """ - if name: - return _download(name) - failed = _download(whisper.DEFAULT_MODEL) - # The live model is an IMPROVEMENT, never a requirement: without it the partials go to the big - # server and arrive in ~2s lumps, which is what they did before it existed. So a failure here - # is reported and does not fail the setup somebody is waiting on. - if _download(whisper.DEFAULT_PARTIAL_MODEL): - _out("[!] the live model did not download — the words will arrive in slower lumps.") - _out(" Try again later: murmurflow setup small") - return failed + return _download(name or whisper.DEFAULT_MODEL) def _download(name: str) -> int: @@ -347,23 +337,6 @@ def _doctor(*, verbs: bool = False) -> int: ) model = whisper.model() rows.append((bool(model), f"model: {model or 'NOT FOUND'}", "murmurflow setup")) - # Its own row, because its absence is INVISIBLE and it is the whole difference between the - # words arriving as you speak and arriving in two-second lumps. Without it the partials go to - # the big server, which is correct and five times slower, and nothing anywhere says so. - live_model = whisper.partial_model() - rows.append( - ( - bool(live_model), - "live model: " - + ( - f"{live_model}" - f"{'' if dictate.server_up(dictate.partial_port()) else ' (server not up yet)'}" - if live_model - else "none — the words arrive in ~2s lumps instead of as you speak" - ), - "murmurflow setup small (~488 MB, ~5x quicker on the live pass)", - ) - ) # A warm server is the difference between the first sentence after a boot taking ~1s and # taking ~13s, and it is invisible either way — the tool just feels slow. Worth its own row # for the second reason too: a server that is up but WEDGED answers every request with an diff --git a/murmurflow/dictate.py b/murmurflow/dictate.py index 25fee8c..3774b61 100644 --- a/murmurflow/dictate.py +++ b/murmurflow/dictate.py @@ -145,9 +145,9 @@ def _cfg() -> dict[str, object]: return {} -#: The highest ``port`` that leaves room for the live server one above it. 65535 would put the -#: second server on 65536, which is not a port: it fails to bind, `partial_at` finds nothing there -#: and every partial silently falls back to the big model. +#: The highest ``port`` that leaves the one above it free. Nothing of ours listens there any +#: more, but :func:`stop_server` still sweeps it to reap the small live server an older MurmurFlow +#: ran, and a port set to 65535 would make that sweep ask about 65536, which is not a port. MAX_PORT = 65534 @@ -570,24 +570,14 @@ def _clear_state_for(pid: int) -> None: # --- warm transcription ----------------------------------------------------------------------- -def partial_port() -> int: - """The loopback port for the SECOND warm server, the small one that answers partials. +def server_url() -> str: + return f"http://127.0.0.1:{port()}" - Derived from ``port`` rather than configured, because it is not a choice anybody has a reason - to make: it is one more than the port they already set, and one setting that can be wrong is - better than two. - """ - return port() + 1 - - -def server_url(at: int = 0) -> str: - return f"http://127.0.0.1:{at or port()}" - -def server_up(at: int = 0) -> bool: +def server_up() -> bool: """True if a warm whisper-server answers on the loopback port.""" try: - with urllib.request.urlopen(f"{server_url(at)}/", timeout=0.5): + with urllib.request.urlopen(f"{server_url()}/", timeout=0.5): return True except (urllib.error.URLError, OSError): return False @@ -600,24 +590,18 @@ def server_up(at: int = 0) -> bool: _OWNERSHIP: dict[int, tuple[float, bool]] = {} -def ours(at: int = 0) -> bool: - """Is the thing listening on ``at`` a whisper-server, rather than whatever got there first. - - ``0`` means the main port, the same as it does to :func:`server_url` and :func:`server_up`. - It has to: this is called with whatever a caller was given, and a caller that was given the - default asked about port ZERO — where nothing is ever listening, so the answer was always no - and the daemon started every morning announcing that its own running server was unavailable. +def ours() -> bool: + """Is the thing listening on our port a whisper-server, rather than whatever got there first. - **Because the answer decides where recorded audio is sent.** Both ports are predictable — one - is a documented default, the other is one above it — and any local process can bind them first - and then receive every clip, and answer with text that gets typed at the cursor. A socket that - accepts a connection proves nothing about who is on the other end of it. + **Because the answer decides where recorded audio is sent.** The port is predictable, so any + local process can bind it first, receive every clip, and answer with text that gets typed at + the cursor. A socket that accepts a connection proves nothing about who is on the other end. So the port has to be held by a `whisper-server` process. Only one process can bind a port, so finding one there IS the answer. Cached for :data:`OWNERSHIP_SECONDS` because this sits on the partial path, which asks it about once a second, and `pgrep` is a process spawn. """ - at = at or port() + at = port() now = time.monotonic() cached = _OWNERSHIP.get(at) if cached is not None and now - cached[0] < OWNERSHIP_SECONDS: @@ -678,11 +662,12 @@ def server_answers() -> tuple[bool, str]: return False, str(error)[:120] -def serve_command(model: str = "", at: int = 0) -> list[str] | None: +def serve_command(model: str = "") -> list[str] | None: """The argv that starts a warm whisper-server, or ``None`` if it cannot be built. - Defaults to the big model on the main port — the server that writes the transcript you keep. - The partials pass their own small model and their own port; see :func:`whisper.partial_model`. + ONE server, and it answers both the live passes and the final transcription. There used to be + a second one holding a small model for the partials; it was retired when the live pass began + typing punctuation, because the marks it chose were the marks the operator kept. """ binary = resolve_bin("whisper-server") model = model or whisper.model() @@ -695,7 +680,7 @@ def serve_command(model: str = "", at: int = 0) -> list[str] | None: "--host", "127.0.0.1", "--port", - str(at or port()), + str(port()), "-t", whisper.threads(), "--convert", # let the server transcode anything ffmpeg reads, not just wav @@ -724,7 +709,7 @@ def serve_command(model: str = "", at: int = 0) -> list[str] | None: ] -def start_server(*, wait: float = 60.0, model: str = "", at: int = 0) -> bool: +def start_server(*, wait: float = 60.0) -> bool: """Spawn the warm whisper-server if it is not already up; block until it answers. Loading large-v3-turbo takes a few seconds, which is exactly the cost we are paying ONCE here @@ -749,11 +734,11 @@ def start_server(*, wait: float = 60.0, model: str = "", at: int = 0) -> bool: place recorded voice lives — whisper-server deletes its converted copy when it is done, so the "empty between sentences" promise there still holds. """ - if server_up(at): + if server_up(): # Adopted only if a whisper-server is what is holding the port — see :func:`ours`. Anything # else answering there would be handed recorded audio and believed about what was said. - return ours(at) - cmd = serve_command(model, at) + return ours() + cmd = serve_command() if cmd is None: return False try: @@ -769,37 +754,18 @@ def start_server(*, wait: float = 60.0, model: str = "", at: int = 0) -> bool: return False deadline = time.time() + wait while time.time() < deadline: - if server_up(at): + if server_up(): return True time.sleep(0.1) return False -def start_partial_server(*, wait: float = 60.0) -> bool: - """Start the small server that answers the live partials. False when there is no small model. - - False is not a failure: the partials fall back to the big server, which is what they did before - this existed. See :func:`whisper.partial_model`. - """ - model = whisper.partial_model() - if not model: - # AND STOP THE ONE A PREVIOUS RUN LEFT, which is not tidiness. `partial_at` sends the - # partials to whatever is answering on the live port, so a small server still up from - # before `livePass` changed keeps on answering them — the setting would appear to do - # nothing until the machine was restarted, and 488 MB would sit there while it did. - # Only one that is up AND ours: this must never be a blind pkill on a port we do not hold. - if ours(partial_port()) and server_up(partial_port()): - stop_server(partial_port()) - return False - return start_server(wait=wait, model=model, at=partial_port()) - - -def stop_server(at: int = 0) -> int: +def stop_server() -> int: """Stop the warm whisper-server this install started. Returns how many were stopped. - ``at`` stops ONE of them. The bounce that heals a wedged main server passes it, because taking - the live server down as collateral and never bringing it back left every partial after the - first bounce on the big model, silently, until the next daemon restart. + The port ABOVE ours is swept too, and it is not a second server of ours: an older MurmurFlow + ran a small model there for the live pass, and a version that no longer starts one must still + stop the one it finds, or ~488 MB stays resident until the machine is next restarted. ``start_server`` detaches it with ``start_new_session=True`` so it outlives the listener, which is the whole point while dictation is installed — and a leak the moment it is not: 1.8 GB @@ -807,7 +773,7 @@ def stop_server(at: int = 0) -> int: OUR two ports, because a whisper-server on any other port belongs to somebody else. """ stopped = 0 - for which in (port(), partial_port()) if at == 0 else (at,): + for which in (port(), port() + 1): try: found = subprocess.run( ["pgrep", "-f", f"whisper-server.*--port {which}"], @@ -926,7 +892,7 @@ def _confidence(payload: str) -> tuple[str, float, str]: return text, float(raw) if isinstance(raw, (int, float)) else 1.0, spoken -def transcribe_warm(wav: Path, *, timeout: float = 60.0, language: str = "", at: int = 0) -> Heard: +def transcribe_warm(wav: Path, *, timeout: float = 60.0, language: str = "") -> Heard: """Transcribe via the warm server; empty text if it is not up or errors. Never raises. Reuses :mod:`whisper`'s language and vocabulary decisions, so every surface that transcribes @@ -940,9 +906,6 @@ def transcribe_warm(wav: Path, *, timeout: float = 60.0, language: str = "", at: on an M4 Pro — and a clip does not change language halfway through, so the partials after the first pin themselves to what the first one heard. See :func:`_stream_loop`. - ``at`` picks WHICH warm server. Default is the big one that writes the transcript you keep; the - partials ask the small one on :func:`partial_port`, which answers ~5x faster and, being a - different process, is never the reason the final transcription is queued. """ if not wav.is_file(): return Heard("") @@ -957,7 +920,7 @@ def transcribe_warm(wav: Path, *, timeout: float = 60.0, language: str = "", at: except OSError: return Heard("") request = urllib.request.Request( - f"{server_url(at)}/inference", data=body, headers={"Content-Type": content_type} + f"{server_url()}/inference", data=body, headers={"Content-Type": content_type} ) try: with urllib.request.urlopen(request, timeout=timeout) as response: @@ -1580,12 +1543,10 @@ def _inject(text: str) -> tuple[bool, str, str]: #: :data:`STREAM_HOLDBACK_WORDS`), so waiting too little does not make the first words arrive #: sooner — it spends a whole pass to find that out. #: -#: One second, and the number moved when the partials got their own small server. While one pass -#: cost 2.2s, waiting less did not make the words arrive sooner — it spent a whole pass to find out -#: there was nothing to commit yet, so 1.5 and 1.0 landed the first words at the same moment and -#: 1.5 landed more of them. A pass now costs ~0.4s (see :func:`whisper.partial_model`), so the -#: cheap first look is worth taking: measured end to end on the same 10.7s clip, 1.0 puts the first -#: word on screen at 1.37s against 1.88s. +#: One second. While a pass cost 2.2s, waiting less did not make the words arrive sooner — it +#: spent a whole pass to find out there was nothing to commit yet, so 1.5 and 1.0 landed the first +#: words at the same moment and 1.5 landed more of them. A pass costs ~1.5s on the big model under +#: 30s of audio (measured), so the cheap first look is worth taking. STREAM_FIRST_SECONDS = 1.0 #: Minimum gap between passes, measured from the START of the previous one. A pass costs about the @@ -1616,7 +1577,7 @@ def _inject(text: str) -> tuple[bool, str, str]: #: because whisper-server answers one request at a time: a partial still decoding when the key is #: released is time the FINAL transcription spends queued behind it, i.e. straight onto the latency #: this whole product is about. -STREAM_TIMEOUT = 10.0 +STREAM_TIMEOUT = 30.0 @dataclass @@ -1638,12 +1599,6 @@ class Stream: blank: int = 0 #: Chunks actually pasted at the cursor. typed: int = 0 - #: The last pass that cleared every gate, and the seconds of audio it had read — what lets - #: `finish` skip its own transcription (see :func:`whole_clip_read`). ONE attribute holding - #: both, because they are read from another thread while this one is still writing them: as - #: two fields a reader could take the new length beside the older transcript, decide the clip - #: was fully read and drop whatever was said in between. One store, one read, no window. - read: tuple[Heard, float] | None = None #: In-flight streams, keyed by the wav they are transcribing. A dict and not an attribute on @@ -1878,7 +1833,7 @@ def missing_mark(pasted: str, settled: str) -> str: return mark.group() -def _partial(live: Path, snapshot: Path, language: str = "") -> Heard: +def _partial(live: Path, snapshot: Path) -> Heard: """Transcribe the audio captured SO FAR. Empty text if there is nothing worth reading. Never raises. The live wav is COPIED and the copy is what gets read, for two reasons that are both about not @@ -1901,12 +1856,11 @@ def _partial(live: Path, snapshot: Path, language: str = "") -> Heard: return Heard("") try: repair_wav(snapshot) + trim_trailing_quiet(snapshot) captured = audio_seconds(snapshot) if captured < MIN_CLIP_SECONDS or peak_dbfs(snapshot) < quiet_floor(): return Heard("") - heard = transcribe_warm( - snapshot, timeout=STREAM_TIMEOUT, language=language, at=partial_at() - ) + heard = transcribe_warm(snapshot, timeout=STREAM_TIMEOUT) if not heard.text or heard.confidence < SPEECH_CONFIDENCE or is_hallucination(heard.text): return Heard("") # AND THE LANGUAGE GATE, which the final transcription has always had and this did not. @@ -1924,16 +1878,6 @@ def _partial(live: Path, snapshot: Path, language: str = "") -> Heard: snapshot.unlink(missing_ok=True) -def partial_at() -> int: - """The port the partials should ask: the small server when it is up, else the big one. - - Checked per pass rather than once, because the small server can be missing at start-up and - appear later, or die mid-afternoon. Falling back to the big server is slower and never wrong, - which is the right way round for something that types. - """ - return partial_port() if ours(partial_port()) and server_up(partial_port()) else 0 - - def stream_note(stream: Stream | None) -> str: """What streaming did, for the daemon log. ``""`` when it never ran. @@ -1990,25 +1934,20 @@ def _stream_loop(rec: Recording, stream: Stream) -> None: """The streaming thread: decode what has been said, paste what is settled, repeat.""" snapshot = rec.wav.with_name(f"{rec.wav.stem}-partial.wav") previous = "" - #: The language the first accepted pass heard, pinned onto every pass after it. Worth ~0.75s of - #: every ~2.2s pass, because `auto` runs a whole extra encoder pass to answer a question whose - #: answer cannot change halfway through one clip. Only ever set from a pass that already - #: cleared the confidence gate in `_partial`, and only to a language the user says they speak - #: when they have said — a partial pinned to a language nobody in the room is speaking would - #: come back as fluent translation, and two of those in a row would agree and get typed. - #: The FINAL transcription is never pinned, so the language gate still judges the real clip. - pinned = "" - spoken = spoken_languages() + # THE LANGUAGE IS NOT PINNED, and the ~0.75s a pin saved is the price of the gate that + # catches invented speech. `auto` runs an extra encoder pass to answer "which language is + # this", and the answer is what `_partial` refuses on — so pinning it to what the FIRST pass + # heard made every pass after that one report the pinned language by construction, whatever it + # had actually decoded. The gate was blind for the whole clip after its first second, and a + # partial is PASTED. Reported as "a lot of gibberish in a different language ... I don't know + # how it got there". The final transcription was never pinned for exactly this reason; a pass + # that types is owed the same. if stream.done.wait(STREAM_FIRST_SECONDS): return while not stream.done.is_set(): started = time.monotonic() stream.passes += 1 - # BEFORE the pass, and off the file SIZE: every byte past the header is a sample, and the - # header of a file ffmpeg is still writing says the clip is empty. Under-reporting here is - # safe — it only makes `whole_clip_read` refuse — where over-reporting would drop speech. - covered = seconds_on_disk(rec.wav) - found = _partial(rec.wav, snapshot, pinned) + found = _partial(rec.wav, snapshot) heard = found.text # A pass that read nothing — still below the quiet floor, a hallucination thrown out, the # server busy — is not a pass that DISAGREED. Keeping the last real reading as `previous` @@ -2018,9 +1957,6 @@ def _stream_loop(rec: Recording, stream: Stream) -> None: stream.blank += 1 stream.done.wait(max(0.0, STREAM_EVERY_SECONDS - (time.monotonic() - started))) continue - if not pinned and found.language and (not spoken or found.language in spoken): - pinned = found.language - stream.read = (found, covered) settled = stable_prefix(previous, heard) previous = heard chunk = stream_tail(stream.text, settled) if settled else "" @@ -2047,41 +1983,6 @@ def _stream_loop(rec: Recording, stream: Stream) -> None: stream.done.wait(max(0.0, STREAM_EVERY_SECONDS - (time.monotonic() - started))) -def whole_clip_read(stream: Stream | None, wav: Path, captured: float) -> Heard | None: - """The live pass's own transcript, when it already read the whole clip. Else ``None``. - - **This is the 8 seconds at the end of a long dictation.** Measured from the operator's own log: - a 73.7s clip spent 3.2s in the final transcription, a 69.0s clip 6.0s, a 79.1s clip 7.8s — and - the row for each says ``→ streamed``, meaning that pass produced NOTHING that was not already - on screen. Worse than wasted: he had stopped, read his sentence, sent it, and the tail then - landed eight seconds later in whatever he was looking at by then ("it just added a lot of - gibberish ... after I already sent the message"). - - A partial only ever reads the audio captured SO FAR, so the question is what was said after the - last one. If that is silence, the last partial read the whole clip and its transcript is the - transcript — the same model, the same audio, the same gates. `covered` is measured from the - file's SIZE before the pass rather than after, so it under-reports and this refuses more often - than it strictly must, which is the right way for a shortcut to be wrong. - - ``AHEAD_SECONDS`` is not a tolerance for missing speech: below it there is not enough audio for - :func:`tail_dbfs` to have an opinion, and a quarter of a second cannot hold a word. - """ - if stream is None or stream.read is None: - return None - heard, covered = stream.read - if not heard.text: - return None - ahead = max(0.0, captured - covered) - if ahead > AHEAD_SECONDS and tail_dbfs(wav, ahead) >= quiet_floor(): - return None # something was said after the last pass looked - return heard - - -#: How much unread audio at the end of a clip is too little to hold a word. See -#: :func:`whole_clip_read`. -AHEAD_SECONDS = 0.25 - - def stop_streaming(wav: Path) -> Stream | None: """Stop the stream for ``wav`` starting any new pass. Returns it, for :func:`streamed`. @@ -2248,17 +2149,66 @@ def _peak_dbfs(frames: bytes) -> float: return 20 * math.log10(min(peak, 32768) / 32768.0) -def seconds_on_disk(wav: Path) -> float: - """How many seconds of audio a clip STILL BEING RECORDED holds. ``0.0`` if it cannot be read. +#: Silence left on the end of a clip, in blocks this long, is cut before anything transcribes it. +#: Small enough to find the end of the last word closely, large enough that one loud sample of +#: keyboard noise does not hold a whole minute of nothing in place. +TRIM_BLOCK_SECONDS = 0.2 + +#: ...and this much is kept after the last block that had sound in it. A word's decay is part of +#: the word, and whisper reads a hard cut at the end of a syllable as a different syllable. +TRIM_KEEP_SECONDS = 0.4 + + +def trim_trailing_quiet(wav: Path) -> bool: + """Cut silence off the end of a clip. True if anything was cut. Never raises. + + **Whisper invents words when it is handed audio with nothing in it**, and this is the fix for + it — measured, after two that were not. The same 12 seconds of speech, three ways: - The same arithmetic as :func:`tail_dbfs` and for the same reason: while ffmpeg is appending, - the RIFF header still says the file is empty, so :func:`audio_seconds` answers 0 for a clip - that is minutes long. + speech alone "...but just in this text box," + + 20s of digital silence "...but just in this text box, Thank you." + + 20s of faint room noise "...but just in this text box.." + + So the invention is not a property of the speech, the model or the prompt. It is the silence, + and the cure is not to hand it over. Reported as "a lot of gibberish in a different language", + which was romanised Japanese appended to a real English sentence, and it reached the cursor + because he had not tapped to stop — so the clip ended with fifteen seconds of nothing. + + What was tried first and REFUSED, both measured: no word list can catch it (this file already + said so — whisper answers silence in a different invented language each time), and whisper's + own per-segment `no_speech_prob`/`avg_logprob` do not either. The invented " Thank you." came + back at `no_speech_prob` **0.000** and `avg_logprob` -0.28, sitting among real speech at -0.05 + to -0.12: confidently wrong, with no threshold between them that does not also cut real quiet + speech. + + Nothing is cut when the clip is quiet all the way through — that is a clip with no speech in + it, and the level gates in :func:`finish` are what should judge it and say so. """ + block = int(TRIM_BLOCK_SECONDS * BYTES_PER_SECOND) try: - return max(0.0, (wav.stat().st_size - 44) / BYTES_PER_SECOND) + size = wav.stat().st_size + with wav.open("rb") as handle: + handle.seek(44) + audio = handle.read() except OSError: - return 0.0 + return False + floor = quiet_floor() + last = -1 + for index in range(len(audio) // block): + if _peak_dbfs(audio[index * block : (index + 1) * block]) >= floor: + last = index + if last < 0: + return False # nothing above the floor anywhere: not ours to judge + keep = 44 + (last + 1) * block + int(TRIM_KEEP_SECONDS * BYTES_PER_SECOND) + if keep >= size: + return False + try: + with wav.open("r+b") as handle: + handle.truncate(keep) + except OSError: + return False + repair_wav(wav) # the RIFF header still claims the length it had before the cut + return True def tail_dbfs(wav: Path, seconds: float) -> float: @@ -2409,6 +2359,10 @@ def finish(rec: Recording | None = None, *, paste: bool = True) -> Result: # surface goes through this function, so cueing at the seam is also the only way they stay in # step. After the two ways this can still be a non-event, so nothing chimes at a brushed key. cue_done() + # BEFORE the level gates and the transcribe: silence on the end is what whisper invents into, + # and a clip closed by the silence watchdog ends with fifteen seconds of it. See + # :func:`trim_trailing_quiet`. + trim_trailing_quiet(wav) level = peak_dbfs(wav) captured = audio_seconds(wav) @@ -2450,10 +2404,7 @@ def retire() -> None: started = time.monotonic() try: - # The live pass may already have read this exact audio with this exact model. If nothing - # was said after it looked, transcribing again buys a copy of what is on screen for the - # seconds the operator is standing there waiting — see :func:`whole_clip_read`. - heard = whole_clip_read(stream, wav, captured) or transcribe(wav) + heard = transcribe(wav) raw = heard.text finally: retire() @@ -3007,10 +2958,6 @@ def emit(line: str) -> None: emit(f"whisper warm on :{port()}") else: emit("whisper-server unavailable — falling back to cold whisper-cli (~1s slower)") - # THE SECOND SERVER, and it is what makes the live words live. On a THREAD, because loading it - # is seconds during which nobody can dictate — the big server is already up by here, so the - # first sentence works whether or not this has finished, only more slowly. - threading.Thread(target=start_partial_server, daemon=True).start() # Streaming is on unless something PHYSICALLY stops it, and the daemon names which — a feature # that is silently absent is the worst kind, because there is nothing anywhere to read. if not double_tap_mode(): @@ -3020,16 +2967,8 @@ def emit(line: str) -> None: ) elif not warm_expected: emit("[!] no warm server answered — partials are warm-only, so the words arrive at the end") - elif whisper.partial_model(): - emit( - f"the words arrive while you talk ({Path(whisper.partial_model()).stem} on the live pass)" - ) else: - emit( - f"the words arrive while you talk ({Path(whisper.model()).stem} on the live pass, so " - "they come in bigger lumps the longer you talk). `murmurflow config set livePass small`" - " is the fast one, with thinner punctuation" - ) + emit(f"the words arrive while you talk ({Path(whisper.model()).stem} on the live pass)") #: Consecutive clips that took the cold path while a warm server was supposed to be answering. cold_streak = [0] @@ -3053,10 +2992,7 @@ def watch_warm(warm: bool | None) -> None: cold_streak[0] = 0 def bounce() -> None: - # THIS PORT ONLY. Bouncing both took the live server down as collateral and never - # brought it back, so every partial after the first bounce ran on the big model, - # silently, until the next daemon restart. - stop_server(port()) + stop_server() emit("[!] the warm whisper-server stopped answering — restarting it") emit( f"whisper warm again on :{port()}" diff --git a/murmurflow/whisper.py b/murmurflow/whisper.py index 20e776f..7576874 100644 --- a/murmurflow/whisper.py +++ b/murmurflow/whisper.py @@ -43,30 +43,6 @@ "ggml-base.en.bin", ) -# The model that answers the LIVE PARTIALS while you are still talking, best first — and it is a -# SMALL one on purpose. Measured on an M4 Pro against a warm server, same clip, same prompt: -# large-v3-turbo answers a partial in 2.2-2.4s, `ggml-small` in 0.35-0.44s, `ggml-base` in 0.15-0.23s. -# -# The speed is only half of why this exists. whisper-server answers ONE request at a time, so a -# partial still decoding when you stop talking is time the FINAL transcription spends queued behind -# it — measured live at 1-2.3s added to the end of every sentence, which is exactly the moment -# somebody is waiting. A model that answers in 0.4s cannot cost more than 0.4s of that. -# -# `small` and not `base`, and that is a measurement rather than caution: on the same German clip -# base typed "das" where the speaker said "dass" and dropped a plural, while small returned -# character-for-character what large-v3-turbo did. A partial is PASTED and there is no un-paste, so -# a model that quietly rewords is not cheaper, it is wrong. `.en` variants rank below their -# multilingual twins for the same reason as in MODEL_PREFERENCE. -PARTIAL_PREFERENCE: tuple[str, ...] = ( - "ggml-small.bin", - "ggml-small.en.bin", - "ggml-base.bin", - "ggml-base.en.bin", -) - -#: What `murmurflow setup` fetches for the partials. ~488 MB, next to the ~1.6 GB main model. -DEFAULT_PARTIAL_MODEL = "ggml-small.bin" - # Where a ggml model may live, in search order. `~/.murmurflow/models/` is ours; the others are what # `brew install whisper-cpp` lays down. _MODEL_DIRS: tuple[str, ...] = ( @@ -118,43 +94,6 @@ def model() -> str: return "" -def partial_model() -> str: - """Path to a SMALL model for the live partials, or ``''`` when none is installed. - - Empty is a working answer and not a failure: the partials then go to the same warm server the - final transcription uses, exactly as they did before this existed. Slower, and never wrong. - - Deliberately ignores the ``model`` config override, which names the model that writes what you - keep. Pinning that to a small model is a choice about the transcript; it must not also silently - become the choice about the partials, and vice versa. - - **The default is now the BIG model, i.e. no small model at all, and that is a measurement.** - Once a live pass could type PUNCTUATION (`dictate.missing_mark`), the model answering the live - pass stopped deciding only the words and started deciding the marks the operator keeps — and - the two models are not close there. Replayed through the whole stream loop on one real 38s - clip, against the big model's own whole-clip transcript: - - live = small 13.9% different "...in the end like when I just stopped my control it - just added a lot of gibberish I'm not sure and that came" - live = big 11.4% different "...in the end, like when I just stopped my control, it - just added a lot of gibberish, I'm not sure. And that - came after a few seconds, after I already sent..." - - The percentages understate it; the sentences are the finding. The operator's call, given both - ("I would rather have it qualitatively high rather than faster"). ``livePass: "small"`` puts - the fast one back, and the cost of the default is real: a pass over a long clip costs seconds - rather than ~1, so the words arrive in bigger lumps the longer you talk. - """ - if str(config.load().get("livePass", "big")).strip().lower() != "small": - return "" - for name in PARTIAL_PREFERENCE: - for directory in _search_dirs(): - candidate = directory / name - if candidate.is_file(): - return str(candidate) - return "" - - def openai_model_name() -> str: """The ``--model`` NAME for an openai-whisper-style CLI, which downloads its own weights. diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..57d1daa --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,45 @@ +"""Nothing in this suite may reach the real keyboard, clipboard or microphone. + +**This file exists because the suite typed into the operator's screen for a whole day.** Two tests +drive `_stream_loop` with a fixed `Heard` and monkeypatch `dictate._inject` — the CLIPBOARD path — +believing that was the way out to the machine. It is not the only one: :func:`dictate.place` tries +`platforms.type_text` FIRST, and that is a real ``CGEventKeyboardSetUnicodeString`` with nothing in +front of it. So every run of the suite typed both fixtures into whatever window had focus, back to +back and with no space between them: + + hello there my friend and also yougokigen you desu ne totemo ii tenki + +Reported as "I keep getting this same random paste everywhere, even tho I'm not using murmurflow", +which is exactly right: it was not MurmurFlow, it was MurmurFlow's tests, and the giveaway was that +the text was byte-identical every time — a hallucination is different every time, a fixture is not. + +A per-test monkeypatch cannot fix this class of bug, because the bug is a test that did not know it +had a second door. So the doors are shut here for EVERY test, autouse, and a test that wants to +watch one opens its own double over the top. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from murmurflow import platforms + + +@pytest.fixture(autouse=True) +def _never_type_on_the_real_keyboard(monkeypatch): + """Shut the door that was open: ``platforms.type_text`` and nothing else. + + ONE door, deliberately. The clipboard and the recorder have tests that drive them on purpose, + with their own doubles over the OS calls inside; stubbing those here would replace the thing + under test with this stub and the assertions would pass against nothing — a suite that reports + green over a function it never ran is worse than the bug being fixed. + + ``""`` is what a keyboard that typed the whole chunk returns, so :func:`dictate.place` reports + success and never falls through to the clipboard behind it. + """ + monkeypatch.setattr(platforms, "type_text", lambda _text: "") diff --git a/tests/test_murmurflow.py b/tests/test_murmurflow.py index 7d6f231..0064985 100644 --- a/tests/test_murmurflow.py +++ b/tests/test_murmurflow.py @@ -27,7 +27,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from murmurflow import cli, config, dictate, service, whisper +from murmurflow import cli, config, dictate, platforms, service, whisper @pytest.fixture(autouse=True) @@ -1140,7 +1140,7 @@ def start(self): monkeypatch.setattr(dictate, "resolve_input", lambda: ("0", "a mic")) monkeypatch.setattr(dictate, "paused", lambda: (False, "")) monkeypatch.setattr(dictate, "start_server", lambda **_k: bool(starts.append(1)) or warm_starts) - monkeypatch.setattr(dictate, "stop_server", lambda at=0: bool(stops.append(at)) or 1) + monkeypatch.setattr(dictate, "stop_server", lambda: bool(stops.append(1)) or 1) monkeypatch.setattr(dictate, "preroll_claim", lambda: dictate.Recording(1, Path("x.wav"), 0.0)) monkeypatch.setattr(dictate, "stream_start", lambda _rec: None) # not what this drives # The ready cue waits for the device to hand over its first buffer, and `_Inline` above runs @@ -1178,10 +1178,7 @@ def test_a_warm_server_that_stopped_answering_is_restarted(monkeypatch): the desk came back as Japanese and was typed. Nothing on screen said the fast path was gone. """ starts, stops = _drive_listener(monkeypatch, [_clip(False), _clip(False), _clip(False)]) - # Bounced once, at the second cold clip — and ONLY the big server's port. Taking the live - # server with it left every later partial on the big model until the next daemon restart. - assert stops == [dictate.port()] - assert dictate.partial_port() not in stops + assert len(stops) == 1 # bounced once, at the second cold clip, and not at the first assert len(starts) == 2 # the one at daemon start, and the one that brought it back @@ -1595,6 +1592,27 @@ def test_the_shipped_ceiling_leaves_months_of_real_use_in_the_file(): assert dictate.LOG_KEEP_BYTES < dictate.LOG_MAX_BYTES +def test_the_suite_can_never_type_on_the_real_keyboard(): + """The suite typed into the operator's screen for a whole day. See `tests/conftest.py`. + + Two tests drive `_stream_loop` with a fixed `Heard` and monkeypatch `dictate._inject`, believing + the clipboard was the way out to the machine. `dictate.place` tries `platforms.type_text` FIRST, + and that is a real CGEvent with nothing in front of it — so both fixtures were typed into + whatever window had focus, back to back and with no space between them: + + hello there my friend and also yougokigen you desu ne totemo ii tenki + + Reported as "I keep getting this same random paste everywhere, even tho I'm not using + murmurflow". The giveaway was that it was byte-identical every time: a hallucination is + different every time, a fixture is not. + """ + assert platforms.type_text.__module__ != "murmurflow.platforms", ( + "tests/conftest.py no longer shuts the keyboard door — the suite can type on the real " + "machine again" + ) + assert platforms.type_text("anything at all") == "" + + # --- streaming --------------------------------------------------------------------------------- @@ -1681,37 +1699,6 @@ def test_a_forgotten_key_still_gets_its_words(monkeypatch): assert starts == [1] # the daemon started its server and then rescued the clip on its own -def test_the_end_is_not_transcribed_twice_when_the_live_pass_already_read_it(monkeypatch, tmp_path): - """The 8 seconds at the end of a long dictation, and the tail that landed after he had sent. - - From the operator's own log: a 73.7s clip spent 3.2s in the final transcription, 69.0s spent - 6.0s, 79.1s spent 7.8s — and each of those rows says `→ streamed`, meaning that pass produced - nothing that was not already on screen. He had stopped, read his sentence and sent it; the - tail then arrived eight seconds later in whatever he was looking at by then. - """ - clip = tmp_path / "c.wav" - clip.write_bytes(b"\x00" * (44 + dictate.BYTES_PER_SECOND * 30)) - live = dictate.Heard("what the live pass already read", 0.99, "en", warm=True) - stream = dictate.Stream(threading.Event()) - - # Nothing read yet: there is nothing to reuse. - assert dictate.whole_clip_read(stream, clip, 30.0) is None - assert dictate.whole_clip_read(None, clip, 30.0) is None - - # It read 29.9 of the 30 seconds — too little left to hold a word. - stream.read = (live, 29.9) - assert dictate.whole_clip_read(stream, clip, 30.0) is live - - # It read 20 of 30, and the ten seconds it never saw are SILENT: still the whole transcript. - stream.read = (live, 20.0) - monkeypatch.setattr(dictate, "tail_dbfs", lambda _wav, _seconds: -90.0) - assert dictate.whole_clip_read(stream, clip, 30.0) is live - - # Same ten seconds, but somebody was talking in them. Now the shortcut must refuse. - monkeypatch.setattr(dictate, "tail_dbfs", lambda _wav, _seconds: -12.0) - assert dictate.whole_clip_read(stream, clip, 30.0) is None - - def test_the_last_seconds_of_a_clip_still_being_recorded_can_be_read(tmp_path): """`wave` cannot answer this and that is the whole reason it exists. @@ -2063,17 +2050,18 @@ def test_a_lent_trigger_does_not_open_the_microphone_early(monkeypatch): assert opened == ["mic"] -def test_a_partial_pins_the_language_the_first_pass_heard(monkeypatch, tmp_path): - """Detecting the language is 0.75s of every 2.2s pass, and one clip does not change language. +def test_a_partial_never_pins_the_language(monkeypatch, tmp_path): + """The pin saved ~0.75s a pass and cost the gate that refuses invented speech. - The pin is only ever taken from a pass that already cleared the confidence gate, and the FINAL - transcription is never pinned — so the "is that one of yours" gate still judges the real clip. + whisper-server reports back whatever language it was TOLD to decode, so a pass pinned to what + the first second heard reported that language by construction, whatever it had actually + decoded. The gate was blind for the rest of the clip — and a partial is PASTED. """ config.set_value("languages", ["de", "en"]) - asked: list[str] = [] + asked: list[tuple] = [] - def _partial(_live, _snapshot, language=""): - asked.append(language) + def _partial(_live, _snapshot, *args): + asked.append(args) return dictate.Heard("hello there my friend and also you", 0.99, "en", warm=True) monkeypatch.setattr(dictate, "_partial", _partial) @@ -2092,8 +2080,8 @@ def _partial(_live, _snapshot, language=""): time.sleep(0.01) stream.done.set() thread.join(timeout=2) - assert asked[0] == "" # the first pass has to detect it - assert asked[1] == "en" and asked[2] == "en" # and nothing after it pays for that again + assert len(asked) >= 3 + assert all(extra == () for extra in asked) # every pass detects it for itself, forever def test_a_language_you_do_not_speak_is_never_pinned(monkeypatch, tmp_path): @@ -2130,46 +2118,74 @@ def _partial(_live, _snapshot, language=""): # --- the live pass has its own small model ---------------------------------------------------- -def test_the_live_model_is_a_small_one_and_never_the_transcript_model(tmp_path, monkeypatch): - """`model` is the transcript you keep; the live pass is a different job with a different cost. +def test_one_model_does_both_jobs(tmp_path, monkeypatch): + """There was a second, small model that typed live while the big one wrote the final. - They must not share a knob: pinning the transcript to a small model is a decision about - accuracy, and it must not silently also become the decision about the live pass, or vice versa. + It was retired when the live pass began typing PUNCTUATION rather than only words: the marks + it chose are the marks the operator keeps, and there the two models are not close. One model, + one server, one queue. """ models = config.home_root() / "models" models.mkdir(parents=True, exist_ok=True) (models / "ggml-large-v3-turbo.bin").write_bytes(b"x") - (models / "ggml-base.bin").write_bytes(b"x") - (models / "ggml-small.bin").write_bytes(b"x") - # THE DEFAULT IS THE BIG MODEL, i.e. no live model at all — the live pass types the - # punctuation the operator keeps, and the two models are not close there. - assert whisper.partial_model() == "" - config.set_value("livePass", "small") - assert whisper.partial_model().endswith("ggml-small.bin") # small beats base: measured German - (models / "ggml-small.bin").unlink() - assert whisper.partial_model().endswith("ggml-base.bin") # base is the fallback below small - assert whisper.model().endswith("ggml-large-v3-turbo.bin") # and the transcript is unmoved - - # An explicit `model` override still names the transcript model, and only that one. + assert whisper.model().endswith("ggml-large-v3-turbo.bin") + assert not hasattr(whisper, "partial_model") + assert not hasattr(dictate, "partial_port") + assert not hasattr(dictate, "partial_at") + assert not hasattr(dictate, "start_partial_server") + + # An explicit `model` override names it, and there is nothing else for it to collide with. config.set_value("model", str(models / "ggml-base.bin")) + (models / "ggml-base.bin").write_bytes(b"x") assert whisper.model().endswith("ggml-base.bin") - assert whisper.partial_model().endswith("ggml-base.bin") -def test_the_live_server_gets_its_own_model_and_its_own_port(monkeypatch): - """Its own PROCESS is the point, not just its own model: whisper-server answers one request at - a time, so a partial sharing the queue is time the FINAL transcription spends waiting. +def test_silence_on_the_end_of_a_clip_is_cut_before_whisper_can_invent_into_it(tmp_path): + """Whisper invents words when it is handed audio with nothing in it. Measured, three ways: + + speech alone "...but just in this text box," + + 20s of digital silence "...but just in this text box, Thank you." + + 20s of faint room noise "...but just in this text box.." + + So the invention is the silence, not the speech, the model or the prompt. Reported as "a lot + of gibberish in a different language" — romanised Japanese appended to a real English sentence + — and it reached the cursor because the clip had ended with fifteen seconds of nothing. """ - monkeypatch.setattr(dictate, "resolve_bin", lambda _n: "/usr/bin/whisper-server") - config.set_value("port", 8479) - command = dictate.serve_command("/models/ggml-small.bin", dictate.partial_port()) - assert command is not None - assert "/models/ggml-small.bin" in command - assert "8480" in command - assert dictate.partial_port() == 8480 + rate = dictate.SAMPLE_RATE + + def clip(name, tail_seconds): + path = tmp_path / name + with wave.open(str(path), "wb") as handle: + handle.setnchannels(1) + handle.setsampwidth(2) + handle.setframerate(rate) + tone = b"".join(struct.pack("