From e97f93234b1c9b720a45e87d0240758e3f48f0b5 Mon Sep 17 00:00:00 2001 From: hannesreinsch Date: Wed, 9 Sep 2026 15:17:15 +0200 Subject: [PATCH 1/2] fix(stream): the mark lands once a later word confirms it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported as "there was a break before, but no punctuation", and it is the cost the last commit named out loud rather than a new defect. A mark rides on the word in front of it, and `stable_prefix` will not type the mark on a word still touching the end of the audio — a pause is how whisper decides a sentence ended, and it takes that decision back the moment the speaker carries on. So the word landed bare and nothing could ever put the mark on afterwards: the word is on screen and there is no un-type. A LATER pass answers the question the earlier one could not. If the word carrying the mark is no longer at the end of the transcript — real speech follows it, and whisper STILL ends the sentence there — the mark was decided WITH the following audio, which is the same test every other word passes before it is typed. `missing_mark` returns it, joined to its word with no space in front, and only ever in the same breath as the word that proves it. So it can never be the lone full stop of #55: no confirming word, no mark. `_reached` is split out of `stream_tail` because both now ask the same question about the same two sequences — where does the screen END inside this transcript — and two answers to that would drift apart word by word. Driven through the reported sequence, the streamed text is now identical to the whole-clip transcript, punctuation included: I did not say the three times. I did, however, say really three times. `_drive_stream` in the tests is the loop's four calls in its own order, so the three earlier reports are replayed against it too and none of them regressed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01P7oJz8M9QzsdJimoM318cj --- murmurflow/dictate.py | 55 +++++++++++++++++++++++--- tests/test_murmurflow.py | 84 +++++++++++++++++++++++++++++++--------- 2 files changed, 116 insertions(+), 23 deletions(-) diff --git a/murmurflow/dictate.py b/murmurflow/dictate.py index 06976e6..4ae9638 100644 --- a/murmurflow/dictate.py +++ b/murmurflow/dictate.py @@ -1790,13 +1790,22 @@ def stream_tail(pasted: str, final: str) -> str: if not already: return final words = final.split() - keys = [_key(word) for word in words] + return " ".join(words[_reached(already, [_key(word) for word in words]) :]) + + +def _reached(already: list[str], keys: list[str]) -> int: + """The index in ``keys`` just past everything that is already on screen. + + Split out of :func:`stream_tail` because :func:`missing_mark` asks the same question about the + same two sequences — where does the screen END inside this transcript — and two answers to that + would drift apart word by word. + """ if keys[: len(already)] == already: - return " ".join(words[len(already) :]) + return len(already) matcher = difflib.SequenceMatcher(a=already, b=keys, autojunk=False) matched = [block for block in matcher.get_matching_blocks() if block.size] if not matched: # nothing corresponds: trust the count, lose nothing - return " ".join(words[len(already) :]) + return len(already) reached = matched[-1] # Words on screen PAST the alignment are ones the final pass said differently. The tail that # follows them is not new text, it is the same words again in the better model's wording, and @@ -1804,7 +1813,40 @@ def stream_tail(pasted: str, final: str) -> str: # exactly what one reworded last word looks like ("the design" + "designs"). One dropped for # one left over: the rewording is skipped and anything genuinely beyond the screen still lands. reworded = len(already) - (reached.a + reached.size) - return " ".join(words[reached.b + reached.size + reworded :]) + return reached.b + reached.size + reworded + + +def missing_mark(pasted: str, settled: str) -> str: + """The mark that belongs directly after ``pasted``, once a later word has confirmed it. + + **This is the punctuation streaming used to lose, and it is the last of it.** A mark rides on + the word in front of it, and :func:`stable_prefix` will not type the mark on a word that is + still touching the end of the audio — a pause is how whisper decides a sentence ended, and it + takes that decision back the moment the speaker carries on. So the word lands bare, and + nothing could ever put the mark on afterwards: the word is already on screen and there is no + un-type. Reported as "there was a break before, but no punctuation". + + A LATER pass answers the question the earlier one could not. If the word carrying the mark is + no longer at the end of the transcript — real speech follows it, and whisper still ends the + sentence there — the mark is a decision made WITH the following audio, which is the same test + every other word passes before it is typed. It goes on with no space in front of it, joined to + the word it belongs to. + + Returns ``""`` unless a word after it has also settled, so this can never be the lone full stop + of :func:`end_mark`'s docstring: the mark is only ever typed in the same breath as the word + that proves it. + """ + already = [_key(word) for word in pasted.split()] + words = settled.split() + if not already or not words: + return "" + index = _reached(already, [_key(word) for word in words]) + if index <= 0 or index >= len(words): + return "" # nothing before it, or nothing after it to confirm it + mark = _TRAILING_MARK.search(words[index - 1]) + if not mark or pasted.rstrip().endswith(mark.group()): + return "" + return mark.group() def _partial(live: Path, snapshot: Path, language: str = "") -> Heard: @@ -1948,6 +1990,9 @@ def _stream_loop(rec: Recording, stream: Stream) -> None: settled = stable_prefix(previous, heard) previous = heard chunk = stream_tail(stream.text, settled) if settled else "" + # The mark on the word already at the end of the screen, now that a later word has + # settled behind it. Only ever together with that word — see :func:`missing_mark`. + mark = missing_mark(stream.text, settled) if chunk else "" if chunk: with _INJECT_LOCK: # Inside the lock, because `stop_streaming` sets this and then takes the lock: past @@ -1961,7 +2006,7 @@ def _stream_loop(rec: Recording, stream: Stream) -> None: # `stream.text` is literally what is on the screen, built from what LANDED rather # than from what was asked for — see :func:`place`. The leading space travels with # the chunk, so this is a concatenation and never a re-join. - landed = place(f" {chunk}" if stream.text else chunk) + landed = place(f"{mark} {chunk}" if stream.text else chunk) if landed: stream.text = f"{stream.text}{landed}".strip() stream.typed += 1 diff --git a/tests/test_murmurflow.py b/tests/test_murmurflow.py index 504c241..6ae7a0d 100644 --- a/tests/test_murmurflow.py +++ b/tests/test_murmurflow.py @@ -1695,6 +1695,57 @@ def test_the_microphone_closes_itself_when_the_second_tap_never_comes(monkeypatc assert dictate.MAX_CLIP_SECONDS == 600 +def _drive_stream(passes): + """What lands on screen when the live pass reads ``passes``, one after another. + + The same four calls `_stream_loop` makes, in the same order, so a sequence that broke a real + dictation can be replayed as a test. Kept beside the tests that use it rather than inside them: + four copies of the loop drift, and a copy that drifts stops testing the loop. + """ + screen = previous = "" + for heard in passes: + settled = dictate.stable_prefix(previous, heard) + previous = heard + chunk = dictate.stream_tail(screen, settled) if settled else "" + mark = dictate.missing_mark(screen, settled) if chunk else "" + if chunk: + screen = f"{screen}{mark} {chunk}".strip() if screen else chunk + return screen + + +def test_the_mark_lands_once_a_later_word_confirms_it(): + """Reported as "there was a break before, but no punctuation". + + A mark rides on the word in front of it, and that word is never typed with its mark while it + still touches the end of the audio — a pause is how whisper decides a sentence ended, and it + takes that back the moment the speaker carries on. So the word landed bare and nothing could + put the mark on afterwards. A later pass answers it: real speech follows and whisper STILL + ends the sentence there, so the mark goes on, joined to the word it belongs to. + """ + reference = "I did not say the three times. I did, however, say really three times." + screen = _drive_stream( + [ + "I did not say the three times", + "I did not say the three times", # the break + "I did not say the three times.", # whisper ends the sentence + "I did not say the three times.", + "I did not say the three times. I", # he carries on + "I did not say the three times. I did however say", + "I did not say the three times. I did, however, say really", + reference, + reference, + ] + ) + # ...plus the one thing only the key release can know: the mark that ends the clip. + landed = screen + dictate.end_mark(screen, reference) + assert landed == reference # streamed, and identical to the whole-clip transcript + # And the fixes it must not undo, driven through the same loop. + assert _drive_stream(["Could you please work on my", "Could you please work on my..."] * 2) == ( + "Could you please work on my" + ) + assert "The The" not in _drive_stream(["Well, yeah. The The The"] * 3) + + def test_a_pause_never_types_a_lone_full_stop_where_the_next_word_goes(): """Reported as "it puts a period instead of the word" after a short break. @@ -1704,24 +1755,21 @@ def test_a_pause_never_types_a_lone_full_stop_where_the_next_word_goes(): screen with no letters in it, so the next alignment read it as something the final pass had reworded and dropped a real word to pay for it. The word this ate, in the report, was "But". """ - screen, previous = "", "" - for heard in ( - "and then I ran the command", - "and then I ran the command", # the pause: the transcript stops growing - "and then I ran the command.", # whisper decides the sentence ended - "and then I ran the command.", - "and then I ran the command. But", # he speaks again - "and then I ran the command. But when I say", - "and then I ran the command. But when I say", - ): - settled = dictate.stable_prefix(previous, heard) - previous = heard - chunk = dictate.stream_tail(screen, settled) if settled else "" - if chunk: - screen = f"{screen} {chunk}".strip() if screen else chunk - assert " ." not in screen - assert "But" in screen - assert screen == "and then I ran the command But when I say" + screen = _drive_stream( + [ + "and then I ran the command", + "and then I ran the command", # the pause: the transcript stops growing + "and then I ran the command.", # whisper decides the sentence ended + "and then I ran the command.", + "and then I ran the command. But", # he speaks again + "and then I ran the command. But when I say", + "and then I ran the command. But when I say", + ] + ) + assert " ." not in screen # never a mark standing on its own + assert "But" in screen # and never a word paid to the alignment for one + # The full stop DOES land, because "But" settled behind it and confirmed it. + assert screen == "and then I ran the command. But when I say" def test_a_pause_does_not_put_a_full_stop_in_the_middle_of_the_sentence(): From 9a77bd1b43a841f4032ae8bcfc6ebdb44ef9c595 Mon Sep 17 00:00:00 2001 From: hannesreinsch Date: Wed, 9 Sep 2026 15:22:31 +0200 Subject: [PATCH 2/2] feat(dictate): the microphone closes when you stop talking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "We should close the microphone after not talking for 15 seconds or something like that. I don't know what is a good timing, you have to decide that yourself." So: fifteen seconds, and the number is the whole decision. Dictation apps that stop on silence sit around two to three, and two to three is WRONG HERE. The gesture is a TAP, not a held key, so nothing is telling the microphone you are still there; and this operator thinks mid-sentence — the pauses that produced every punctuation bug in this file were real. A clip cut at three seconds would have ended half of them mid-thought, with the rest of the sentence spoken into a closed microphone. That failure is worse than the one being fixed, because a forgotten microphone loses nothing and a truncated sentence loses the sentence. Fifteen is longer than any pause measured here and still closes a forgotten one in a quarter of a minute rather than two. `tail_dbfs` is the reader `wave` cannot be: while ffmpeg is appending, the RIFF header still holds the lengths it was born with — zero — so every header-respecting reader sees an empty file. Every byte past the header is a sample, so "the last fifteen seconds" is a seek from the END and nothing else. A clip too short to have been quiet that long returns 0.0, "no opinion", which is well above any floor: a fresh recording can never read as silence and close itself. `peak_dbfs` and it share one `_peak_dbfs`, so they cannot disagree about what quiet is. It rides the watchdog that was already there, so it is one loop with two reasons to fire and one `claim()` at the end of both. `silenceStop` moves it, 0 switches it off, and the 120s `maxHold` cap stays as the backstop for a microphone left open in a silent room the floor never quite reaches. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01P7oJz8M9QzsdJimoM318cj --- murmurflow/dictate.py | 78 +++++++++++++++++++++++++++++++++++++--- tests/test_murmurflow.py | 54 +++++++++++++++++++++++++++- 2 files changed, 126 insertions(+), 6 deletions(-) diff --git a/murmurflow/dictate.py b/murmurflow/dictate.py index 4ae9638..59c01fd 100644 --- a/murmurflow/dictate.py +++ b/murmurflow/dictate.py @@ -1043,6 +1043,20 @@ def strip_fillers(transcript: str) -> str: # real dictation measured on this machine is ~68s; `maxHold` moves it. AUTO_STOP_SECONDS = 120.0 +# How long the microphone stays open with nothing being said before the daemon closes it and types +# what it has. The operator asked for it and left the number to me ("I don't know what is a good +# timing, you have to decide that yourself"), so: fifteen seconds. +# +# Dictation apps that stop on silence sit around two to three, and two to three is WRONG HERE. The +# gesture is a tap, not a held key, so nothing is telling the microphone you are still there; and +# this operator thinks mid-sentence — the pauses that produced every punctuation bug in this file +# were real, and a clip cut at three seconds would have ended half of them mid-thought, with the +# rest of the sentence spoken into a closed microphone. That failure is worse than the one being +# fixed, because a forgotten microphone loses nothing and a truncated sentence loses the sentence. +# Fifteen is longer than any pause measured here and still closes a forgotten one in a quarter of +# a minute rather than two. `silenceStop` moves it; 0 switches it off. +SILENCE_STOP_SECONDS = 15.0 + # What whisper emits when handed near-silence: it does not return "", it confidently returns one of # its training-set boilerplate lines. Untrapped, these get TYPED INTO YOUR DOCUMENT, which # is the worst failure this tool has — silence should produce nothing, never words you did not @@ -2115,6 +2129,14 @@ def auto_stop_seconds() -> float: return AUTO_STOP_SECONDS +def silence_stop_seconds() -> float: + """How long a silent microphone stays open. ``silenceStop`` overrides; ``0`` switches it off.""" + raw = _cfg().get("silenceStop") + if isinstance(raw, (int, float)) and float(raw) >= 0: + return float(raw) + return SILENCE_STOP_SECONDS + + def quiet_floor() -> float: """The level below which a clip is a room, not a sentence. ``quietFloor`` overrides.""" raw = _cfg().get("quietFloor") @@ -2143,6 +2165,18 @@ def peak_dbfs(wav: Path) -> float: frames = handle.readframes(handle.getnframes()) except Exception: # noqa: BLE001 — a diagnostic must never break a dictation return 0.0 + return _peak_dbfs(frames) + + +#: The recorder's own format — see the `-ar`/`-ac` flags in :func:`start`. 16 kHz mono 16-bit is +#: 32000 bytes of file per second of audio, which is what lets :func:`tail_dbfs` find "the last ten +#: seconds" by seeking from the END of a file whose header has not been written yet. +SAMPLE_RATE = 16000 +BYTES_PER_SECOND = SAMPLE_RATE * 2 + + +def _peak_dbfs(frames: bytes) -> float: + """Peak of 16-bit PCM ``frames`` in dBFS. ``-inf`` for silence or nothing.""" if not frames: return float("-inf") samples = array.array("h") @@ -2159,6 +2193,33 @@ def peak_dbfs(wav: Path) -> float: return 20 * math.log10(min(peak, 32768) / 32768.0) +def tail_dbfs(wav: Path, seconds: float) -> float: + """Peak of the LAST ``seconds`` of a clip that is STILL BEING RECORDED. ``0.0`` = no opinion. + + Read by seeking from the end of the file rather than through :mod:`wave`, because while ffmpeg + is still appending, the RIFF header holds the lengths it was born with — zero — so every + header-respecting reader sees an empty file (this is what :func:`repair_wav` exists to undo, + and it may not be run against the file the recorder is writing). + + Every byte past the header is a sample, so "the last ten seconds" is the last + ``10 * BYTES_PER_SECOND`` bytes, and a seek is the whole implementation. A clip that has not + yet run that long has no opinion — ``0.0``, the same "no opinion" :func:`peak_dbfs` returns for + a format it will not judge, which is well above any real floor and so is never read as silence. + """ + want = int(seconds * BYTES_PER_SECOND) + if want <= 0: + return 0.0 + try: + size = wav.stat().st_size + if size < want + 44: # 44 = the standard PCM wav header ffmpeg writes + return 0.0 # not enough audio yet to have been quiet for that long + with wav.open("rb") as handle: + handle.seek(size - want) + return _peak_dbfs(handle.read(want)) + except OSError: + return 0.0 + + def audio_seconds(wav: Path) -> float: """How long the RECORDING is, from its own header. ``0.0`` if it cannot be read. @@ -2995,16 +3056,23 @@ def _forgot(rec: Recording) -> None: watching the trigger key, so anything that blocks there is a listener that misses a tap. """ limit = auto_stop_seconds() - if limit <= 0: + quiet = silence_stop_seconds() + if limit <= 0 and quiet <= 0: return - deadline = time.monotonic() + limit - while time.monotonic() < deadline: + deadline = time.monotonic() + (limit if limit > 0 else float("inf")) + why = "" + while not why: if not mine or mine[0] is not rec: return # a tap, or an abort, already ended it - time.sleep(0.5) + if time.monotonic() >= deadline: + why = f"after {limit:.0f}s" + elif quiet > 0 and tail_dbfs(rec.wav, quiet) < quiet_floor(): + why = f"after {quiet:.0f}s of silence" + else: + time.sleep(0.5) if claim(rec) is None: return - emit(f"[--] closed the microphone after {limit:.0f}s — you did not tap to stop") + emit(f"[--] closed the microphone {why} — you did not tap to stop") _land(finish(rec)) def _land(result: Result) -> None: diff --git a/tests/test_murmurflow.py b/tests/test_murmurflow.py index 6ae7a0d..31ba9ae 100644 --- a/tests/test_murmurflow.py +++ b/tests/test_murmurflow.py @@ -13,11 +13,14 @@ import io import itertools import json +import math import os +import struct import sys import threading import time import types +import wave from pathlib import Path import pytest @@ -1111,7 +1114,7 @@ def test_the_peak_is_the_loudest_sample_in_either_direction(tmp_path): # --- a warm server that answers wrongly is bounced ------------------------------------------------ -def _drive_listener(monkeypatch, results, *, warm_starts=True, hold=0, release=True): +def _drive_listener(monkeypatch, results, *, warm_starts=True, hold=0, release=True, quiet=0): """Run `listen_loop` over a fixed list of dictations. Returns (server starts, server stops). ``hold`` is `maxHold`, and it is 0 — OFF — for every caller but the one testing it. The @@ -1146,6 +1149,7 @@ def start(self): monkeypatch.setattr(dictate, "current", lambda: dictate.Recording(1, Path("x.wav"), 0.0)) monkeypatch.setattr(dictate, "cue_ready", lambda: cues.append(1)) config.set_value("maxHold", hold) + config.set_value("silenceStop", quiet) pending = list(results) monkeypatch.setattr(dictate, "finish", lambda _rec: pending.pop(0)) @@ -1677,6 +1681,54 @@ def test_a_forgotten_key_still_gets_its_words(monkeypatch): assert starts == [1] # the daemon started its server and then rescued the clip on its own +def test_the_last_seconds_of_a_clip_still_being_recorded_can_be_read(tmp_path): + """`wave` cannot answer this and that is the whole reason it exists. + + While ffmpeg is appending, the RIFF header still holds the lengths it was born with — zero — + so every header-respecting reader sees an empty file. Every byte past the header is a sample, + so "the last ten seconds" is a seek from the END. + """ + clip = tmp_path / "growing.wav" + with wave.open(str(clip), "wb") as handle: + handle.setnchannels(1) + handle.setsampwidth(2) + handle.setframerate(dictate.SAMPLE_RATE) + tone = b"".join( + struct.pack(" -20 # fifteen reaches back into the speech + # A clip too short to have been quiet that long has NO OPINION, and 0.0 is well above any + # floor — a fresh recording must never read as silence and close itself. + assert dictate.tail_dbfs(clip, 60) == 0.0 + assert dictate.tail_dbfs(clip, 0) == 0.0 + assert dictate.tail_dbfs(tmp_path / "nothing.wav", 5) == 0.0 + assert dictate.tail_dbfs(clip, 5) < dictate.quiet_floor() # what the watchdog actually asks + + +def test_the_microphone_closes_itself_after_a_stretch_of_silence(monkeypatch): + """ "We should close the microphone after not talking for 15 seconds or something like that." + + Fifteen and not the two or three that dictation apps use: the gesture is a TAP, so nothing is + telling the microphone you are still there, and this operator thinks mid-sentence. A clip cut + at three seconds would end half his sentences mid-thought, with the rest spoken into a closed + microphone — worse than the forgotten microphone being fixed, because that one loses nothing. + """ + assert dictate.silence_stop_seconds() == dictate.SILENCE_STOP_SECONDS == 15.0 + config.set_value("silenceStop", 8) + assert dictate.silence_stop_seconds() == 8.0 + config.set_value("silenceStop", 0) + assert dictate.silence_stop_seconds() == 0.0 # switched off + config.set_value("silenceStop", -1) + assert dictate.silence_stop_seconds() == dictate.SILENCE_STOP_SECONDS # never a negative + config.set_value("silenceStop", 0) + # Driven through the real listener: never a tap, never the 120s cap, only silence. + monkeypatch.setattr(dictate, "tail_dbfs", lambda _wav, _seconds: -90.0) + starts, _ = _drive_listener(monkeypatch, [_clip(True)], hold=0, quiet=0.05, release=False) + assert starts == [1] # the clip was finished, by silence alone, with the hold cap OFF + + def test_the_microphone_closes_itself_when_the_second_tap_never_comes(monkeypatch): """ "What happens a lot of times is that I forget to close the microphone."