diff --git a/murmurflow/dictate.py b/murmurflow/dictate.py index 06976e6..59c01fd 100644 --- a/murmurflow/dictate.py +++ b/murmurflow/dictate.py @@ -1043,6 +1043,20 @@ def strip_fillers(transcript: str) -> str: # real dictation measured on this machine is ~68s; `maxHold` moves it. AUTO_STOP_SECONDS = 120.0 +# How long the microphone stays open with nothing being said before the daemon closes it and types +# what it has. The operator asked for it and left the number to me ("I don't know what is a good +# timing, you have to decide that yourself"), so: fifteen seconds. +# +# Dictation apps that stop on silence sit around two to three, and two to three is WRONG HERE. The +# gesture is a tap, not a held key, so nothing is telling the microphone you are still there; and +# this operator thinks mid-sentence — the pauses that produced every punctuation bug in this file +# were real, and a clip cut at three seconds would have ended half of them mid-thought, with the +# rest of the sentence spoken into a closed microphone. That failure is worse than the one being +# fixed, because a forgotten microphone loses nothing and a truncated sentence loses the sentence. +# Fifteen is longer than any pause measured here and still closes a forgotten one in a quarter of +# a minute rather than two. `silenceStop` moves it; 0 switches it off. +SILENCE_STOP_SECONDS = 15.0 + # What whisper emits when handed near-silence: it does not return "", it confidently returns one of # its training-set boilerplate lines. Untrapped, these get TYPED INTO YOUR DOCUMENT, which # is the worst failure this tool has — silence should produce nothing, never words you did not @@ -1790,13 +1804,22 @@ def stream_tail(pasted: str, final: str) -> str: if not already: return final words = final.split() - keys = [_key(word) for word in words] + return " ".join(words[_reached(already, [_key(word) for word in words]) :]) + + +def _reached(already: list[str], keys: list[str]) -> int: + """The index in ``keys`` just past everything that is already on screen. + + Split out of :func:`stream_tail` because :func:`missing_mark` asks the same question about the + same two sequences — where does the screen END inside this transcript — and two answers to that + would drift apart word by word. + """ if keys[: len(already)] == already: - return " ".join(words[len(already) :]) + return len(already) matcher = difflib.SequenceMatcher(a=already, b=keys, autojunk=False) matched = [block for block in matcher.get_matching_blocks() if block.size] if not matched: # nothing corresponds: trust the count, lose nothing - return " ".join(words[len(already) :]) + return len(already) reached = matched[-1] # Words on screen PAST the alignment are ones the final pass said differently. The tail that # follows them is not new text, it is the same words again in the better model's wording, and @@ -1804,7 +1827,40 @@ def stream_tail(pasted: str, final: str) -> str: # exactly what one reworded last word looks like ("the design" + "designs"). One dropped for # one left over: the rewording is skipped and anything genuinely beyond the screen still lands. reworded = len(already) - (reached.a + reached.size) - return " ".join(words[reached.b + reached.size + reworded :]) + return reached.b + reached.size + reworded + + +def missing_mark(pasted: str, settled: str) -> str: + """The mark that belongs directly after ``pasted``, once a later word has confirmed it. + + **This is the punctuation streaming used to lose, and it is the last of it.** A mark rides on + the word in front of it, and :func:`stable_prefix` will not type the mark on a word that is + still touching the end of the audio — a pause is how whisper decides a sentence ended, and it + takes that decision back the moment the speaker carries on. So the word lands bare, and + nothing could ever put the mark on afterwards: the word is already on screen and there is no + un-type. Reported as "there was a break before, but no punctuation". + + A LATER pass answers the question the earlier one could not. If the word carrying the mark is + no longer at the end of the transcript — real speech follows it, and whisper still ends the + sentence there — the mark is a decision made WITH the following audio, which is the same test + every other word passes before it is typed. It goes on with no space in front of it, joined to + the word it belongs to. + + Returns ``""`` unless a word after it has also settled, so this can never be the lone full stop + of :func:`end_mark`'s docstring: the mark is only ever typed in the same breath as the word + that proves it. + """ + already = [_key(word) for word in pasted.split()] + words = settled.split() + if not already or not words: + return "" + index = _reached(already, [_key(word) for word in words]) + if index <= 0 or index >= len(words): + return "" # nothing before it, or nothing after it to confirm it + mark = _TRAILING_MARK.search(words[index - 1]) + if not mark or pasted.rstrip().endswith(mark.group()): + return "" + return mark.group() def _partial(live: Path, snapshot: Path, language: str = "") -> Heard: @@ -1948,6 +2004,9 @@ def _stream_loop(rec: Recording, stream: Stream) -> None: settled = stable_prefix(previous, heard) previous = heard chunk = stream_tail(stream.text, settled) if settled else "" + # The mark on the word already at the end of the screen, now that a later word has + # settled behind it. Only ever together with that word — see :func:`missing_mark`. + mark = missing_mark(stream.text, settled) if chunk else "" if chunk: with _INJECT_LOCK: # Inside the lock, because `stop_streaming` sets this and then takes the lock: past @@ -1961,7 +2020,7 @@ def _stream_loop(rec: Recording, stream: Stream) -> None: # `stream.text` is literally what is on the screen, built from what LANDED rather # than from what was asked for — see :func:`place`. The leading space travels with # the chunk, so this is a concatenation and never a re-join. - landed = place(f" {chunk}" if stream.text else chunk) + landed = place(f"{mark} {chunk}" if stream.text else chunk) if landed: stream.text = f"{stream.text}{landed}".strip() stream.typed += 1 @@ -2070,6 +2129,14 @@ def auto_stop_seconds() -> float: return AUTO_STOP_SECONDS +def silence_stop_seconds() -> float: + """How long a silent microphone stays open. ``silenceStop`` overrides; ``0`` switches it off.""" + raw = _cfg().get("silenceStop") + if isinstance(raw, (int, float)) and float(raw) >= 0: + return float(raw) + return SILENCE_STOP_SECONDS + + def quiet_floor() -> float: """The level below which a clip is a room, not a sentence. ``quietFloor`` overrides.""" raw = _cfg().get("quietFloor") @@ -2098,6 +2165,18 @@ def peak_dbfs(wav: Path) -> float: frames = handle.readframes(handle.getnframes()) except Exception: # noqa: BLE001 — a diagnostic must never break a dictation return 0.0 + return _peak_dbfs(frames) + + +#: The recorder's own format — see the `-ar`/`-ac` flags in :func:`start`. 16 kHz mono 16-bit is +#: 32000 bytes of file per second of audio, which is what lets :func:`tail_dbfs` find "the last ten +#: seconds" by seeking from the END of a file whose header has not been written yet. +SAMPLE_RATE = 16000 +BYTES_PER_SECOND = SAMPLE_RATE * 2 + + +def _peak_dbfs(frames: bytes) -> float: + """Peak of 16-bit PCM ``frames`` in dBFS. ``-inf`` for silence or nothing.""" if not frames: return float("-inf") samples = array.array("h") @@ -2114,6 +2193,33 @@ def peak_dbfs(wav: Path) -> float: return 20 * math.log10(min(peak, 32768) / 32768.0) +def tail_dbfs(wav: Path, seconds: float) -> float: + """Peak of the LAST ``seconds`` of a clip that is STILL BEING RECORDED. ``0.0`` = no opinion. + + Read by seeking from the end of the file rather than through :mod:`wave`, because while ffmpeg + is still appending, the RIFF header holds the lengths it was born with — zero — so every + header-respecting reader sees an empty file (this is what :func:`repair_wav` exists to undo, + and it may not be run against the file the recorder is writing). + + Every byte past the header is a sample, so "the last ten seconds" is the last + ``10 * BYTES_PER_SECOND`` bytes, and a seek is the whole implementation. A clip that has not + yet run that long has no opinion — ``0.0``, the same "no opinion" :func:`peak_dbfs` returns for + a format it will not judge, which is well above any real floor and so is never read as silence. + """ + want = int(seconds * BYTES_PER_SECOND) + if want <= 0: + return 0.0 + try: + size = wav.stat().st_size + if size < want + 44: # 44 = the standard PCM wav header ffmpeg writes + return 0.0 # not enough audio yet to have been quiet for that long + with wav.open("rb") as handle: + handle.seek(size - want) + return _peak_dbfs(handle.read(want)) + except OSError: + return 0.0 + + def audio_seconds(wav: Path) -> float: """How long the RECORDING is, from its own header. ``0.0`` if it cannot be read. @@ -2950,16 +3056,23 @@ def _forgot(rec: Recording) -> None: watching the trigger key, so anything that blocks there is a listener that misses a tap. """ limit = auto_stop_seconds() - if limit <= 0: + quiet = silence_stop_seconds() + if limit <= 0 and quiet <= 0: return - deadline = time.monotonic() + limit - while time.monotonic() < deadline: + deadline = time.monotonic() + (limit if limit > 0 else float("inf")) + why = "" + while not why: if not mine or mine[0] is not rec: return # a tap, or an abort, already ended it - time.sleep(0.5) + if time.monotonic() >= deadline: + why = f"after {limit:.0f}s" + elif quiet > 0 and tail_dbfs(rec.wav, quiet) < quiet_floor(): + why = f"after {quiet:.0f}s of silence" + else: + time.sleep(0.5) if claim(rec) is None: return - emit(f"[--] closed the microphone after {limit:.0f}s — you did not tap to stop") + emit(f"[--] closed the microphone {why} — you did not tap to stop") _land(finish(rec)) def _land(result: Result) -> None: diff --git a/tests/test_murmurflow.py b/tests/test_murmurflow.py index 504c241..31ba9ae 100644 --- a/tests/test_murmurflow.py +++ b/tests/test_murmurflow.py @@ -13,11 +13,14 @@ import io import itertools import json +import math import os +import struct import sys import threading import time import types +import wave from pathlib import Path import pytest @@ -1111,7 +1114,7 @@ def test_the_peak_is_the_loudest_sample_in_either_direction(tmp_path): # --- a warm server that answers wrongly is bounced ------------------------------------------------ -def _drive_listener(monkeypatch, results, *, warm_starts=True, hold=0, release=True): +def _drive_listener(monkeypatch, results, *, warm_starts=True, hold=0, release=True, quiet=0): """Run `listen_loop` over a fixed list of dictations. Returns (server starts, server stops). ``hold`` is `maxHold`, and it is 0 — OFF — for every caller but the one testing it. The @@ -1146,6 +1149,7 @@ def start(self): monkeypatch.setattr(dictate, "current", lambda: dictate.Recording(1, Path("x.wav"), 0.0)) monkeypatch.setattr(dictate, "cue_ready", lambda: cues.append(1)) config.set_value("maxHold", hold) + config.set_value("silenceStop", quiet) pending = list(results) monkeypatch.setattr(dictate, "finish", lambda _rec: pending.pop(0)) @@ -1677,6 +1681,54 @@ def test_a_forgotten_key_still_gets_its_words(monkeypatch): assert starts == [1] # the daemon started its server and then rescued the clip on its own +def test_the_last_seconds_of_a_clip_still_being_recorded_can_be_read(tmp_path): + """`wave` cannot answer this and that is the whole reason it exists. + + While ffmpeg is appending, the RIFF header still holds the lengths it was born with — zero — + so every header-respecting reader sees an empty file. Every byte past the header is a sample, + so "the last ten seconds" is a seek from the END. + """ + clip = tmp_path / "growing.wav" + with wave.open(str(clip), "wb") as handle: + handle.setnchannels(1) + handle.setsampwidth(2) + handle.setframerate(dictate.SAMPLE_RATE) + tone = b"".join( + struct.pack(" -20 # fifteen reaches back into the speech + # A clip too short to have been quiet that long has NO OPINION, and 0.0 is well above any + # floor — a fresh recording must never read as silence and close itself. + assert dictate.tail_dbfs(clip, 60) == 0.0 + assert dictate.tail_dbfs(clip, 0) == 0.0 + assert dictate.tail_dbfs(tmp_path / "nothing.wav", 5) == 0.0 + assert dictate.tail_dbfs(clip, 5) < dictate.quiet_floor() # what the watchdog actually asks + + +def test_the_microphone_closes_itself_after_a_stretch_of_silence(monkeypatch): + """ "We should close the microphone after not talking for 15 seconds or something like that." + + Fifteen and not the two or three that dictation apps use: the gesture is a TAP, so nothing is + telling the microphone you are still there, and this operator thinks mid-sentence. A clip cut + at three seconds would end half his sentences mid-thought, with the rest spoken into a closed + microphone — worse than the forgotten microphone being fixed, because that one loses nothing. + """ + assert dictate.silence_stop_seconds() == dictate.SILENCE_STOP_SECONDS == 15.0 + config.set_value("silenceStop", 8) + assert dictate.silence_stop_seconds() == 8.0 + config.set_value("silenceStop", 0) + assert dictate.silence_stop_seconds() == 0.0 # switched off + config.set_value("silenceStop", -1) + assert dictate.silence_stop_seconds() == dictate.SILENCE_STOP_SECONDS # never a negative + config.set_value("silenceStop", 0) + # Driven through the real listener: never a tap, never the 120s cap, only silence. + monkeypatch.setattr(dictate, "tail_dbfs", lambda _wav, _seconds: -90.0) + starts, _ = _drive_listener(monkeypatch, [_clip(True)], hold=0, quiet=0.05, release=False) + assert starts == [1] # the clip was finished, by silence alone, with the hold cap OFF + + def test_the_microphone_closes_itself_when_the_second_tap_never_comes(monkeypatch): """ "What happens a lot of times is that I forget to close the microphone." @@ -1695,6 +1747,57 @@ def test_the_microphone_closes_itself_when_the_second_tap_never_comes(monkeypatc assert dictate.MAX_CLIP_SECONDS == 600 +def _drive_stream(passes): + """What lands on screen when the live pass reads ``passes``, one after another. + + The same four calls `_stream_loop` makes, in the same order, so a sequence that broke a real + dictation can be replayed as a test. Kept beside the tests that use it rather than inside them: + four copies of the loop drift, and a copy that drifts stops testing the loop. + """ + screen = previous = "" + for heard in passes: + settled = dictate.stable_prefix(previous, heard) + previous = heard + chunk = dictate.stream_tail(screen, settled) if settled else "" + mark = dictate.missing_mark(screen, settled) if chunk else "" + if chunk: + screen = f"{screen}{mark} {chunk}".strip() if screen else chunk + return screen + + +def test_the_mark_lands_once_a_later_word_confirms_it(): + """Reported as "there was a break before, but no punctuation". + + A mark rides on the word in front of it, and that word is never typed with its mark while it + still touches the end of the audio — a pause is how whisper decides a sentence ended, and it + takes that back the moment the speaker carries on. So the word landed bare and nothing could + put the mark on afterwards. A later pass answers it: real speech follows and whisper STILL + ends the sentence there, so the mark goes on, joined to the word it belongs to. + """ + reference = "I did not say the three times. I did, however, say really three times." + screen = _drive_stream( + [ + "I did not say the three times", + "I did not say the three times", # the break + "I did not say the three times.", # whisper ends the sentence + "I did not say the three times.", + "I did not say the three times. I", # he carries on + "I did not say the three times. I did however say", + "I did not say the three times. I did, however, say really", + reference, + reference, + ] + ) + # ...plus the one thing only the key release can know: the mark that ends the clip. + landed = screen + dictate.end_mark(screen, reference) + assert landed == reference # streamed, and identical to the whole-clip transcript + # And the fixes it must not undo, driven through the same loop. + assert _drive_stream(["Could you please work on my", "Could you please work on my..."] * 2) == ( + "Could you please work on my" + ) + assert "The The" not in _drive_stream(["Well, yeah. The The The"] * 3) + + def test_a_pause_never_types_a_lone_full_stop_where_the_next_word_goes(): """Reported as "it puts a period instead of the word" after a short break. @@ -1704,24 +1807,21 @@ def test_a_pause_never_types_a_lone_full_stop_where_the_next_word_goes(): screen with no letters in it, so the next alignment read it as something the final pass had reworded and dropped a real word to pay for it. The word this ate, in the report, was "But". """ - screen, previous = "", "" - for heard in ( - "and then I ran the command", - "and then I ran the command", # the pause: the transcript stops growing - "and then I ran the command.", # whisper decides the sentence ended - "and then I ran the command.", - "and then I ran the command. But", # he speaks again - "and then I ran the command. But when I say", - "and then I ran the command. But when I say", - ): - settled = dictate.stable_prefix(previous, heard) - previous = heard - chunk = dictate.stream_tail(screen, settled) if settled else "" - if chunk: - screen = f"{screen} {chunk}".strip() if screen else chunk - assert " ." not in screen - assert "But" in screen - assert screen == "and then I ran the command But when I say" + screen = _drive_stream( + [ + "and then I ran the command", + "and then I ran the command", # the pause: the transcript stops growing + "and then I ran the command.", # whisper decides the sentence ended + "and then I ran the command.", + "and then I ran the command. But", # he speaks again + "and then I ran the command. But when I say", + "and then I ran the command. But when I say", + ] + ) + assert " ." not in screen # never a mark standing on its own + assert "But" in screen # and never a word paid to the alignment for one + # The full stop DOES land, because "But" settled behind it and confirmed it. + assert screen == "and then I ran the command. But when I say" def test_a_pause_does_not_put_a_full_stop_in_the_middle_of_the_sentence():