feat(dictate): the microphone closes when you stop talking

murmurflow/dictate.py

@@ -1043,6 +1043,20 @@ def strip_fillers(transcript: str) -> str:
 # real dictation measured on this machine is ~68s; `maxHold` moves it.
 AUTO_STOP_SECONDS = 120.0
 
+# How long the microphone stays open with nothing being said before the daemon closes it and types
+# what it has. The operator asked for it and left the number to me ("I don't know what is a good
+# timing, you have to decide that yourself"), so: fifteen seconds.
+#
+# Dictation apps that stop on silence sit around two to three, and two to three is WRONG HERE. The
+# gesture is a tap, not a held key, so nothing is telling the microphone you are still there; and
+# this operator thinks mid-sentence — the pauses that produced every punctuation bug in this file
+# were real, and a clip cut at three seconds would have ended half of them mid-thought, with the
+# rest of the sentence spoken into a closed microphone. That failure is worse than the one being
+# fixed, because a forgotten microphone loses nothing and a truncated sentence loses the sentence.
+# Fifteen is longer than any pause measured here and still closes a forgotten one in a quarter of
+# a minute rather than two. `silenceStop` moves it; 0 switches it off.
+SILENCE_STOP_SECONDS = 15.0
+
 # What whisper emits when handed near-silence: it does not return "", it confidently returns one of
 # its training-set boilerplate lines. Untrapped, these get TYPED INTO YOUR DOCUMENT, which
 # is the worst failure this tool has — silence should produce nothing, never words you did not
@@ -1790,21 +1804,63 @@ def stream_tail(pasted: str, final: str) -> str:
     if not already:
         return final
     words = final.split()
-    keys = [_key(word) for word in words]
+    return " ".join(words[_reached(already, [_key(word) for word in words]) :])
+
+
+def _reached(already: list[str], keys: list[str]) -> int:
+    """The index in ``keys`` just past everything that is already on screen.
+
+    Split out of :func:`stream_tail` because :func:`missing_mark` asks the same question about the
+    same two sequences — where does the screen END inside this transcript — and two answers to that
+    would drift apart word by word.
+    """
     if keys[: len(already)] == already:
-        return " ".join(words[len(already) :])
+        return len(already)
     matcher = difflib.SequenceMatcher(a=already, b=keys, autojunk=False)
     matched = [block for block in matcher.get_matching_blocks() if block.size]
     if not matched:  # nothing corresponds: trust the count, lose nothing
-        return " ".join(words[len(already) :])
+        return len(already)
     reached = matched[-1]
     # Words on screen PAST the alignment are ones the final pass said differently. The tail that
     # follows them is not new text, it is the same words again in the better model's wording, and
     # typing it puts both on screen — reported as "it just adds another word at the end", which is
     # exactly what one reworded last word looks like ("the design" + "designs"). One dropped for
     # one left over: the rewording is skipped and anything genuinely beyond the screen still lands.
     reworded = len(already) - (reached.a + reached.size)
-    return " ".join(words[reached.b + reached.size + reworded :])
+    return reached.b + reached.size + reworded
+
+
+def missing_mark(pasted: str, settled: str) -> str:
+    """The mark that belongs directly after ``pasted``, once a later word has confirmed it.
+
+    **This is the punctuation streaming used to lose, and it is the last of it.** A mark rides on
+    the word in front of it, and :func:`stable_prefix` will not type the mark on a word that is
+    still touching the end of the audio — a pause is how whisper decides a sentence ended, and it
+    takes that decision back the moment the speaker carries on. So the word lands bare, and
+    nothing could ever put the mark on afterwards: the word is already on screen and there is no
+    un-type. Reported as "there was a break before, but no punctuation".
+
+    A LATER pass answers the question the earlier one could not. If the word carrying the mark is
+    no longer at the end of the transcript — real speech follows it, and whisper still ends the
+    sentence there — the mark is a decision made WITH the following audio, which is the same test
+    every other word passes before it is typed. It goes on with no space in front of it, joined to
+    the word it belongs to.
+
+    Returns ``""`` unless a word after it has also settled, so this can never be the lone full stop
+    of :func:`end_mark`'s docstring: the mark is only ever typed in the same breath as the word
+    that proves it.
+    """
+    already = [_key(word) for word in pasted.split()]
+    words = settled.split()
+    if not already or not words:
+        return ""
+    index = _reached(already, [_key(word) for word in words])
+    if index <= 0 or index >= len(words):
+        return ""  # nothing before it, or nothing after it to confirm it
+    mark = _TRAILING_MARK.search(words[index - 1])
+    if not mark or pasted.rstrip().endswith(mark.group()):
+        return ""
+    return mark.group()
 
 
 def _partial(live: Path, snapshot: Path, language: str = "") -> Heard:
@@ -1948,6 +2004,9 @@ def _stream_loop(rec: Recording, stream: Stream) -> None:
         settled = stable_prefix(previous, heard)
         previous = heard
         chunk = stream_tail(stream.text, settled) if settled else ""
+        # The mark on the word already at the end of the screen, now that a later word has
+        # settled behind it. Only ever together with that word — see :func:`missing_mark`.
+        mark = missing_mark(stream.text, settled) if chunk else ""
         if chunk:
             with _INJECT_LOCK:
                 # Inside the lock, because `stop_streaming` sets this and then takes the lock: past
@@ -1961,7 +2020,7 @@ def _stream_loop(rec: Recording, stream: Stream) -> None:
                 # `stream.text` is literally what is on the screen, built from what LANDED rather
                 # than from what was asked for — see :func:`place`. The leading space travels with
                 # the chunk, so this is a concatenation and never a re-join.
-                landed = place(f" {chunk}" if stream.text else chunk)
+                landed = place(f"{mark} {chunk}" if stream.text else chunk)
                 if landed:
                     stream.text = f"{stream.text}{landed}".strip()
                     stream.typed += 1
@@ -2070,6 +2129,14 @@ def auto_stop_seconds() -> float:
     return AUTO_STOP_SECONDS
 
 
+def silence_stop_seconds() -> float:
+    """How long a silent microphone stays open. ``silenceStop`` overrides; ``0`` switches it off."""
+    raw = _cfg().get("silenceStop")
+    if isinstance(raw, (int, float)) and float(raw) >= 0:
+        return float(raw)
+    return SILENCE_STOP_SECONDS
+
+
 def quiet_floor() -> float:
     """The level below which a clip is a room, not a sentence. ``quietFloor`` overrides."""
     raw = _cfg().get("quietFloor")
@@ -2098,6 +2165,18 @@ def peak_dbfs(wav: Path) -> float:
             frames = handle.readframes(handle.getnframes())
     except Exception:  # noqa: BLE001 — a diagnostic must never break a dictation
         return 0.0
+    return _peak_dbfs(frames)
+
+
+#: The recorder's own format — see the `-ar`/`-ac` flags in :func:`start`. 16 kHz mono 16-bit is
+#: 32000 bytes of file per second of audio, which is what lets :func:`tail_dbfs` find "the last ten
+#: seconds" by seeking from the END of a file whose header has not been written yet.
+SAMPLE_RATE = 16000
+BYTES_PER_SECOND = SAMPLE_RATE * 2
+
+
+def _peak_dbfs(frames: bytes) -> float:
+    """Peak of 16-bit PCM ``frames`` in dBFS. ``-inf`` for silence or nothing."""
     if not frames:
         return float("-inf")
     samples = array.array("h")
@@ -2114,6 +2193,33 @@ def peak_dbfs(wav: Path) -> float:
     return 20 * math.log10(min(peak, 32768) / 32768.0)
 
 
+def tail_dbfs(wav: Path, seconds: float) -> float:
+    """Peak of the LAST ``seconds`` of a clip that is STILL BEING RECORDED. ``0.0`` = no opinion.
+
+    Read by seeking from the end of the file rather than through :mod:`wave`, because while ffmpeg
+    is still appending, the RIFF header holds the lengths it was born with — zero — so every
+    header-respecting reader sees an empty file (this is what :func:`repair_wav` exists to undo,
+    and it may not be run against the file the recorder is writing).
+
+    Every byte past the header is a sample, so "the last ten seconds" is the last
+    ``10 * BYTES_PER_SECOND`` bytes, and a seek is the whole implementation. A clip that has not
+    yet run that long has no opinion — ``0.0``, the same "no opinion" :func:`peak_dbfs` returns for
+    a format it will not judge, which is well above any real floor and so is never read as silence.
+    """
+    want = int(seconds * BYTES_PER_SECOND)
+    if want <= 0:
+        return 0.0
+    try:
+        size = wav.stat().st_size
+        if size < want + 44:  # 44 = the standard PCM wav header ffmpeg writes
+            return 0.0  # not enough audio yet to have been quiet for that long
+        with wav.open("rb") as handle:
+            handle.seek(size - want)
+            return _peak_dbfs(handle.read(want))
+    except OSError:
+        return 0.0
+
+
 def audio_seconds(wav: Path) -> float:
     """How long the RECORDING is, from its own header. ``0.0`` if it cannot be read.
 
@@ -2950,16 +3056,23 @@ def _forgot(rec: Recording) -> None:
         watching the trigger key, so anything that blocks there is a listener that misses a tap.
         """
         limit = auto_stop_seconds()
-        if limit <= 0:
+        quiet = silence_stop_seconds()
+        if limit <= 0 and quiet <= 0:
             return
-        deadline = time.monotonic() + limit
-        while time.monotonic() < deadline:
+        deadline = time.monotonic() + (limit if limit > 0 else float("inf"))
+        why = ""
+        while not why:
             if not mine or mine[0] is not rec:
                 return  # a tap, or an abort, already ended it
-            time.sleep(0.5)
+            if time.monotonic() >= deadline:
+                why = f"after {limit:.0f}s"
+            elif quiet > 0 and tail_dbfs(rec.wav, quiet) < quiet_floor():
+                why = f"after {quiet:.0f}s of silence"
+            else:
+                time.sleep(0.5)
         if claim(rec) is None:
             return
-        emit(f"[--] closed the microphone after {limit:.0f}s — you did not tap to stop")
+        emit(f"[--] closed the microphone {why} — you did not tap to stop")
         _land(finish(rec))
 
     def _land(result: Result) -> None:

tests/test_murmurflow.py

@@ -13,11 +13,14 @@
 import io
 import itertools
 import json
+import math
 import os
+import struct
 import sys
 import threading
 import time
 import types
+import wave
 from pathlib import Path
 
 import pytest
@@ -1111,7 +1114,7 @@ def test_the_peak_is_the_loudest_sample_in_either_direction(tmp_path):
 # --- a warm server that answers wrongly is bounced ------------------------------------------------
 
 
-def _drive_listener(monkeypatch, results, *, warm_starts=True, hold=0, release=True):
+def _drive_listener(monkeypatch, results, *, warm_starts=True, hold=0, release=True, quiet=0):
     """Run `listen_loop` over a fixed list of dictations. Returns (server starts, server stops).
 
     ``hold`` is `maxHold`, and it is 0 — OFF — for every caller but the one testing it. The
@@ -1146,6 +1149,7 @@ def start(self):
     monkeypatch.setattr(dictate, "current", lambda: dictate.Recording(1, Path("x.wav"), 0.0))
     monkeypatch.setattr(dictate, "cue_ready", lambda: cues.append(1))
     config.set_value("maxHold", hold)
+    config.set_value("silenceStop", quiet)
     pending = list(results)
     monkeypatch.setattr(dictate, "finish", lambda _rec: pending.pop(0))
 
@@ -1677,6 +1681,54 @@ def test_a_forgotten_key_still_gets_its_words(monkeypatch):
     assert starts == [1]  # the daemon started its server and then rescued the clip on its own
 
 
+def test_the_last_seconds_of_a_clip_still_being_recorded_can_be_read(tmp_path):
+    """`wave` cannot answer this and that is the whole reason it exists.
+
+    While ffmpeg is appending, the RIFF header still holds the lengths it was born with — zero —
+    so every header-respecting reader sees an empty file. Every byte past the header is a sample,
+    so "the last ten seconds" is a seek from the END.
+    """
+    clip = tmp_path / "growing.wav"
+    with wave.open(str(clip), "wb") as handle:
+        handle.setnchannels(1)
+        handle.setsampwidth(2)
+        handle.setframerate(dictate.SAMPLE_RATE)
+        tone = b"".join(
+            struct.pack("<h", int(8000 * math.sin(i / 8))) for i in range(dictate.SAMPLE_RATE * 10)
+        )
+        handle.writeframes(tone + b"\x00" * (dictate.SAMPLE_RATE * 10 * 2))
+    assert dictate.tail_dbfs(clip, 5) == float("-inf")  # the last five seconds are silence
+    assert dictate.tail_dbfs(clip, 15) > -20  # fifteen reaches back into the speech
+    # A clip too short to have been quiet that long has NO OPINION, and 0.0 is well above any
+    # floor — a fresh recording must never read as silence and close itself.
+    assert dictate.tail_dbfs(clip, 60) == 0.0
+    assert dictate.tail_dbfs(clip, 0) == 0.0
+    assert dictate.tail_dbfs(tmp_path / "nothing.wav", 5) == 0.0
+    assert dictate.tail_dbfs(clip, 5) < dictate.quiet_floor()  # what the watchdog actually asks
+
+
+def test_the_microphone_closes_itself_after_a_stretch_of_silence(monkeypatch):
+    """ "We should close the microphone after not talking for 15 seconds or something like that."
+
+    Fifteen and not the two or three that dictation apps use: the gesture is a TAP, so nothing is
+    telling the microphone you are still there, and this operator thinks mid-sentence. A clip cut
+    at three seconds would end half his sentences mid-thought, with the rest spoken into a closed
+    microphone — worse than the forgotten microphone being fixed, because that one loses nothing.
+    """
+    assert dictate.silence_stop_seconds() == dictate.SILENCE_STOP_SECONDS == 15.0
+    config.set_value("silenceStop", 8)
+    assert dictate.silence_stop_seconds() == 8.0
+    config.set_value("silenceStop", 0)
+    assert dictate.silence_stop_seconds() == 0.0  # switched off
+    config.set_value("silenceStop", -1)
+    assert dictate.silence_stop_seconds() == dictate.SILENCE_STOP_SECONDS  # never a negative
+    config.set_value("silenceStop", 0)
+    # Driven through the real listener: never a tap, never the 120s cap, only silence.
+    monkeypatch.setattr(dictate, "tail_dbfs", lambda _wav, _seconds: -90.0)
+    starts, _ = _drive_listener(monkeypatch, [_clip(True)], hold=0, quiet=0.05, release=False)
+    assert starts == [1]  # the clip was finished, by silence alone, with the hold cap OFF
+
+
 def test_the_microphone_closes_itself_when_the_second_tap_never_comes(monkeypatch):
     """ "What happens a lot of times is that I forget to close the microphone."
 
@@ -1695,6 +1747,57 @@ def test_the_microphone_closes_itself_when_the_second_tap_never_comes(monkeypatc
     assert dictate.MAX_CLIP_SECONDS == 600
 
 
+def _drive_stream(passes):
+    """What lands on screen when the live pass reads ``passes``, one after another.
+
+    The same four calls `_stream_loop` makes, in the same order, so a sequence that broke a real
+    dictation can be replayed as a test. Kept beside the tests that use it rather than inside them:
+    four copies of the loop drift, and a copy that drifts stops testing the loop.
+    """
+    screen = previous = ""
+    for heard in passes:
+        settled = dictate.stable_prefix(previous, heard)
+        previous = heard
+        chunk = dictate.stream_tail(screen, settled) if settled else ""
+        mark = dictate.missing_mark(screen, settled) if chunk else ""
+        if chunk:
+            screen = f"{screen}{mark} {chunk}".strip() if screen else chunk
+    return screen
+
+
+def test_the_mark_lands_once_a_later_word_confirms_it():
+    """Reported as "there was a break before, but no punctuation".
+
+    A mark rides on the word in front of it, and that word is never typed with its mark while it
+    still touches the end of the audio — a pause is how whisper decides a sentence ended, and it
+    takes that back the moment the speaker carries on. So the word landed bare and nothing could
+    put the mark on afterwards. A later pass answers it: real speech follows and whisper STILL
+    ends the sentence there, so the mark goes on, joined to the word it belongs to.
+    """
+    reference = "I did not say the three times. I did, however, say really three times."
+    screen = _drive_stream(
+        [
+            "I did not say the three times",
+            "I did not say the three times",  # the break
+            "I did not say the three times.",  # whisper ends the sentence
+            "I did not say the three times.",
+            "I did not say the three times. I",  # he carries on
+            "I did not say the three times. I did however say",
+            "I did not say the three times. I did, however, say really",
+            reference,
+            reference,
+        ]
+    )
+    # ...plus the one thing only the key release can know: the mark that ends the clip.
+    landed = screen + dictate.end_mark(screen, reference)
+    assert landed == reference  # streamed, and identical to the whole-clip transcript
+    # And the fixes it must not undo, driven through the same loop.
+    assert _drive_stream(["Could you please work on my", "Could you please work on my..."] * 2) == (
+        "Could you please work on my"
+    )
+    assert "The The" not in _drive_stream(["Well, yeah. The The The"] * 3)
+
+
 def test_a_pause_never_types_a_lone_full_stop_where_the_next_word_goes():
     """Reported as "it puts a period instead of the word" after a short break.
 
@@ -1704,24 +1807,21 @@ def test_a_pause_never_types_a_lone_full_stop_where_the_next_word_goes():
     screen with no letters in it, so the next alignment read it as something the final pass had
     reworded and dropped a real word to pay for it. The word this ate, in the report, was "But".
     """
-    screen, previous = "", ""
-    for heard in (
-        "and then I ran the command",
-        "and then I ran the command",  # the pause: the transcript stops growing
-        "and then I ran the command.",  # whisper decides the sentence ended
-        "and then I ran the command.",
-        "and then I ran the command. But",  # he speaks again
-        "and then I ran the command. But when I say",
-        "and then I ran the command. But when I say",
-    ):
-        settled = dictate.stable_prefix(previous, heard)
-        previous = heard
-        chunk = dictate.stream_tail(screen, settled) if settled else ""
-        if chunk:
-            screen = f"{screen} {chunk}".strip() if screen else chunk
-    assert " ." not in screen
-    assert "But" in screen
-    assert screen == "and then I ran the command But when I say"
+    screen = _drive_stream(
+        [
+            "and then I ran the command",
+            "and then I ran the command",  # the pause: the transcript stops growing
+            "and then I ran the command.",  # whisper decides the sentence ended
+            "and then I ran the command.",
+            "and then I ran the command. But",  # he speaks again
+            "and then I ran the command. But when I say",
+            "and then I ran the command. But when I say",
+        ]
+    )
+    assert " ." not in screen  # never a mark standing on its own
+    assert "But" in screen  # and never a word paid to the alignment for one
+    # The full stop DOES land, because "But" settled behind it and confirmed it.
+    assert screen == "and then I ran the command. But when I say"
 
 
 def test_a_pause_does_not_put_a_full_stop_in_the_middle_of_the_sentence():