diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index dd761b0b0..f6e12db3d 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -295,6 +295,10 @@ interface Window { message?: string; error?: string; }>; + getAudioPeaks: ( + filePath: string, + durationSec: number, + ) => Promise; readFileChunk: ( filePath: string, offset: number, @@ -390,6 +394,7 @@ interface Window { transcribe: ( request: import("./stt/transcriptionContract").SttTranscribeRequest, ) => Promise; + cancel: () => Promise; onStatus: ( callback: (event: import("./stt/transcriptionContract").SttStatusEvent) => void, ) => () => void; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 5e0c3af82..c74a7397b 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -55,6 +55,7 @@ import { LlmConfigStore } from "../ai-edition/llm-config-store"; import { mainLogBuffer } from "../diagnostics/main-log-buffer"; import { mainT } from "../i18n"; import { RECORDINGS_DIR } from "../main"; +import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks"; import { readCursorRecordingFile as readCursorRecordingFileFrom, readCursorSidecar, @@ -3093,6 +3094,31 @@ export function registerIpcHandlers( } }); + // Waveform peaks for a timeline clip, decoded natively (see media/audioPeaks). + // The renderer's own pipelines take ~12s on a 32-minute recording because they + // decode the whole track in Chromium; ffmpeg does the same work in ~2s off the + // UI process, and the result is cached on disk so it is paid once per file. + // `peaks: null` means "no native path available" — the caller falls back to + // its own decoding rather than losing the waveform. + ipcMain.handle( + "get-audio-peaks", + async (_, filePath: string, durationSec: number): Promise => { + try { + // Same approval gate as every other read of a renderer-supplied path. + const normalizedPath = await approveReadableVideoPath(filePath); + if (!normalizedPath) { + return { success: false, message: "File path is not approved" }; + } + const peaks = await getAudioPeaks(normalizedPath, durationSec); + return { success: true, peaks }; + } catch (error) { + // A clip with no audio track lands here. Degrade quietly: the renderer + // draws no waveform, which is correct, and logs its own warning. + return { success: false, message: String(error) }; + } + }, + ); + // Cap renderer-requested chunk sizes so a buggy or compromised renderer // cannot make the main process allocate an arbitrarily large buffer. const MAX_IPC_CHUNK_BYTES = 64 * 1024 * 1024; diff --git a/electron/media/__fixtures__/peaks-sample.m4a b/electron/media/__fixtures__/peaks-sample.m4a new file mode 100644 index 000000000..431172aa9 Binary files /dev/null and b/electron/media/__fixtures__/peaks-sample.m4a differ diff --git a/electron/media/audioPeaks.test.ts b/electron/media/audioPeaks.test.ts new file mode 100644 index 000000000..fb20b2677 --- /dev/null +++ b/electron/media/audioPeaks.test.ts @@ -0,0 +1,100 @@ +// @vitest-environment node +import { existsSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { ffmpegCandidates, peakBlockCount, resolveFfmpeg } from "./audioPeaks"; + +const ROOT = path.resolve(__dirname, "..", ".."); + +describe("peakBlockCount", () => { + it("matches the browser pipelines' block maths", () => { + // Same formula as audioPeaksWorker.ts / streamingAudioPeaks.ts: a clip must + // not change shape depending on which pipeline drew it. + expect(peakBlockCount(10)).toBe(2000); + expect(peakBlockCount(60)).toBe(12000); + // Capped, so a 30-minute recording costs the same DOM/array budget as a + // 2-minute one. + expect(peakBlockCount(1951)).toBe(24000); + expect(peakBlockCount(99999)).toBe(24000); + }); + + it("never returns zero blocks for a sliver of audio", () => { + expect(peakBlockCount(0.001)).toBe(1); + }); +}); + +describe("ffmpeg resolution", () => { + it("prefers the shared build the installer actually ships", () => { + const candidates = ffmpegCandidates(ROOT); + const shared = candidates.findIndex((c) => c.endsWith("ffmpeg-shared.exe")); + const vendorTree = candidates.findIndex((c) => c.includes("lgpl-shared")); + if (process.platform === "win32") { + expect(shared).toBeGreaterThanOrEqual(0); + // The static ffmpeg.exe is excluded from the Windows installer + // ("!win32-*/ffmpeg.exe"), so resolving to it would work in dev and fail + // in production. It must not be a candidate at all. + expect( + candidates.some((c) => c.endsWith(`bin${path.sep}win32-x64${path.sep}ffmpeg.exe`)), + ).toBe(false); + expect(shared).toBeLessThan(vendorTree); + } + }); + + it("honours the env override first", () => { + process.env.OPENSCREEN_FFMPEG_PATH = "/custom/ffmpeg"; + try { + expect(ffmpegCandidates(ROOT)[0]).toBe("/custom/ffmpeg"); + } finally { + process.env.OPENSCREEN_FFMPEG_PATH = undefined; + } + }); + + it("returns null rather than throwing when nothing is staged", () => { + expect(resolveFfmpeg(path.join(ROOT, "does", "not", "exist"))).toBeNull(); + }); +}); + +// Only runs where the binary is actually staged; skipped elsewhere rather than +// failing a checkout that has not run scripts/fetch-ffmpeg.mjs. +const staged = resolveFfmpeg(ROOT); +describe.runIf(staged)("decoding a real file", () => { + it("produces peaks in range, with real signal in them", async () => { + // A synthetic 5s 440 Hz tone from ffmpeg's own lavfi source — no user + // recording in the repo, and a signal whose shape is known rather than + // "whatever this capture happened to contain". + const fixture = path.join(ROOT, "electron", "media", "__fixtures__", "peaks-sample.m4a"); + if (!existsSync(fixture)) return; + const { getAudioPeaks } = await import("./audioPeaks"); + const peaks = await getAudioPeaks(fixture, 5); + expect(peaks).not.toBeNull(); + if (!peaks) return; + expect(peaks.length).toBe(peakBlockCount(5) * 2); + // [min, max] pairs, both inside [-1, 1], min <= 0 <= max (the folder starts + // each block at the silence baseline, like the worker does). + for (let i = 0; i < peaks.length; i += 2) { + expect(peaks[i]).toBeLessThanOrEqual(0); + expect(peaks[i + 1]).toBeGreaterThanOrEqual(0); + expect(peaks[i]).toBeGreaterThanOrEqual(-1); + expect(peaks[i + 1]).toBeLessThanOrEqual(1); + } + // Not all silence — otherwise everything above would pass on a pipeline + // that returned a zeroed array. + // + // The bound is tight rather than "> 0" because loose is the same as + // absent here: the mistakes worth catching are all scale errors — int16 + // divided by 65536 instead of 32768, a stereo downmix halving the signal, + // a block whose samples never get compared — and every one of them is a + // factor of two. 0.214 is what ffmpeg itself decodes this fixture to + // (verified with `-f s16le` straight to a file), so this asserts the fold + // agrees with the decoder rather than restating the fixture's nominal + // amplitude, which lavfi's volume filter does not actually deliver. + let mn = 0; + let mx = 0; + for (const v of peaks) { + if (v < mn) mn = v; + if (v > mx) mx = v; + } + expect(mx).toBeCloseTo(0.214, 1); + expect(mn).toBeCloseTo(-0.205, 1); + }, 120_000); +}); diff --git a/electron/media/audioPeaks.ts b/electron/media/audioPeaks.ts new file mode 100644 index 000000000..056cab54e --- /dev/null +++ b/electron/media/audioPeaks.ts @@ -0,0 +1,288 @@ +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { app } from "electron"; + +/** + * Waveform peaks for the timeline, computed in the main process with ffmpeg. + * + * WHY THIS EXISTS. The renderer had two pipelines and both decode the whole + * audio track in Chromium, which is the entire cost. Measured head-to-head on a + * 32-minute screen recording (68 MB): + * + * decodeAudioData (whole track) 12003 ms, 714 MB resident + * WebCodecs chunk-by-chunk streaming 12259 ms, ~192 kB resident + * ffmpeg -vn -ac 1 -ar 16000 ~2000 ms, nothing resident + * + * The two browser paths differ only in memory; ffmpeg is ~6x faster than both + * because a native AAC decoder is simply faster than Chromium's, and it runs + * off the UI process entirely. The peaks then get cached on disk, so the cost + * is paid once per recording rather than once per session. + * + * ponytail: the CLI, not libav bindings in the compositor addon. The addon + * would avoid a process spawn — worth ~20 ms against a ~2000 ms decode — for a + * new Rust surface, an N-API entry point and a build story on three platforms. + * Revisit only if peaks ever need to share a decode with something else. + */ + +/** IPC reply. `peaks: null` on success means "no native ffmpeg here" — a + * fallback signal, not a failure. */ +export interface AudioPeaksResult { + success: boolean; + peaks?: Float32Array | null; + message?: string; +} + +/** PCM the peaks are computed from. Mono (ffmpeg downmixes) so channels are + * already averaged, and 16 kHz because peak buckets are at most 200/s: that + * still leaves 80 samples per bucket, far more than a min/max needs. */ +const PCM_RATE = 16_000; + +/** Matches `audioPeaksWorker.ts` and `streamingAudioPeaks.ts` so all three + * render identically — a clip must not change shape with the pipeline. */ +const MAX_PEAK_BLOCKS = 24_000; +const PEAK_BLOCKS_PER_SEC = 200; + +/** A recording whose audio takes longer than this to decode is not a recording, + * it is a wedged ffmpeg. ~30x the worst measured case. */ +const DECODE_TIMEOUT_MS = 60_000; + +/** + * Where to find an ffmpeg that actually exists at runtime, in priority order. + * + * Note the Windows shape: `electron-builder.json5` deliberately excludes the + * STATIC `ffmpeg.exe` (109 MB) from the installer, so resolving to it would + * work in dev and fail in production — the exact class of bug that is invisible + * until someone runs the packaged app. The SHARED build is 1 MB and links the + * same `av*.dll` set the compositor already ships, so that is the one that gets + * packaged (see the `filter` in electron-builder.json5) and the one preferred + * here. + */ +export function ffmpegCandidates(here: string = process.cwd()): string[] { + const tag = `${process.platform}-${process.arch}`; + const exe = process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg"; + const env = process.env.OPENSCREEN_FFMPEG_PATH?.trim(); + const roots: string[] = []; + // `app` is absent when this module is imported by a test. + const appPath = (() => { + try { + return typeof app?.getAppPath === "function" ? app.getAppPath() : null; + } catch { + return null; + } + })(); + if (appPath) roots.push(appPath); + if (process.resourcesPath) roots.push(process.resourcesPath); + roots.push(here); + + const names = + process.platform === "win32" + ? [ + // Staged flat by fetch-ffmpeg.mjs, beside the av*.dll set it links + // against — the only ffmpeg the Windows installer carries. + "ffmpeg-shared.exe", + // The unpacked vendor tree, present in a dev checkout that has not + // re-run the fetch script. + path.join("ffmpeg-n8.1.2-win64-lgpl-shared", "bin", exe), + ] + : [exe]; + return [ + ...(env ? [env] : []), + ...roots.flatMap((root) => + names.map((n) => path.join(root, "electron", "native", "bin", tag, n)), + ), + ]; +} + +let cachedFfmpeg: string | null | undefined; + +/** First candidate that exists, or null when none does (callers fall back). */ +export function resolveFfmpeg(here?: string): string | null { + if (cachedFfmpeg !== undefined && here === undefined) return cachedFfmpeg; + const found = ffmpegCandidates(here).find((p) => existsSync(p)) ?? null; + if (here === undefined) cachedFfmpeg = found; + return found; +} + +/** Number of min/max blocks for a clip of `durationSec`. */ +export function peakBlockCount(durationSec: number): number { + return Math.min(MAX_PEAK_BLOCKS, Math.max(1, Math.ceil(durationSec * PEAK_BLOCKS_PER_SEC))); +} + +/** + * Folds a stream of mono int16 samples into `[min0, max0, min1, max1, ...]`. + * + * Incremental on purpose: the PCM for a 32-minute recording is 62 MB and never + * needs to exist all at once. Kept as a class rather than a closure so it holds + * only the counters it needs, not an enclosing scope. + */ +class PeakFolder { + private readonly peaks: Float32Array; + private readonly samplesPerBlock: number; + private sampleIndex = 0; + /** int16 straddling a chunk boundary: its low byte arrived, its high byte did not. */ + private pendingLowByte: number | null = null; + + constructor( + private readonly blocks: number, + totalSamples: number, + ) { + this.peaks = new Float32Array(blocks * 2); + this.samplesPerBlock = Math.max(1, totalSamples / blocks); + } + + push(chunk: Buffer): void { + let offset = 0; + if (this.pendingLowByte !== null && chunk.length > 0) { + this.addSample((chunk[0] << 8) | this.pendingLowByte); + this.pendingLowByte = null; + offset = 1; + } + const end = chunk.length - ((chunk.length - offset) % 2); + for (let i = offset; i < end; i += 2) { + this.addSample(chunk.readInt16LE(i)); + } + if (end < chunk.length) this.pendingLowByte = chunk[end]; + } + + private addSample(raw: number): void { + // readInt16LE is signed; the hand-assembled straddling sample is not. + const signed = raw > 32767 ? raw - 65536 : raw; + const value = signed / 32768; + const block = Math.min(this.blocks - 1, Math.floor(this.sampleIndex / this.samplesPerBlock)); + const lo = block * 2; + if (value < this.peaks[lo]) this.peaks[lo] = value; + if (value > this.peaks[lo + 1]) this.peaks[lo + 1] = value; + this.sampleIndex++; + } + + result(): Float32Array { + return this.peaks; + } +} + +/** Runs ffmpeg and folds its PCM straight into peaks. Never buffers the audio. */ +async function decodePeaks( + ffmpeg: string, + filePath: string, + durationSec: number, +): Promise { + const blocks = peakBlockCount(durationSec); + const folder = new PeakFolder(blocks, durationSec * PCM_RATE); + const child = spawn( + ffmpeg, + [ + "-hide_banner", + "-loglevel", + "error", + "-i", + filePath, + "-vn", + "-ac", + "1", + "-ar", + String(PCM_RATE), + "-f", + "s16le", + "-", + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + + return new Promise((resolve, reject) => { + let stderr = ""; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`ffmpeg timed out after ${DECODE_TIMEOUT_MS}ms on ${filePath}`)); + }, DECODE_TIMEOUT_MS); + + child.stdout.on("data", (c: Buffer) => folder.push(c)); + child.stderr.on("data", (c: Buffer) => { + stderr = (stderr + c.toString()).slice(-2048); + }); + child.once("error", (err) => { + clearTimeout(timer); + reject(err); + }); + child.once("close", (code) => { + clearTimeout(timer); + // A file with no audio track exits non-zero. That is not an error worth + // surfacing — it is a clip that legitimately has no waveform — but it is + // the caller's job to decide, so it still rejects, with the reason. + if (code !== 0) { + reject(new Error(`ffmpeg exited ${code}${stderr ? `: ${stderr.trim()}` : ""}`)); + return; + } + resolve(folder.result()); + }); + }); +} + +/** + * Cache key: path plus size plus mtime. A recording is immutable in practice, + * but keying on identity alone would serve stale peaks for a re-encoded or + * replaced file, and that failure is silent and confusing. + */ +async function cacheKey(filePath: string): Promise { + const info = await stat(filePath); + return createHash("sha1") + .update(`${filePath}:${info.size}:${info.mtimeMs}`) + .digest("hex") + .slice(0, 32); +} + +/** Null outside Electron (tests, any headless use): decoding still works, it + * just is not cached, rather than the whole call failing on a missing `app`. */ +function cacheDir(): string | null { + try { + return typeof app?.getPath === "function" + ? path.join(app.getPath("userData"), "audio-peaks") + : null; + } catch { + return null; + } +} + +/** + * Peaks for `filePath`, from disk when they have been computed before. + * + * The cache is what makes this feel instant: peaks for a given recording never + * change, so the ~2s decode is paid once ever rather than once per session. + * Returns null when no ffmpeg is available, so the renderer can fall back to + * its own pipelines instead of losing the waveform. + */ +export async function getAudioPeaks( + filePath: string, + durationSec: number, +): Promise { + const ffmpeg = resolveFfmpeg(); + if (!ffmpeg || !durationSec || durationSec <= 0) return null; + + const dir = cacheDir(); + const cachePath = dir ? path.join(dir, `${await cacheKey(filePath)}.f32`) : null; + if (cachePath) { + try { + const cached = await readFile(cachePath); + // A Buffer's memory may not be 4-byte aligned and its byteOffset is + // almost never 0 — copy rather than viewing it in place. + return new Float32Array( + cached.buffer.slice(cached.byteOffset, cached.byteOffset + cached.byteLength), + ); + } catch { + // Not cached yet. + } + } + + const peaks = await decodePeaks(ffmpeg, filePath, durationSec); + if (cachePath && dir) { + try { + await mkdir(dir, { recursive: true }); + await writeFile(cachePath, Buffer.from(peaks.buffer)); + } catch { + // A cache we cannot write is a slower next launch, not a failure. + } + } + return peaks; +} diff --git a/electron/preload.ts b/electron/preload.ts index dfc9db84f..898f92902 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -284,6 +284,10 @@ contextBridge.exposeInMainWorld("electronAPI", { getReadableFileInfo: (filePath: string) => { return ipcRenderer.invoke("get-readable-file-info", filePath); }, + /** Native waveform peaks, disk-cached. See electron/media/audioPeaks.ts. */ + getAudioPeaks: (filePath: string, durationSec: number) => { + return ipcRenderer.invoke("get-audio-peaks", filePath, durationSec); + }, readFileChunk: (filePath: string, offset: number, length: number) => { return ipcRenderer.invoke("read-file-chunk", filePath, offset, length); }, @@ -422,6 +426,8 @@ contextBridge.exposeInMainWorld("electronAPI", { transcribe: (request: SttTranscribeRequest): Promise => { return ipcRenderer.invoke("stt:transcribe", request) as Promise; }, + /** Stop the running transcription at its next chunk boundary. */ + cancel: (): Promise => ipcRenderer.invoke("stt:cancel") as Promise, onStatus: (callback: (event: SttStatusEvent) => void) => { const listener = (_event: unknown, payload: SttStatusEvent) => callback(payload); ipcRenderer.on("stt:status", listener); diff --git a/electron/stt/chunking.test.ts b/electron/stt/chunking.test.ts new file mode 100644 index 000000000..db19cbcfb --- /dev/null +++ b/electron/stt/chunking.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; +import { planChunks } from "./chunking"; + +const RATE = 16_000; + +/** Loud tone with silent gaps punched in at the given [startSec, endSec) ranges. */ +function toneWithSilences(durationSec: number, silences: [number, number][]): Float32Array { + const samples = new Float32Array(Math.round(durationSec * RATE)); + for (let i = 0; i < samples.length; i++) { + samples[i] = Math.sin((i / RATE) * 2 * Math.PI * 440); + } + for (const [from, to] of silences) { + samples.fill(0, Math.round(from * RATE), Math.round(to * RATE)); + } + return samples; +} + +describe("planChunks", () => { + it("covers the whole buffer with contiguous chunks", () => { + const samples = toneWithSilences(25, []); + const chunks = planChunks(samples, RATE, { targetSec: 10, searchSec: 1 }); + expect(chunks[0].startSample).toBe(0); + expect(chunks[chunks.length - 1].endSample).toBe(samples.length); + for (let i = 1; i < chunks.length; i++) { + expect(chunks[i].startSample).toBe(chunks[i - 1].endSample); + } + }); + + it("cuts inside a pause rather than on the fixed grid", () => { + // Pause at 9.5-9.9s: the ideal 10s boundary should be pulled back into it. + const samples = toneWithSilences(25, [ + [9.5, 9.9], + [19.4, 19.8], + ]); + const chunks = planChunks(samples, RATE, { targetSec: 10, searchSec: 1 }); + const cutSec = chunks[0].endSample / RATE; + expect(cutSec).toBeGreaterThanOrEqual(9.5); + expect(cutSec).toBeLessThan(9.9); + }); + + it("returns a single chunk when the recording is shorter than the target", () => { + const samples = toneWithSilences(5, []); + expect(planChunks(samples, RATE, { targetSec: 120 })).toEqual([ + { startSample: 0, endSample: samples.length }, + ]); + }); + + it("keeps the target when every frame ties, on a fully silent buffer", () => { + // Every frame scores exactly 0, so the tie-break is what decides. Keeping + // the earliest one pulled every cut back to `ideal - searchSec` — 5s chunks + // here instead of 10s, i.e. twice the requests and twice the seams, on the + // audio most likely to tie (a muted track, a gap between takes). + const samples = new Float32Array(60 * RATE); + const chunks = planChunks(samples, RATE, { targetSec: 10, searchSec: 5 }); + expect(chunks.map((c) => c.endSample / RATE)).toEqual([10, 20, 30, 40, 50, 60]); + for (const chunk of chunks) { + expect(chunk.endSample).toBeGreaterThan(chunk.startSample); + } + }); + + it("handles an empty buffer", () => { + expect(planChunks(new Float32Array(0), RATE)).toEqual([]); + }); + + it("makes progress even when the target is shorter than one energy frame", () => { + // Degenerate but reachable through the options: below one frame the scan has + // nothing to measure, and the boundary must still move or the loop spins. + const chunks = planChunks(new Float32Array(RATE), RATE, { targetSec: 0.001 }); + for (const chunk of chunks) expect(chunk.endSample).toBeGreaterThan(chunk.startSample); + expect(chunks[chunks.length - 1].endSample).toBe(RATE); + }); +}); diff --git a/electron/stt/chunking.ts b/electron/stt/chunking.ts new file mode 100644 index 000000000..5f36251da --- /dev/null +++ b/electron/stt/chunking.ts @@ -0,0 +1,132 @@ +/** + * Splits a long recording into inference-sized chunks for the STT pipeline. + * + * Why chunk at all: whisper-stt-server answers a `/inference` request only once + * it has transcribed the WHOLE upload, so a 30-minute recording was one ~10 + * minute request with no progress and no recovery — one hiccup lost everything + * (and undici's 300s `headersTimeout` killed it outright before it ever + * finished). Per-chunk requests give the caller a progress signal, a retry unit, + * and requests short enough that no transport timeout is in play. + * + * Where the cut lands matters: slicing on a fixed grid cuts mid-word, and + * whisper then mis-transcribes both halves. So the boundary is nudged to the + * quietest 20ms frame within a search window around the ideal position — a + * pause between words in practice. + * + * ponytail: energy minimum, not a real VAD. whisper.cpp ships a Silero VAD, but + * it lives behind the server's own `--vad` flag and would run per REQUEST — it + * can't tell us where to cut BEFORE we upload. A plain RMS scan over a few + * seconds is enough to find a pause and costs nothing. If a recording is so + * dense that no pause exists in the window, the cut lands at the quietest point + * anyway and one word may be split; upgrade path is an overlap + de-duplication + * pass on the seam, which is a lot more code than it is worth today. + */ + +/** One chunk of the source buffer: `[startSample, endSample)`. */ +export interface SttChunkPlan { + startSample: number; + /** Exclusive. */ + endSample: number; +} + +/** Energy is measured over frames this long; a cut lands on a frame boundary. */ +const FRAME_MS = 20; + +export interface PlanChunksOptions { + /** Ideal chunk length. Shorter = smoother progress, more per-request overhead. */ + targetSec?: number; + /** How far on either side of the ideal boundary to hunt for a pause. */ + searchSec?: number; +} + +/** + * 90s is a compromise between three pressures: progress granularity (the bar + * only moves once a chunk lands), whisper's own quality (it decodes in 30s + * windows and loses cross-chunk context at every seam, so more seams is worse), + * and `whisperServer`'s 280s per-request ceiling — 90s of audio has to + * transcribe in under that on the SLOWEST machine we care about (~0.3x realtime; + * this Vulkan box does 3.1x, i.e. ~29s per chunk). + */ +const DEFAULT_TARGET_SEC = 90; + +/** + * Index of the quietest frame start in `[from, to)`, breaking ties toward the + * frame nearest `preferred`. Both bounds are clamped by the caller; returns + * `from` when the range holds less than one full frame. + * + * The tie-break is not decoration. Digital silence — a muted track, a gap + * between takes — makes every frame in the window score exactly 0, and keeping + * the first one would pull every boundary back to `from`, i.e. shorten every + * chunk by the whole search window (90s → 87s by default, and 10s → 5s in the + * silent test case). That is extra requests and extra seams bought for nothing, + * against the very context loss `DEFAULT_TARGET_SEC` is sized to limit. + */ +function quietestFrameStart( + samples: Float32Array, + from: number, + to: number, + frameSamples: number, + preferred: number, +): number { + let bestStart = from; + let bestEnergy = Number.POSITIVE_INFINITY; + for (let start = from; start + frameSamples <= to; start += frameSamples) { + let energy = 0; + for (let i = start; i < start + frameSamples; i++) { + energy += samples[i] * samples[i]; + } + if ( + energy < bestEnergy || + (energy === bestEnergy && Math.abs(start - preferred) < Math.abs(bestStart - preferred)) + ) { + bestEnergy = energy; + bestStart = start; + } + } + return bestStart; +} + +/** + * Plan the chunk boundaries for `samples`. Chunks are contiguous and cover the + * whole buffer: `chunks[0].startSample === 0`, each `endSample` is the next + * `startSample`, and the last one ends at `samples.length`. + */ +export function planChunks( + samples: Float32Array, + sampleRate: number, + options: PlanChunksOptions = {}, +): SttChunkPlan[] { + if (samples.length === 0 || sampleRate <= 0) return []; + const frameSamples = Math.max(1, Math.round((FRAME_MS / 1000) * sampleRate)); + // One frame is the floor: a target shorter than the unit the scan works in + // would put the ideal boundary BEFORE the earliest legal cut, which is the + // only way the loop below could fail to make progress. + const targetSamples = Math.max( + frameSamples, + Math.round((options.targetSec ?? DEFAULT_TARGET_SEC) * sampleRate), + ); + const searchSamples = Math.max(0, Math.round((options.searchSec ?? 3) * sampleRate)); + + const chunks: SttChunkPlan[] = []; + let start = 0; + while (start < samples.length) { + const ideal = start + targetSamples; + // Last chunk: what's left is at most one target long, so there's nothing to cut. + if (ideal >= samples.length) { + chunks.push({ startSample: start, endSample: samples.length }); + break; + } + // The search window never reaches back to `start` (a zero-length chunk would + // loop forever) and never past the end of the buffer. + const from = Math.max(start + frameSamples, ideal - searchSamples); + const to = Math.min(samples.length, ideal + searchSamples); + // No clamping needed on either side: `ideal >= from` because `targetSamples` + // is at least one frame, and every value the scan can return lies in + // `[from, to)` — so the cut is always past `start` and inside the buffer. + const endSample = + to > from ? quietestFrameStart(samples, from, to, frameSamples, ideal) : ideal; + chunks.push({ startSample: start, endSample }); + start = endSample; + } + return chunks; +} diff --git a/electron/stt/index.test.ts b/electron/stt/index.test.ts index df94fc102..ad0eaf976 100644 --- a/electron/stt/index.test.ts +++ b/electron/stt/index.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { planChunks } from "./chunking"; import { _resetSttManagerForTests, SttManager } from "./index"; import type { SttStatusEvent, SttTranscribeResponse } from "./transcriptionContract"; @@ -87,6 +88,112 @@ describe("SttManager", () => { expect(fakeWhisperServer.transcribe).toHaveBeenCalledOnce(); }); + it("splits a long recording and shifts each chunk's timestamps to absolute time", async () => { + // Every chunk reports the same relative segment at 1.0s; correct merging + // turns those into one absolute timestamp per chunk start. + fakeWhisperServer.transcribe.mockResolvedValue({ + segments: [{ text: "hello", startSec: 1, endSec: 1.5 }], + wordSegments: [{ word: "hello", startSec: 1, endSec: 1.5 }], + detectedLanguage: "en", + backend: "whispercpp-cpu", + }); + const samples = new Float32Array(200 * 16000); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + const result = await mgr.transcribe({ samples, language: "en" }); + + const expectedOffsets = planChunks(samples, 16000).map((c) => c.startSample / 16000); + expect(expectedOffsets.length).toBeGreaterThan(1); + expect(fakeWhisperServer.transcribe).toHaveBeenCalledTimes(expectedOffsets.length); + expect(result.segments.map((s) => s.startSec)).toEqual(expectedOffsets.map((o) => o + 1)); + expect(result.wordSegments.map((w) => w.startSec)).toEqual(expectedOffsets.map((o) => o + 1)); + }); + + it("reports monotonic progress that ends on the full duration", async () => { + const sink = vi.fn<(e: SttStatusEvent) => void>(); + const samples = new Float32Array(200 * 16000); + const mgr = new SttManager(); + await mgr.init({ statusSink: sink, modelsBaseDir: "/tmp/fake-stt-models" }); + sink.mockClear(); + await mgr.transcribe({ samples, language: "en" }); + + const progress = sink.mock.calls + .map(([event]) => event) + .filter((event) => event.completedSec !== undefined); + expect(progress[0].completedSec).toBe(0); + expect(progress[progress.length - 1].completedSec).toBe(200); + for (const event of progress) expect(event.totalSec).toBe(200); + for (let i = 1; i < progress.length; i++) { + expect(progress[i].completedSec).toBeGreaterThan(progress[i - 1].completedSec ?? -1); + } + }); + + // Both spellings of "detect it for me". `"auto"` is the one the request + // contract documents, and it is truthy — which is exactly how it used to slip + // past the pin and let every chunk detect its own language. + it.each([ + ["omitted", undefined] as const, + ["auto", "auto"] as const, + ])("pins later chunks to the language detected on the first one (%s)", async (_label, language) => { + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + await mgr.transcribe({ samples: new Float32Array(200 * 16000), language }); + const languages = fakeWhisperServer.transcribe.mock.calls.map(([req]) => req.language); + expect(languages[0]).toBeUndefined(); + expect(languages.slice(1).every((l) => l === "en")).toBe(true); + }); + + it("retries a failed chunk instead of losing the whole transcription", async () => { + fakeWhisperServer.transcribe.mockRejectedValueOnce(new Error("helper died")).mockResolvedValue({ + segments: [{ text: "hello", startSec: 0, endSec: 0.5 }], + wordSegments: [{ word: "hello", startSec: 0, endSec: 0.5 }], + detectedLanguage: "en", + backend: "whispercpp-cpu", + }); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + const result = await mgr.transcribe({ samples: new Float32Array(16000), language: "en" }); + expect(result.segments).toHaveLength(1); + // start() again on the retry — the usual cause is a dead helper. + expect(fakeWhisperServer.start.mock.calls.length).toBeGreaterThan(1); + }); + + it("fails the request when a chunk never succeeds, saying how far it got", async () => { + fakeWhisperServer.transcribe.mockRejectedValue(new Error("helper wedged")); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + // 200s in, the failure is on chunk 2 of 3 — "Transcription failed" alone + // tells the user nothing about a recording this long. + await expect(mgr.transcribe({ samples: new Float32Array(200 * 16000) })).rejects.toThrow( + /transcription failed \d+s into a 200s recording \(chunk 1\/3\).*helper wedged/, + ); + }); + + it("stops at the next chunk boundary when cancelled", async () => { + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + // Cancel lands while the first chunk is in flight — the loop must not go on + // to the remaining ones, which is what left "regenerate" waiting on a run + // nobody wanted any more. + fakeWhisperServer.transcribe.mockImplementation(async () => { + mgr.cancel(); + return { + segments: [], + wordSegments: [], + detectedLanguage: "en", + backend: "whispercpp-cpu" as const, + }; + }); + const samples = new Float32Array(300 * 16000); + expect(planChunks(samples, 16000).length).toBeGreaterThan(1); + + const error = await mgr.transcribe({ samples }).catch((e: unknown) => e); + // `AbortError` by name, so the renderer treats it as "the user asked" and + // drops the job silently instead of toasting an engine failure. + expect((error as Error).name).toBe("AbortError"); + expect(fakeWhisperServer.transcribe).toHaveBeenCalledOnce(); + }); + it("shutdown() stops whisper-stt-server", async () => { const mgr = new SttManager(); await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); @@ -114,12 +221,25 @@ describe("SttManager", () => { expect(fakeWhisperServer.start).toHaveBeenCalledOnce(); }); - it("setStatusSink replaces the previous sink (last call wins)", () => { + it("fans status out to every sink, and detaching one leaves the others", async () => { const mgr = new SttManager(); - const a = vi.fn(); - const b = vi.fn(); - mgr.setStatusSink(a); - mgr.setStatusSink(b); - expect(mgr.getStatusSink()).toBe(b); + const a = vi.fn<(e: SttStatusEvent) => void>(); + const b = vi.fn<(e: SttStatusEvent) => void>(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + const detachA = mgr.addStatusSink(a); + mgr.addStatusSink(b); + await mgr.transcribe({ samples: new Float32Array(16000), language: "en" }); + expect(a).toHaveBeenCalled(); + expect(b).toHaveBeenCalled(); + + // The whole point of the Set. Two overlapping IPC requests each attach a + // sink; when the first finishes and detaches, the second must keep getting + // its own progress instead of falling silent for the rest of its run. + detachA(); + a.mockClear(); + b.mockClear(); + await mgr.transcribe({ samples: new Float32Array(16000), language: "en" }); + expect(a).not.toHaveBeenCalled(); + expect(b).toHaveBeenCalled(); }); }); diff --git a/electron/stt/index.ts b/electron/stt/index.ts index 641a3abe6..aaa597184 100644 --- a/electron/stt/index.ts +++ b/electron/stt/index.ts @@ -1,10 +1,13 @@ import path from "node:path"; import { app, type IpcMain } from "electron"; +import { planChunks } from "./chunking"; import { ensureModels, modelPaths } from "./modelManager"; import type { + SttPhraseSegment, SttStatusEvent, SttTranscribeRequest, SttTranscribeResponse, + SttWordSegment, } from "./transcriptionContract"; import { WhisperServerManager } from "./whisperServer"; @@ -13,17 +16,50 @@ import { WhisperServerManager } from "./whisperServer"; * * Workflow: * 1. `init()` spawns `whisper-stt-server` (or queues the call if it's busy). - * 2. `transcribe()` proxies the renderer's `Float32Array` through - * whisper-stt-server's HTTP `/inference`, which returns both phrase- and - * word-level segments in one pass (see whisperServer.ts). Word - * timestamps come from whisper.cpp's native DTW token timestamps - * (`t_dtw`, SMALL aheads preset, `flash_attn = false`), see + * 2. `transcribe()` splits the renderer's `Float32Array` into chunks + * (`chunking.ts`) and runs each through whisper-stt-server's HTTP + * `/inference`, which returns both phrase- and word-level segments in one + * pass (see whisperServer.ts). Word timestamps come from whisper.cpp's + * native DTW token timestamps (`t_dtw`, SMALL aheads preset, + * `flash_attn = false`), see * technical-documentation/architecture/transcription-and-captions.md § Decision rationale. * 3. `shutdown()` tears down on app quit. * - * Status events fan out via `statusSink` so the renderer can drive its + * Status events fan out to every attached sink so the renderer can drive its * "loading model" / "transcribing" indicator. + * + * Why chunked rather than one request: a 30-minute recording took ~10 minutes + * in a single `/inference` call — no progress to show, no way to recover from a + * transient failure without redoing everything, and long enough that the HTTP + * client's own header timeout killed it before whisper ever answered. Chunks + * turn that into a progress signal, a retry unit, and — via `cancel()` — the + * only point where a run in flight can be stopped at all. + * + * Chunks run SEQUENTIALLY, and that is a measured choice, not an omission: + * whisper-stt-server holds a single model context, so concurrent `/inference` + * calls don't just serialize — they get SLOWER. Two 120s chunks took 76.9s one + * after the other and 144.1s fired together (0.53x, i.e. ~1.9x slower) on this + * Vulkan backend. A client-side worker pool is therefore a pessimisation. Real + * parallelism would need several server processes, each with its own copy of + * the model resident on the GPU; that trade (VRAM + spawn cost per worker) is + * worth revisiting only if a much smaller model ever becomes the default. + */ + +/** The renderer always sends mono 16 kHz (see `extractMono16kFromVideoUrl`). */ +const SAMPLE_RATE = 16_000; + +/** Attempts per chunk before the whole transcription fails. */ +const CHUNK_ATTEMPTS = 3; + +/** + * `AbortError` by name so the renderer's `isAbortError` recognizes it as "the + * user asked for this" rather than an engine failure worth a toast. */ +function cancelledError(): Error { + const error = new Error("Transcription cancelled"); + error.name = "AbortError"; + return error; +} export interface SttManagerInitOptions { statusSink?: (event: SttStatusEvent) => void; @@ -34,21 +70,48 @@ export interface SttManagerInitOptions { export class SttManager { private readonly server = new WhisperServerManager(); private modelsBaseDir: string | null = null; - private statusSink: ((event: SttStatusEvent) => void) | null = null; + private readonly statusSinks = new Set<(event: SttStatusEvent) => void>(); private initPromise: Promise | null = null; + /** Kept from `prepare()` so a chunk retry can respawn a helper that died mid-run. */ + private modelPath: string | null = null; + /** + * Bumped by `cancel()`. The chunk loop compares it against the value it + * captured on entry, so a cancel that lands after a new run started cannot + * kill that new run. + */ + private cancelEpoch = 0; - /** Wire a sink for the renderer status channel. */ - setStatusSink(sink: ((event: SttStatusEvent) => void) | null): void { - this.statusSink = sink; + /** + * Attach a sink for the renderer status channel; returns its detach function. + * + * A SET rather than one slot: this used to be a single field that each IPC + * invocation saved and restored, so with two overlapping transcriptions the + * first to finish restored the sink captured at ITS start and left the other + * one emitting into nothing — no progress for the rest of its run, which + * reads as a hang. + */ + addStatusSink(sink: (event: SttStatusEvent) => void): () => void { + this.statusSinks.add(sink); + return () => { + this.statusSinks.delete(sink); + }; } - /** Read the currently-installed status sink (mostly for tests). */ - getStatusSink(): ((event: SttStatusEvent) => void) | null { - return this.statusSink; + private emit(event: SttStatusEvent): void { + for (const sink of this.statusSinks) sink(event); } - private emit(event: SttStatusEvent): void { - this.statusSink?.(event); + /** + * Stop the in-flight transcription at the next chunk boundary. + * + * ponytail: one epoch for the whole manager, not a handle per request. The + * pipeline runs one transcription at a time by construction (the renderer's + * queue serializes, and `WhisperServerManager` single-flights on top), so + * "cancel what is running" is the only question anyone can ask. Per-request + * tokens the day two recordings can transcribe at once. + */ + cancel(): void { + this.cancelEpoch++; } /** @@ -56,7 +119,7 @@ export class SttManager { * means the second caller just awaits the same completion. */ init(options: SttManagerInitOptions = {}): Promise { - if (options.statusSink) this.statusSink = options.statusSink; + if (options.statusSink) this.addStatusSink(options.statusSink); if (options.modelsBaseDir) this.modelsBaseDir = options.modelsBaseDir; if (!this.initPromise) { // A REJECTED init must not be cached. `prepare()` downloads a 253 MB @@ -97,23 +160,138 @@ export class SttManager { }); const paths = modelPaths(modelsDir); + this.modelPath = paths.whisper; await this.server.start({ modelPath: paths.whisper }); this.emit({ phase: "transcribe" }); } - /** Run one transcription request through whisper-stt-server. */ + /** + * Run one chunk, retrying a few times before giving up on the whole request. + * + * A failure here is usually the helper process dying (OOM, driver reset) + * rather than a bad chunk, so each retry first re-runs `server.start()` — + * idempotent when the helper is alive, a respawn when it isn't. That is what + * makes a 30-minute transcription survive a helper that dies once mid-run. + * + * What it does NOT do is salvage a chunk that fails all three attempts: the + * request fails whole and the chunks that already succeeded go with it. A + * transcript silently missing 90 seconds in the middle is worse than no + * transcript, since nothing downstream (captions, trims, the transcript + * editor) could tell the gap from a silence. The caller is told how far it + * got instead — see the wrapper in `transcribe()`. + */ + private async transcribeChunk( + samples: Float32Array, + language: string | undefined, + ): Promise>> { + let lastError: unknown; + for (let attempt = 1; attempt <= CHUNK_ATTEMPTS; attempt++) { + try { + return await this.server.transcribe({ samples, language }); + } catch (error) { + lastError = error; + if (attempt === CHUNK_ATTEMPTS) break; + if (this.modelPath) { + await this.server.start({ modelPath: this.modelPath }).catch(() => undefined); + } + await new Promise((resolve) => setTimeout(resolve, 500 * attempt)); + } + } + throw lastError instanceof Error ? lastError : new Error(String(lastError)); + } + + /** Transcribe a whole recording, chunk by chunk, reporting progress as it goes. */ async transcribe(req: SttTranscribeRequest): Promise { await this.init(); - this.emit({ phase: "transcribe" }); - const phrase = await this.server.transcribe({ - samples: req.samples, - language: req.language, - }); - const backend = phrase.backend ?? this.server.status.backend ?? "whispercpp-cpu"; + + const epoch = this.cancelEpoch; + const totalSec = req.samples.length / SAMPLE_RATE; + const chunks = planChunks(req.samples, SAMPLE_RATE); + this.emit({ phase: "transcribe", completedSec: 0, totalSec }); + + const segments: SttPhraseSegment[] = []; + const wordSegments: SttWordSegment[] = []; + let detectedLanguage: string | null = null; + let backend = this.server.status.backend ?? "whispercpp-cpu"; + // Only the first chunk auto-detects; every later chunk is forced onto the + // language it resolved, so whisper cannot flip mid-recording on a chunk + // that opens with a proper noun or a silence and "transcribe" the rest as + // another language. + // + // `"auto"` must collapse to `undefined` here rather than merely falsy + // values: `SttTranscribeRequest` documents it as the explicit way to ask + // for detection, and it is TRUTHY — left in place it makes the pin below + // unreachable for every caller that spells its intent out. + // + // This depends on the helper reporting what it RESOLVED rather than + // echoing the request, which it only does since cc781806 (30/07/2026, + // `whisper_full_lang_id()` in electron/native/whisper-stt/src/main.cpp). + // A stale `electron/native/bin//whisper-stt-server` — the directory + // is gitignored, so a dev tree keeps whatever was last staged there — + // silently reverts this to "every chunk detects on its own": the echo + // comes back as the literal "auto", the guard below rejects it, and + // nothing anywhere says why. `scripts/stage-whisper-stt.sh` refuses to + // overwrite a local binary by design, so it will not rescue you either. + // Verified end-to-end on 352s of real speech: `[undefined,"en","en","en"]`. + let language = req.language && req.language !== "auto" ? req.language : undefined; + + for (const [index, chunk] of chunks.entries()) { + // Between chunks is the only place this loop can be interrupted, and it + // is enough: a chunk is bounded by `whisperServer`'s own request ceiling. + if (this.cancelEpoch !== epoch) throw cancelledError(); + const offsetSec = chunk.startSample / SAMPLE_RATE; + const result = await this.transcribeChunk( + req.samples.subarray(chunk.startSample, chunk.endSample), + language, + ).catch((error) => { + // Say where it died. Without this the user gets "Transcription + // failed" for a 30-minute recording with no hint that 18 of those + // minutes were fine and the helper fell over at one specific spot. + // `Object.assign` rather than the `{ cause }` constructor option: the + // project targets ES2020, where that overload does not exist (see + // `BackgroundLoadError` in src/lib/wallpaper.ts for the same dance). + throw Object.assign( + new Error( + `transcription failed ${Math.round(offsetSec)}s into a ${Math.round(totalSec)}s ` + + `recording (chunk ${index + 1}/${chunks.length}): ` + + `${error instanceof Error ? error.message : String(error)}`, + ), + { cause: error }, + ); + }); + // Chunk-relative timestamps → absolute, the only thing every consumer + // (captions, transcript editor, trims) reads. + for (const segment of result.segments) { + segments.push({ + text: segment.text, + startSec: segment.startSec + offsetSec, + endSec: segment.endSec + offsetSec, + }); + } + for (const word of result.wordSegments) { + wordSegments.push({ + word: word.word, + startSec: word.startSec + offsetSec, + endSec: word.endSec + offsetSec, + confidence: word.confidence, + }); + } + if (!detectedLanguage && result.detectedLanguage && result.detectedLanguage !== "auto") { + detectedLanguage = result.detectedLanguage; + if (!language) language = detectedLanguage; + } + backend = result.backend ?? backend; + this.emit({ + phase: "transcribe", + completedSec: chunk.endSample / SAMPLE_RATE, + totalSec, + }); + } + return { - segments: phrase.segments, - wordSegments: phrase.wordSegments, - detectedLanguage: phrase.detectedLanguage, + segments, + wordSegments, + detectedLanguage: detectedLanguage ?? language ?? "auto", backend, }; } @@ -138,10 +316,11 @@ export function _resetSttManagerForTests(): void { } /** - * Wire the IPC channel. Call this from `registerIpcHandlers` so the renderer - * can `invoke("stt:transcribe", request)` and receive `SttTranscribeResponse`. - * Status events fan out on `"stt:status"` (main → renderer push), scoped to - * the calling `webContents` so two windows don't cross-talk. + * Wire the IPC channels. Call this from `registerIpcHandlers` so the renderer + * can `invoke("stt:transcribe", request)` and receive `SttTranscribeResponse`, + * and `invoke("stt:cancel")` to stop a run it no longer wants. Status events + * fan out on `"stt:status"` (main → renderer push), scoped to the calling + * `webContents` so two windows don't cross-talk. */ export function registerSttIpc(ipcMain: IpcMain): void { const manager = getSttManager(); @@ -149,8 +328,9 @@ export function registerSttIpc(ipcMain: IpcMain): void { "stt:transcribe", async (event, req: SttTranscribeRequest): Promise => { const senderId = event.sender.id; - const previous = manager.getStatusSink(); - manager.setStatusSink((statusEvent) => { + // Attach for the life of THIS request only. Overlapping requests each + // own their own sink, so neither can silence the other on the way out. + const detach = manager.addStatusSink((statusEvent) => { if (event.sender.id === senderId && !event.sender.isDestroyed()) { event.sender.send("stt:status", statusEvent); } @@ -158,8 +338,11 @@ export function registerSttIpc(ipcMain: IpcMain): void { try { return await manager.transcribe(req); } finally { - manager.setStatusSink(previous); + detach(); } }, ); + ipcMain.handle("stt:cancel", () => { + manager.cancel(); + }); } diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.ts index 85b6c7152..2fa8a88eb 100644 --- a/electron/stt/transcriptionContract.ts +++ b/electron/stt/transcriptionContract.ts @@ -50,6 +50,14 @@ export interface SttStatusEvent { totalBytes?: number; /** Which model is downloading. */ model?: "whisper"; + /** + * Seconds of audio transcribed so far, and the total for this request. Only + * when `phase === "transcribe"`. Progress is reported per CHUNK (see + * `chunking.ts`), so it steps rather than sweeps — whisper gives no + * sub-request progress signal to interpolate from. + */ + completedSec?: number; + totalSec?: number; } /** IPC request: renderer → main. */ diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.ts index 4762ebf6d..087073a2e 100644 --- a/electron/stt/whisperServer.ts +++ b/electron/stt/whisperServer.ts @@ -14,6 +14,24 @@ import { cleanupWav, writeSamplesAsWav } from "./wav"; /** whisper.cpp helper is stdio-shaped: stdin ignored, stdout/stderr captured. */ type WhisperChild = ChildProcessByStdio; +/** + * Per-request ceiling. whisper answers `/inference` only once the WHOLE upload + * is transcribed, so this bounds one chunk, not one recording. + * + * ponytail: 280s because Node's global fetch (undici) applies its own + * undocumented 300s `headersTimeout` that we cannot configure without taking a + * direct dependency on `undici` — going over it produces an opaque + * "TypeError: fetch failed" instead of anything actionable (this is exactly how + * a single 30-minute request died: killed at 300s while whisper needed 574s). + * Aborting at 280s keeps the failure OURS: named, logged with the helper's + * stderr, and retried by SttManager. The real ceiling this leaves is a machine + * so slow that one 90s chunk needs more than 280s (~0.3x realtime); the upgrade + * path is a direct `undici` dependency and + * `new Agent({ headersTimeout: 0, bodyTimeout: 0 })` as the fetch dispatcher, + * which removes the cliff entirely. + */ +const REQUEST_TIMEOUT_MS = 280_000; + /** * Owns the long-lived `whisper-stt-server` process used to recognize speech. * @@ -291,12 +309,55 @@ export class WhisperServerManager { form.set("file", blob, path.basename(opts.wavPath)); form.set("response_format", "verbose_json"); form.set("language", opts.language && opts.language !== "auto" ? opts.language : "auto"); - const res = await fetch(url, { method: "POST", body: form }); + let res: Response; + try { + res = await fetch(url, { + method: "POST", + body: form, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + } catch (error) { + // Name the failure. Node's global fetch reports BOTH a real transport + // error and its own header timeout as a bare "fetch failed", which is + // how a too-long request used to reach the user as an unactionable + // "Transcription failed" toast. + // + // The timeout wording is reserved for an ACTUAL timeout: a helper that + // died a moment ago rejects in a millisecond, and telling the reader it + // spent 280s on an over-long chunk sends them to the wrong problem + // entirely. Everything else carries its own message, plus `cause` so the + // errno survives. + // `Object.assign` rather than the `{ cause }` constructor option: the + // project targets ES2020, where that overload does not exist. + throw Object.assign( + new Error( + error instanceof Error && error.name === "TimeoutError" + ? `whisper-stt-server /inference timed out after ${Math.round(REQUEST_TIMEOUT_MS / 1000)}s ` + + `(audio chunk too long for this machine, or the helper is wedged); ` + + `stderr=${this.stderrTail.slice(-256)}` + : `whisper-stt-server /inference failed: ` + + `${error instanceof Error ? error.message : String(error)}; ` + + `stderr=${this.stderrTail.slice(-256)}`, + ), + { cause: error }, + ); + } if (!res.ok) { const text = await res.text().catch(() => ""); throw new Error(`whisper-stt-server /inference HTTP ${res.status}: ${text.slice(0, 512)}`); } - return (await res.json()) as WhisperJsonResponse; + // The same timeout still covers the body: it can fire between the headers + // and the last byte, and an unnamed `AbortError` here would read exactly + // like the "fetch failed" this whole block exists to replace. + return (await res.json().catch((error: unknown) => { + throw Object.assign( + new Error( + `whisper-stt-server /inference response was unreadable: ` + + `${error instanceof Error ? error.message : String(error)}`, + ), + { cause: error }, + ); + })) as WhisperJsonResponse; } /** Defensive number parse for `verbose_json` values that may arrive as strings. */ diff --git a/nix/package.nix b/nix/package.nix index 0ec078312..6245dcb32 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -11,7 +11,11 @@ buildNpmPackage { nodejs = nodejs_22; pname = "openscreen"; - version = "1.8.0"; + # Read, not restated. A hand-copied version is one more thing to remember at + # release time and it had already drifted two minors behind the app it names. + # (`npmDepsHash` below still has to be updated by hand — that is Nix, not a + # choice — but it fails loudly, where a stale version number never does.) + version = (lib.importJSON ../package.json).version; src = let diff --git a/scripts/fetch-ffmpeg-macos.mjs b/scripts/fetch-ffmpeg-macos.mjs index 6104a2d96..d611aa627 100644 --- a/scripts/fetch-ffmpeg-macos.mjs +++ b/scripts/fetch-ffmpeg-macos.mjs @@ -59,7 +59,10 @@ function run(cmd, args, opts = {}) { * * When the pin moves, a mirror may not carry the new version yet. That is not * a failure mode to design around — the list is tried in order and a source - * that 404s is simply skipped, with every attempt reported if none works. + * that 404s falls through to the next, with every attempt reported if none + * works. (`--retry-all-errors` does not except a 404, so a missing mirror costs + * its retries — a few seconds, once per pin bump. Narrowing that would mean + * giving up the retry on connection resets, which is the whole point.) */ const TARBALL_URLS = [ `https://ffmpeg.org/releases/ffmpeg-${VERSION}.tar.xz`, @@ -80,17 +83,28 @@ function downloadTarball(dest) { const failures = []; for (const url of TARBALL_URLS) { console.log(`Downloading ffmpeg ${VERSION} from ${new URL(url).host}…`); - // --connect-timeout bounds the fallback, and is not decoration: a throttled - // origin does not refuse, it hangs. Measured against ffmpeg.org after a few - // rapid fetches, a single connect sat for 75s before failing — times four - // attempts, that is five minutes of a build spent before the second source + // Two ceilings, because a stuck download stalls in two different ways. + // --connect-timeout covers the handshake: a throttled origin does not + // refuse, it hangs — measured against ffmpeg.org after a few rapid + // fetches, a single connect sat for 75s before failing, and times four + // attempts that is five minutes of a build spent before the second source // is even tried. 20s is far above any healthy handshake. + // --speed-limit/--speed-time covers everything AFTER the handshake, which + // --connect-timeout does not reach at all: an origin that accepts the + // connection and then trickles (or simply stops sending) never errors, so + // nothing here would retry or fall through — the job would sit until the + // runner's own six-hour limit. Aborting below 1 KiB/s sustained for 30s + // cannot fire on a link that is merely slow: the tarball is ~10 MB. const r = spawnSync( "curl", [ "-fsSL", "--connect-timeout", "20", + "--speed-limit", + "1024", + "--speed-time", + "30", "--retry", "3", "--retry-delay", @@ -103,7 +117,10 @@ function downloadTarball(dest) { { stdio: "inherit" }, ); if (r.status !== 0) { - failures.push(` ${url}\n curl exited with ${r.status}`); + // `r.error` is the case `status` cannot express: no curl on PATH gives + // `{ status: null, error: ENOENT }`, and reporting only "exited with + // null" twice sends the reader hunting a network problem. + failures.push(` ${url}\n ${r.error ? r.error.message : `curl exited with ${r.status}`}`); continue; } const actual = crypto.createHash("sha256").update(fs.readFileSync(dest)).digest("hex"); diff --git a/scripts/fetch-ffmpeg.mjs b/scripts/fetch-ffmpeg.mjs index 1c42a92ce..9a98bcf1a 100644 --- a/scripts/fetch-ffmpeg.mjs +++ b/scripts/fetch-ffmpeg.mjs @@ -457,7 +457,15 @@ async function fetchSharedDlls(tag, binDir) { fs.copyFileSync(lib, dest); } } - console.log(`Vendored ${libs.length} shared librar(ies) -> ${binDir}`); + // The shared CLI too, beside the DLLs it links against. 1 MB, against + // the 109 MB of the static exe the installer excludes — and unlike that + // one, this is spawned at runtime: electron/media/audioPeaks.ts decodes + // waveform peaks with it, ~6x faster than either browser pipeline and + // off the UI process. Named apart from `ffmpeg.exe` on purpose, so the + // packager's `!win32-*/ffmpeg.exe` rule keeps dropping the static build + // while this one ships under the plain `win32-*/*` include. + fs.copyFileSync(exe, path.join(binDir, "ffmpeg-shared.exe")); + console.log(`Vendored ${libs.length} shared librar(ies) + ffmpeg-shared.exe -> ${binDir}`); } if (sdkDest) vendorFfmpegSdk(tmp, sdkDest); console.log("LGPL verified: safe to ship with an MIT app."); diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index 0490f3105..5ac70471a 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -1042,7 +1042,22 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ // words (inside a skip range) render red+strikethrough with a hover bin. // `isCue` highlights the word the playback head is currently inside with // an accent underline (matches axcut's `word.transcript-word.cue` rule). -function TranscriptWord({ +// +// `memo` for the same reason as `TranscriptClipBlock`, one level down — and it +// is the level that actually decides the cost. The block's memo assumes +// `cueWordId` moves "a few times per second, not sixty", which holds for +// playback at 1x and NOT for a scrub: dragging the playhead crosses many words +// per frame, so `cueWordId` changes on essentially every frame and the block +// re-renders. Without a memo here that meant re-rendering one component per +// transcript word, every frame. Measured over a 40-frame scrub in jsdom: +// 19.6 ms/frame at 100 words, 132.6 ms at 4501 (a real 30-minute recording) — +// the cost was simply proportional to transcript length. With the memo only +// the two words whose `isCue` actually flipped re-render. +// +// This holds because every other prop is referentially stable across a +// playhead tick: `cw` comes from the memoised `sections`, `target` from a +// `useMemo`, and both callbacks from `useCallback`s that do not depend on time. +const TranscriptWord = memo(function TranscriptWord({ cw, isCue, target, @@ -1198,7 +1213,7 @@ function TranscriptWord({ ) : null} ); -} +}); // ─── Caret / selection helpers ──────────────────────────────────── // Ponytail port of axcut's findCollapsedDeletionWordId. The non-collapsed diff --git a/src/components/ai-edition/TranscriptionStatus.tsx b/src/components/ai-edition/TranscriptionStatus.tsx index 5e17bd276..5d74dd55f 100644 --- a/src/components/ai-edition/TranscriptionStatus.tsx +++ b/src/components/ai-edition/TranscriptionStatus.tsx @@ -9,7 +9,10 @@ import { Loader2 } from "lucide-react"; import { useScopedT } from "@/contexts/I18nContext"; -import type { AssetTranscriptionView } from "@/lib/ai-edition/transcription/status"; +import { + type AssetTranscriptionView, + progressFraction, +} from "@/lib/ai-edition/transcription/status"; /** Human-readable state of one asset's transcript, in the user's language. */ export function useTranscriptionLabel(): (view: AssetTranscriptionView) => string { @@ -20,8 +23,21 @@ export function useTranscriptionLabel(): (view: AssetTranscriptionView) => strin return t("mediaStage.transcriptReady"); case "queued": return t("mediaStage.pendingTranscription"); - case "running": - return t("mediaStage.transcribing"); + case "running": { + // The first-run model download is a 253 MB wait with nothing else on + // screen to explain it, so it gets its own words rather than being + // labelled "Transcribing" — this is the phase most often mistaken for + // a hang, and the one `phase` was carried through the store for. + if (view.phase === "loading-model") return t("mediaStage.downloadingModel"); + // Transcribing a long recording runs for minutes. A bare + // "Transcribing…" for that whole time is indistinguishable from a + // hang, so append the percentage as soon as the main process reports + // chunk progress — and only then (see `TranscriptionProgressBar`). + const fraction = progressFraction(view.progress); + return fraction === null + ? t("mediaStage.transcribing") + : `${t("mediaStage.transcribing")} ${Math.round(fraction * 100)}%`; + } case "empty": return t("mediaStage.noSpeechDetected"); case "failed": @@ -79,3 +95,45 @@ export function TranscriptionStatusDot({ /> ); } + +/** + * Determinate progress bar for a running transcription. Renders nothing unless + * the job actually reports measurable progress — a job that is queued, + * extracting audio or downloading the model has no meaningful fraction, and a + * bar pinned at 0% reads as "stuck" where the spinner reads as "working". + * + * It owns its own spacing on purpose. A wrapper in the caller cannot render + * itself away with the bar, and in a flex column (where margins don't collapse) + * an empty one still takes its margins — 8px of dead gap under every media card + * that isn't transcribing. + */ +export function TranscriptionProgressBar({ view }: { view: AssetTranscriptionView }) { + const label = useTranscriptionLabel(); + const fraction = view.status === "running" ? progressFraction(view.progress) : null; + if (fraction === null) return null; + return ( +
+
+
+ ); +} diff --git a/src/components/ai-edition/v4/MediaStage.tsx b/src/components/ai-edition/v4/MediaStage.tsx index 8c4bca65c..a9933bc8f 100644 --- a/src/components/ai-edition/v4/MediaStage.tsx +++ b/src/components/ai-edition/v4/MediaStage.tsx @@ -14,7 +14,11 @@ import type { AssetTranscriptionView, } from "@/lib/ai-edition/transcription/status"; import { formatBytes } from "@/utils/formatBytes"; -import { TranscriptionStatusDot, useTranscriptionLabel } from "../TranscriptionStatus"; +import { + TranscriptionProgressBar, + TranscriptionStatusDot, + useTranscriptionLabel, +} from "../TranscriptionStatus"; import styles from "./EditorShellV4.module.css"; const ASSET_MIME = "application/x-axcut-asset"; @@ -266,8 +270,36 @@ export function MediaStage() { {transcriptionLabel(selectedTranscription)} + {/* The language whisper resolved on the first chunk, which every later + chunk was then pinned to. It had a pill in SourceTranscriptModal, + but that lives under LeftPanel's `MediaPane` — and the only mount + site is ``, a literal, so it renders + `ChatStripPanel` and nothing else. The value was reaching the + document and being displayed nowhere. It belongs next to + "Regenerate as" below in any case: that selector is the control + you set BECAUSE of what was detected. */} + {transcript?.language && transcript.language !== "auto" ? ( + + {t("mediaStage.detectedLanguage", { language: transcript.language })} + + ) : null}
+ {/* Renders itself away — spacing included — unless the run reports progress. */} + + {selectedTranscription.failure ? (

{ if (!peaks || peaks.length === 0 || !assetDurationSec) return null; const totalBlocks = Math.floor(peaks.length / 2); @@ -518,6 +520,11 @@ export function V4Timeline({ const startScrub = useCallback( (e: ReactPointerEvent) => { if (e.button !== 0) return; + // Media has no playhead rendered, so there is nothing to scrub. Guarded + // here rather than at the three call sites: seeking an invisible cursor + // would still move `currentTimeSec`, i.e. silently reposition the Edit + // tab's preview from a screen that shows no time at all. + if (!showLanes) return; const target = e.target as HTMLElement; if (target.closest("[data-clip-id]") || target.closest(`.${styles.lanePill}`)) return; tl.clearSelection(); @@ -543,7 +550,7 @@ export function V4Timeline({ window.addEventListener("pointermove", move); window.addEventListener("pointerup", up); }, - [seekToClientX, tl, setCurrentTime], + [seekToClientX, tl, setCurrentTime, showLanes], ); const [activePillDrag, setActivePillDrag] = useState<{ @@ -718,6 +725,9 @@ export function V4Timeline({ useEffect(() => { const el = tracksRef.current; if (!el) return; + // Media shows no zoom window, so leave the wheel alone there: a zoom with + // no control to undo it and no ruler reading to explain it is a trap. + if (!showLanes) return; const onWheelNative = (e: WheelEvent) => { const r = el.getBoundingClientRect(); const viewportPct = Math.min(1, Math.max(0, (e.clientX - r.left) / r.width)); @@ -751,7 +761,7 @@ export function V4Timeline({ }; el.addEventListener("wheel", onWheelNative, { passive: false }); return () => el.removeEventListener("wheel", onWheelNative); - }, []); + }, [showLanes]); // Track the tracks' content width for the ruler. .tlTracks and .tlRulerRow // carry the same horizontal padding and the tracks' scrollbar is hidden, so @@ -1323,7 +1333,19 @@ export function V4Timeline({ ) : ( -

+ // Media is an ARRANGING surface: add, remove, reorder. Nothing here + // plays or edits, so the transport, the scroll hints, the zoom nav and + // the playhead are absent rather than inert — this caption is the whole + // header, and it centres because it is alone in the row. +
{t("toolbar.arrangeClips")} @@ -1332,23 +1354,27 @@ export function V4Timeline({
)} - -
- - Shift+Scroll {t("labels.pan")} - - - Ctrl+Scroll {t("labels.zoom")} - -
+ {showLanes ? ( + <> + +
+ + Shift+Scroll {t("labels.pan")} + + + Ctrl+Scroll {t("labels.zoom")} + +
+ + ) : null}
{/* Ruler + tracks share one relative wrapper so a single playhead overlay @@ -1516,41 +1542,48 @@ export function V4Timeline({ {/* Single playhead overlay spanning the ruler + tracks: fixed vertically (a cursor, so it doesn't scroll with the lanes) and sharing the exact same zoom/pan transform + width as the canvases, so its line stays - continuous from the ruler down through the clips and its head aligns. */} - + continuous from the ruler down through the clips and its head aligns. + Edit only: there is no playback to follow on the Media surface. */} + {showLanes ? ( + + ) : null} -
-
-
startNavDrag("pan", e)} - /> -
startNavDrag("left", e)} - > - -
-
startNavDrag("right", e)} - > - + {/* Zoom/pan window. Edit only: arranging clips needs the whole timeline + on screen at once, and there is nothing to zoom INTO without lanes. */} + {showLanes ? ( +
+
+
startNavDrag("pan", e)} + /> +
startNavDrag("left", e)} + > + +
+
startNavDrag("right", e)} + > + +
-
+ ) : null}
); } diff --git a/src/hooks/useAudioPeaks.test.ts b/src/hooks/useAudioPeaks.test.ts new file mode 100644 index 000000000..77a850b78 --- /dev/null +++ b/src/hooks/useAudioPeaks.test.ts @@ -0,0 +1,87 @@ +// Two properties that decide whether a long recording's waveform appears +// quickly or not at all: which pipeline a file is routed to, and how many times +// it is decoded. +import { act, cleanup, renderHook, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useAudioPeaks } from "./useAudioPeaks"; + +const streamingCalls = vi.fn(); +const inMemoryCalls = vi.fn(); + +vi.mock("./streamingAudioPeaks", () => ({ + computePeaksFromFileStreaming: async () => { + streamingCalls(); + return new Float32Array([0, 1]); + }, +})); + +vi.mock("@/lib/exporter/localSourceFile", () => ({ + materializeLocalSourceFile: async (_url: string, name: string) => ({ name }), + releaseLocalSourceFile: () => {}, +})); + +vi.mock("@/lib/exporter/streamingDecoder", () => ({ + loadFileAsArrayBuffer: async () => { + inMemoryCalls(); + return { data: new ArrayBuffer(8) }; + }, +})); + +// A 68 MB file — comfortably under the 256 MB in-memory threshold, which is +// exactly why routing on file size sent a 32-minute recording down the +// decode-everything path. +const FILE_BYTES = 68 * 1024 * 1024; +const THIRTY_TWO_MINUTES = 1951; + +beforeEach(() => { + streamingCalls.mockClear(); + inMemoryCalls.mockClear(); + (window as unknown as { electronAPI: unknown }).electronAPI = { + getReadableFileInfo: async () => ({ success: true, size: FILE_BYTES }), + }; +}); + +afterEach(cleanup); + +describe("useAudioPeaks", () => { + it("streams a long recording instead of decoding it whole", async () => { + const { result } = renderHook(() => useAudioPeaks("/tmp/long-a.mp4", THIRTY_TWO_MINUTES)); + await waitFor(() => expect(result.current).not.toBeNull()); + expect(streamingCalls).toHaveBeenCalledOnce(); + // The whole point: 68 MB on disk is 656 MB decoded, so this must NOT be + // the path that reads the file and hands it to decodeAudioData. + expect(inMemoryCalls).not.toHaveBeenCalled(); + }); + + it("keeps decoding short clips in memory", async () => { + // Only the ROUTE is asserted: the in-memory path then needs a real + // AudioContext and a Worker, neither of which jsdom has. + renderHook(() => useAudioPeaks("/tmp/short-a.mp4", 20)); + await waitFor(() => expect(inMemoryCalls).toHaveBeenCalled()); + expect(streamingCalls).not.toHaveBeenCalled(); + }); + + it("decodes a file once, however many clips mount it and however often", async () => { + const url = "/tmp/long-b.mp4"; + // Three clips of the same asset, mounted together. + const a = renderHook(() => useAudioPeaks(url, THIRTY_TWO_MINUTES)); + const b = renderHook(() => useAudioPeaks(url, THIRTY_TWO_MINUTES)); + const c = renderHook(() => useAudioPeaks(url, THIRTY_TWO_MINUTES)); + await waitFor(() => expect(a.result.current).not.toBeNull()); + await waitFor(() => expect(b.result.current).not.toBeNull()); + await waitFor(() => expect(c.result.current).not.toBeNull()); + expect(streamingCalls).toHaveBeenCalledOnce(); + + // Unmount everything — this is a Media↔Edit tab switch — and come back. + // With the cache scoped to a component ref, this re-decoded the whole + // recording every single time. + act(() => { + a.unmount(); + b.unmount(); + c.unmount(); + }); + const again = renderHook(() => useAudioPeaks(url, THIRTY_TWO_MINUTES)); + await waitFor(() => expect(again.result.current).not.toBeNull()); + expect(streamingCalls).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/hooks/useAudioPeaks.ts b/src/hooks/useAudioPeaks.ts index daa0abf09..da7512e2c 100644 --- a/src/hooks/useAudioPeaks.ts +++ b/src/hooks/useAudioPeaks.ts @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState } from "react"; +import { useEffect, useState } from "react"; import { materializeLocalSourceFile, releaseLocalSourceFile } from "@/lib/exporter/localSourceFile"; import { MAX_IN_MEMORY_SOURCE_BYTES } from "@/lib/exporter/sourceFileLimits"; import { loadFileAsArrayBuffer } from "@/lib/exporter/streamingDecoder"; @@ -62,17 +62,59 @@ function computePeaksInWorker( } /** - * Routes to the right peaks pipeline for the source size. Small/remote files - * use the original decodeAudioData → worker path. Local recordings above the - * in-memory limit stream instead: the file is materialized into OPFS (reused by - * the export afterwards) and its audio is decoded chunk-by-chunk into peaks, so - * the whole recording is never held in memory. + * Bytes one second of decoded audio occupies in an `AudioBuffer`: Float32, + * stereo, 44.1 kHz. An estimate on purpose — it picks the pipeline, before + * anything has been decoded and while the real rate is still unknown. */ -async function computePeaksForUrl(videoUrl: string, signal?: AbortSignal): Promise { +const DECODED_BYTES_PER_SEC = 44_100 * 2 * 4; + +/** + * Routes to the right peaks pipeline. Small/remote files use the original + * decodeAudioData → worker path. Recordings too big to hold decoded stream + * instead: the file is materialized into OPFS (reused by the export afterwards) + * and its audio is decoded chunk-by-chunk into peaks, so the whole recording is + * never held in memory. + * + * "Too big" is measured on the DECODED size, estimated from duration — not on + * the file's bytes, which is close to meaningless here and is what this used to + * compare. Compression ratio is the entire point of a screen recording: a + * 32-minute capture is 68 MB on disk and 656 MB decoded, and the in-memory path + * then `slice()`s every channel again for the worker transfer. That is ~1.4 GB + * of transient allocation to draw 400 bars, and it sat comfortably under a + * 256 MB *file* threshold — so the streaming path built for exactly this case + * never ran. (`ffmpeg -vn -f null` decodes the same track in 2.1s: that is the + * floor all that allocation was being piled onto.) + */ +async function computePeaksForUrl( + videoUrl: string, + signal?: AbortSignal, + durationSec?: number, +): Promise { const isRemoteUrl = /^(https?:|blob:|data:)/i.test(videoUrl); + + // Native first. Both browser pipelines below decode the whole track in + // Chromium — 12s on a 32-minute recording, whichever one runs — where ffmpeg + // in the main process takes ~2s and caches the result on disk, so the second + // time it is free. Anything that stops this from working (no ffmpeg staged, + // an unapproved path, a clip with no audio) falls through rather than + // dropping the waveform. + if (!isRemoteUrl && durationSec && window.electronAPI?.getAudioPeaks) { + try { + const native = await window.electronAPI.getAudioPeaks(videoUrl, durationSec); + if (native.success && native.peaks && native.peaks.length > 0) return native.peaks; + } catch { + // Fall through to the browser pipelines. + } + } + if (!isRemoteUrl && window.electronAPI?.getReadableFileInfo) { const info = await window.electronAPI.getReadableFileInfo(videoUrl); - if (info.success && typeof info.size === "number" && info.size > MAX_IN_MEMORY_SOURCE_BYTES) { + const decodedBytes = (durationSec ?? 0) * DECODED_BYTES_PER_SEC; + if ( + info.success && + ((typeof info.size === "number" && info.size > MAX_IN_MEMORY_SOURCE_BYTES) || + decodedBytes > MAX_IN_MEMORY_SOURCE_BYTES) + ) { const filename = (videoUrl.split(/[\\/]/).pop() || "video").replace(/^file:/, ""); // signal also aborts the OPFS copy (unless the export shares it). const file = await materializeLocalSourceFile(videoUrl, filename, { signal }); @@ -89,16 +131,49 @@ async function computePeaksForUrl(videoUrl: string, signal?: AbortSignal): Promi return computePeaksInWorker(audioBuffer, signal); } +/** + * Peaks describe a FILE, so they are cached per file, at module scope. + * + * This used to be a `useRef` Map, i.e. one cache per mounted component. Peaks + * for a 32-minute recording cost seconds and (before the routing fix above) a + * gigabyte-plus of transient allocation, and that was paid again for every clip + * of the same asset, and again from scratch on every remount — switching + * Media↔Edit re-decoded the whole recording, which is what "the waveform takes + * ages to appear" actually was. + * + * `inFlight` is the other half: N clips of one asset mounting together must + * share a single decode instead of racing N of them. + */ +const peaksCache = new Map(); +const peaksInFlight = new Map>(); + +function loadPeaks(videoUrl: string, durationSec?: number): Promise { + const existing = peaksInFlight.get(videoUrl); + if (existing) return existing; + // Deliberately NOT wired to any component's AbortSignal: the work is shared, + // so one subscriber unmounting must not cancel it for the others. An unmount + // drops the result instead — and the cache means the next mount is free. + const promise = computePeaksForUrl(videoUrl, undefined, durationSec) + .then((p) => { + peaksCache.set(videoUrl, p); + return p; + }) + .finally(() => { + peaksInFlight.delete(videoUrl); + }); + peaksInFlight.set(videoUrl, promise); + return promise; +} + /** * Decodes audio from `videoUrl` into paired [min, max] peaks (length = 2 * N * blocks). Returns `null` while decoding, and stays `null` on no audio track or - * decode failure (silent degradation). Results are cached in a ref scoped to the - * hook instance, so they survive re-renders and waveform toggles but not unmount. + * decode failure (silent degradation). `durationSec` only picks the pipeline + * (see `computePeaksForUrl`); omitting it is safe, just slower on long files. */ -export function useAudioPeaks(videoUrl?: string): Float32Array | null { - const cacheRef = useRef>(new Map()); +export function useAudioPeaks(videoUrl?: string, durationSec?: number): Float32Array | null { const [peaks, setPeaks] = useState(() => - videoUrl ? (cacheRef.current.get(videoUrl) ?? null) : null, + videoUrl ? (peaksCache.get(videoUrl) ?? null) : null, ); useEffect(() => { @@ -107,7 +182,7 @@ export function useAudioPeaks(videoUrl?: string): Float32Array | null { return; } - const cached = cacheRef.current.get(videoUrl); + const cached = peaksCache.get(videoUrl); if (cached) { setPeaks(cached); return; @@ -115,27 +190,21 @@ export function useAudioPeaks(videoUrl?: string): Float32Array | null { setPeaks(null); let cancelled = false; - const controller = new AbortController(); - (async () => { - try { - const p = await computePeaksForUrl(videoUrl, controller.signal); - if (cancelled) return; - cacheRef.current.set(videoUrl, p); - setPeaks(p); - } catch (err) { - // AbortError means the effect cleaned up, so no state update needed. + loadPeaks(videoUrl, durationSec) + .then((p) => { + if (!cancelled) setPeaks(p); + }) + .catch((err: unknown) => { if (err instanceof DOMException && err.name === "AbortError") return; // No audio track or unsupported format: degrade to no waveform, but log // so an unexpectedly-missing waveform is diagnosable. console.warn("useAudioPeaks: could not decode audio for waveform:", err); if (!cancelled) setPeaks(null); - } - })(); + }); return () => { cancelled = true; - controller.abort(); }; }, [videoUrl]); diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index 255bad4c1..81f5fe6ca 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "النص فارغ.", "notGeneratedHint": "لم يُنشأ بعد — اختر لغة وانقر على إعادة الإنشاء.", "transcribing": "جارٍ النسخ", + "downloadingModel": "جارٍ تنزيل نموذج الكلام", "transcribingEllipsis": "جارٍ النسخ…", "pendingTranscription": "بانتظار النسخ", "transcriptionFailed": "فشل النسخ", diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index 2e270ffb3..ae85f0954 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -134,6 +134,7 @@ "close": "Close", "transcriptReady": "Transcript ready", "transcribing": "Transcribing", + "downloadingModel": "Downloading speech model", "transcribingEllipsis": "Transcribing…", "pendingTranscription": "Pending transcription", "transcriptionFailed": "Transcription failed", diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index 8f5ba5bec..a5773c4b7 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "La transcripción está vacía.", "notGeneratedHint": "Aún no generada — elige un idioma y haz clic en regenerar.", "transcribing": "Transcribiendo", + "downloadingModel": "Descargando modelo de voz", "transcribingEllipsis": "Transcribiendo…", "pendingTranscription": "Transcripción pendiente", "transcriptionFailed": "Error de transcripción", diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index 5cda995b1..ba0afda62 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "La transcription est vide.", "notGeneratedHint": "Pas encore générée — choisissez une langue et cliquez sur régénérer.", "transcribing": "Transcription en cours", + "downloadingModel": "Téléchargement du modèle vocal", "transcribingEllipsis": "Transcription en cours…", "pendingTranscription": "Transcription en attente", "transcriptionFailed": "Échec de la transcription", diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index ecc998b7d..926b5638c 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "La trascrizione è vuota.", "notGeneratedHint": "Non ancora generata — scegli una lingua e clicca su rigenera.", "transcribing": "Trascrizione in corso", + "downloadingModel": "Download del modello vocale", "transcribingEllipsis": "Trascrizione in corso…", "pendingTranscription": "Trascrizione in attesa", "transcriptionFailed": "Trascrizione non riuscita", diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index 615cf15a5..7f83fcc59 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "文字起こしは空です。", "notGeneratedHint": "まだ生成されていません — 言語を選んで再生成をクリックしてください。", "transcribing": "文字起こし中", + "downloadingModel": "音声モデルをダウンロード中", "transcribingEllipsis": "文字起こし中…", "pendingTranscription": "文字起こし待機中", "transcriptionFailed": "文字起こしに失敗しました", diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index 190b9af0b..6e2bc4659 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "대본이 비어 있습니다.", "notGeneratedHint": "아직 생성되지 않음 — 언어를 선택하고 재생성을 클릭하세요.", "transcribing": "받아쓰는 중", + "downloadingModel": "음성 모델 다운로드 중", "transcribingEllipsis": "받아쓰는 중…", "pendingTranscription": "받아쓰기 대기 중", "transcriptionFailed": "받아쓰기 실패", diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index b7d90914d..bf26c473c 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "A transcrição está vazia.", "notGeneratedHint": "Ainda não gerada — escolha um idioma e clique em regenerar.", "transcribing": "Transcrevendo", + "downloadingModel": "Baixando modelo de voz", "transcribingEllipsis": "Transcrevendo…", "pendingTranscription": "Transcrição pendente", "transcriptionFailed": "Falha na transcrição", diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index b50d5a986..ccd84010e 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "Транскрипт пуст.", "notGeneratedHint": "Ещё не создан — выберите язык и нажмите «Пересоздать».", "transcribing": "Расшифровка", + "downloadingModel": "Загрузка речевой модели", "transcribingEllipsis": "Расшифровка…", "pendingTranscription": "Ожидает расшифровки", "transcriptionFailed": "Ошибка расшифровки", diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index 0c1d77344..cc11d3c00 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "Metin dökümü boş.", "notGeneratedHint": "Henüz oluşturulmadı — bir dil seçin ve yeniden oluştur'a tıklayın.", "transcribing": "Metne dökülüyor", + "downloadingModel": "Konuşma modeli indiriliyor", "transcribingEllipsis": "Metne dökülüyor…", "pendingTranscription": "Metne dökme bekliyor", "transcriptionFailed": "Metne dökme başarısız oldu", diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index 0ad0a044c..ba7acfc61 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "Bản ghi lời thoại trống.", "notGeneratedHint": "Chưa được tạo — chọn ngôn ngữ và nhấp vào tạo lại.", "transcribing": "Đang phiên âm", + "downloadingModel": "Đang tải mô hình giọng nói", "transcribingEllipsis": "Đang phiên âm…", "pendingTranscription": "Đang chờ phiên âm", "transcriptionFailed": "Phiên âm thất bại", diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index 35610a0b6..d4dfe1792 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "转录内容为空。", "notGeneratedHint": "尚未生成 — 选择语言并点击重新生成。", "transcribing": "正在转录", + "downloadingModel": "正在下载语音模型", "transcribingEllipsis": "正在转录…", "pendingTranscription": "等待转录", "transcriptionFailed": "转录失败", diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index c3e7e55e4..46017c8ab 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "逐字稿為空。", "notGeneratedHint": "尚未產生 — 選擇語言並點擊重新產生。", "transcribing": "轉錄中", + "downloadingModel": "正在下載語音模型", "transcribingEllipsis": "轉錄中…", "pendingTranscription": "等待轉錄", "transcriptionFailed": "轉錄失敗", diff --git a/src/lib/ai-edition/document/transcribe.ts b/src/lib/ai-edition/document/transcribe.ts index 460828b60..0c0a23aa8 100644 --- a/src/lib/ai-edition/document/transcribe.ts +++ b/src/lib/ai-edition/document/transcribe.ts @@ -9,9 +9,20 @@ import { toFileUrl } from "@/components/video-editor/projectPersistence"; import { extractMono16kFromVideoUrl, transcribeMono16kToSegments } from "@/lib/captioning"; import type { AxcutDocument, AxcutTranscript, AxcutTranscriptSegment, AxcutWord } from "../schema"; +/** + * What the caller can show while a transcription runs. `completedSec` / + * `totalSec` arrive only during `"transcribing"`, once the main process starts + * landing chunks — until then the phase alone is all there is to show. + */ +export interface TranscribeStatus { + phase: "extracting-audio" | "loading-model" | "transcribing"; + completedSec?: number; + totalSec?: number; +} + export interface TranscribeAssetOptions { language?: string; - onStatus?: (status: string) => void; + onStatus?: (status: TranscribeStatus) => void; signal?: AbortSignal; } @@ -27,12 +38,12 @@ export async function transcribeAsset( const videoUrl = toFileUrl(asset.originalPath); - options.onStatus?.("extracting-audio"); + options.onStatus?.({ phase: "extracting-audio" }); const audioResult = await extractMono16kFromVideoUrl(videoUrl, { signal: options.signal, }); - options.onStatus?.("transcribing"); + options.onStatus?.({ phase: "transcribing" }); // Only pass `language` to the worker when the caller forced a specific // code. `"auto"` (or any falsy value) leaves Whisper to detect from // the audio. The pipeline tags every chunk with the language it used @@ -44,6 +55,15 @@ export async function transcribeAsset( trimRegions: [], signal: options.signal, language: forcedLanguage, + // Forward the main process's per-chunk progress. Without this the status + // callback only ever fired the two coarse phases above, so a 30-minute + // recording showed one static "transcribing" for ten minutes. + onStatus: (status) => + options.onStatus?.({ + phase: status.phase === "model" ? "loading-model" : "transcribing", + completedSec: status.completedSec, + totalSec: status.totalSec, + }), }); const segments: AxcutTranscriptSegment[] = []; diff --git a/src/lib/ai-edition/store/transcriptionStore.ts b/src/lib/ai-edition/store/transcriptionStore.ts index 9d765afc3..1cee92d11 100644 --- a/src/lib/ai-edition/store/transcriptionStore.ts +++ b/src/lib/ai-edition/store/transcriptionStore.ts @@ -40,6 +40,7 @@ import { type TranscriptGate, type TranscriptionFailure, type TranscriptionPhase, + type TranscriptionProgress, transcriptRelevantAssetIds, } from "../transcription/status"; import { useProjectStore } from "./projectStore"; @@ -50,6 +51,8 @@ export interface TranscriptionJob { * finishes after the user asked for another one cannot clear its successor. */ runId?: number; phase?: TranscriptionPhase; + /** Chunk progress while transcribing; absent until the first chunk lands. */ + progress?: TranscriptionProgress; /** `"auto"` unless the user forced a language from the media card. */ language: string; failure?: TranscriptionFailure; @@ -273,7 +276,7 @@ function failRemainingQueue(projectId: string, failure: TranscriptionFailure): v for (const assetId of queued) { const job = jobs[assetId]; if (job?.status !== "queued") continue; - jobs[assetId] = { ...job, status: "failed", phase: undefined, failure }; + jobs[assetId] = { ...job, status: "failed", phase: undefined, progress: undefined, failure }; } return { jobs }; }); @@ -365,7 +368,19 @@ async function runJob(assetId: string, job: TranscriptionJob): Promise { const transcript = await transcribeAsset(doc, assetId, { language: job.language, signal: controller.signal, - onStatus: (phase) => patchJob(assetId, runId, { phase: phase as TranscriptionPhase }), + // `TranscribeStatus` and `TranscriptionPhase` are the same vocabulary on + // purpose (see status.ts), so this no longer needs a cast. `progress` + // only arrives during "transcribing"; carrying it through undefined the + // rest of the time is what lets the UI fall back to a spinner instead + // of a bar frozen at 0%. + onStatus: (status) => + patchJob(assetId, runId, { + phase: status.phase, + progress: + status.completedSec !== undefined && status.totalSec !== undefined + ? { completedSec: status.completedSec, totalSec: status.totalSec } + : undefined, + }), }); if (controller.signal.aborted) { dropJob(assetId, runId); @@ -400,7 +415,7 @@ async function runJob(assetId: string, job: TranscriptionJob): Promise { } if (!isCurrentRun(assetId, runId)) return; // superseded by a newer request const failure = classifyTranscriptionError(error); - patchJob(assetId, runId, { status: "failed", phase: undefined, failure }); + patchJob(assetId, runId, { status: "failed", phase: undefined, progress: undefined, failure }); flushSettleWaiters(assetId); await persistPermanentFailure(projectId, assetId, failure); // A transient failure is about the ENGINE, not about this media: the model diff --git a/src/lib/ai-edition/transcription/status.test.ts b/src/lib/ai-edition/transcription/status.test.ts index adc21cd35..00d8864e1 100644 --- a/src/lib/ai-edition/transcription/status.test.ts +++ b/src/lib/ai-edition/transcription/status.test.ts @@ -5,6 +5,7 @@ import { classifyTranscriptionError, deriveAssetStatus, isPermanentFailure, + progressFraction, resolveTranscriptGate, transcriptHasSpeech, transcriptRelevantAssetIds, @@ -93,6 +94,30 @@ describe("deriveAssetStatus", () => { expect(derived).toEqual({ assetId: "asset_1", status: "running", phase: "transcribing" }); }); + it("carries chunk progress through to the view", () => { + const derived = deriveAssetStatus({ + assetId: "asset_1", + job: { + status: "running", + phase: "transcribing", + progress: { completedSec: 90, totalSec: 300 }, + }, + }); + expect(derived.progress).toEqual({ completedSec: 90, totalSec: 300 }); + expect(progressFraction(derived.progress)).toBeCloseTo(0.3); + }); + + it("leaves progress absent while nothing measurable is running", () => { + // Audio extraction and the model download have no fraction to report; the + // UI must get `undefined` so it keeps the spinner instead of a 0% bar. + const derived = deriveAssetStatus({ + assetId: "asset_1", + job: { status: "running", phase: "extracting-audio" }, + }); + expect(derived.progress).toBeUndefined(); + expect(progressFraction(derived.progress)).toBeNull(); + }); + it("reports ready from the document, with no job at all", () => { expect( deriveAssetStatus({ assetId: "asset_1", transcript: transcript("asset_1", ["hello"]) }) diff --git a/src/lib/ai-edition/transcription/status.ts b/src/lib/ai-edition/transcription/status.ts index fced8390a..f05454b66 100644 --- a/src/lib/ai-edition/transcription/status.ts +++ b/src/lib/ai-edition/transcription/status.ts @@ -17,8 +17,28 @@ export interface TranscriptionFailure { message: string; } -/** Which half of the pipeline a running job is in (mirrors `TranscribeAssetOptions.onStatus`). */ -export type TranscriptionPhase = "extracting-audio" | "transcribing"; +/** Which part of the pipeline a running job is in (mirrors `TranscribeAssetOptions.onStatus`). */ +export type TranscriptionPhase = "extracting-audio" | "loading-model" | "transcribing"; + +/** + * How far a running transcription has got, in seconds of audio. + * + * Only the `"transcribing"` phase reports this, and only once the main process + * starts landing chunks — audio extraction and the first-run model download + * have nothing to measure. Absent means "running, no measurable progress", not + * "zero": the UI must fall back to an indeterminate spinner rather than render + * a bar stuck at 0%. + */ +export interface TranscriptionProgress { + completedSec: number; + totalSec: number; +} + +/** `0..1`, or null when the job reports no measurable progress. */ +export function progressFraction(progress: TranscriptionProgress | undefined): number | null { + if (!progress || !(progress.totalSec > 0)) return null; + return Math.min(1, Math.max(0, progress.completedSec / progress.totalSec)); +} /** * A media that has no audio track (or one Whisper cannot read) will fail the @@ -72,6 +92,7 @@ export interface AssetTranscriptionView { assetId: string; status: AssetTranscriptionStatus; phase?: TranscriptionPhase; + progress?: TranscriptionProgress; failure?: TranscriptionFailure; } @@ -79,6 +100,7 @@ export interface AssetTranscriptionView { export interface TranscriptionJobLike { status: "queued" | "running" | "failed"; phase?: TranscriptionPhase; + progress?: TranscriptionProgress; failure?: TranscriptionFailure; } @@ -122,7 +144,7 @@ export function deriveAssetStatus(input: { }): AssetTranscriptionView { const { assetId, job, transcript, persistedFailure } = input; if (job && job.status !== "failed") { - return { assetId, status: job.status, phase: job.phase }; + return { assetId, status: job.status, phase: job.phase, progress: job.progress }; } if (transcript) { return { diff --git a/src/lib/captioning/transcribe.test.ts b/src/lib/captioning/transcribe.test.ts index 18c6cfa79..82e630b9a 100644 --- a/src/lib/captioning/transcribe.test.ts +++ b/src/lib/captioning/transcribe.test.ts @@ -23,7 +23,13 @@ import { transcribeMono16kToSegments } from "./transcribe"; * worker that the previous Web-Worker pipeline owned, so they run in any env. */ -type Listener = (event: { phase: "model" | "transcribe" }) => void; +// Mirrors `SttRendererStatus` — the mock must accept the progress fields, since +// carrying them across the IPC hop is exactly what this file asserts. +type Listener = (event: { + phase: "model" | "transcribe"; + completedSec?: number; + totalSec?: number; +}) => void; type RendererSttApi = { transcribe: (request: { samples: Float32Array; language?: string }) => Promise<{ @@ -115,12 +121,12 @@ describe("transcribeMono16kToSegments", () => { expect(result.segments).toEqual([{ text: "hello world", startSec: 0, endSec: 0.65 }]); }); - it("forwards 'model' / 'transcribe' phases to onStatus and tears the listener down", async () => { + it("forwards the whole status event to onStatus and tears the listener down", async () => { const onStatus = vi.fn(); mockApi.transcribe.mockImplementationOnce(async () => { // Simulate the IPC handler emitting a status event mid-flight. lastStatusCb?.({ phase: "model" }); - lastStatusCb?.({ phase: "transcribe" }); + lastStatusCb?.({ phase: "transcribe", completedSec: 90, totalSec: 300 }); return { segments: [], wordSegments: [{ word: "ok", startSec: 0, endSec: 0.1 }], @@ -130,8 +136,14 @@ describe("transcribeMono16kToSegments", () => { }); await transcribeMono16kToSegments(new Float32Array(1600), { onStatus }); - expect(onStatus).toHaveBeenCalledWith("model"); - expect(onStatus).toHaveBeenCalledWith("transcribe"); + expect(onStatus).toHaveBeenCalledWith({ phase: "model" }); + // The chunk progress must survive the hop, not just the phase — it is what + // drives the progress bar. + expect(onStatus).toHaveBeenCalledWith({ + phase: "transcribe", + completedSec: 90, + totalSec: 300, + }); // onStatus listener is detached once the promise settles. expect(lastStatusCb).toBeNull(); }); diff --git a/src/lib/captioning/transcribe.ts b/src/lib/captioning/transcribe.ts index a1b39d439..0ac43da2b 100644 --- a/src/lib/captioning/transcribe.ts +++ b/src/lib/captioning/transcribe.ts @@ -28,6 +28,18 @@ export interface TranscribeMono16kResult { export type SttRendererStatusPhase = "model" | "transcribe"; +/** + * Progress the main process reports while a transcription runs. `completedSec` / + * `totalSec` are present only during `"transcribe"`, and only once chunking has + * started — they let the UI show a real bar instead of an indeterminate spinner + * for what can be several minutes of work. + */ +export interface SttRendererStatus { + phase: SttRendererStatusPhase; + completedSec?: number; + totalSec?: number; +} + interface RendererSttApi { transcribe: (request: { samples: Float32Array; language?: string }) => Promise<{ segments: CaptionSegment[]; @@ -35,7 +47,8 @@ interface RendererSttApi { detectedLanguage: string; backend: string; }>; - onStatus?: (callback: (event: { phase: SttRendererStatusPhase }) => void) => () => void; + cancel?: () => Promise; + onStatus?: (callback: (event: SttRendererStatus) => void) => () => void; } /** @@ -52,7 +65,7 @@ export function transcribeMono16kToSegments( samples: Float32Array, options?: { trimRegions?: TrimRegion[]; - onStatus?: (phase: SttRendererStatusPhase) => void; + onStatus?: (status: SttRendererStatus) => void; signal?: AbortSignal; language?: string; }, @@ -67,8 +80,13 @@ export function transcribeMono16kToSegments( return Promise.resolve({ segments: [], granularity: "word" }); } - const unsubscribe = - options?.onStatus && api.onStatus?.((event) => options.onStatus?.(event.phase)); + const unsubscribe = options?.onStatus && api.onStatus?.((event) => options.onStatus?.(event)); + // Aborting has to reach the MAIN process: the work is a chunk loop over there, + // and a renderer that merely stops awaiting still leaves the helper busy for + // minutes — with the replacement request queued behind it, which is what made + // "regenerate in another language" look dead. + const onAbort = () => void api.cancel?.(); + options?.signal?.addEventListener("abort", onAbort, { once: true }); const forcedLanguage = options?.language && options.language !== "auto" ? options.language : undefined; // ponytail: word timestamps come back already absolute from whisper.cpp @@ -104,7 +122,15 @@ export function transcribeMono16kToSegments( } return { segments, granularity, detectedLanguage: result.detectedLanguage }; }) + .catch((error: unknown) => { + // A run the caller cancelled surfaces as an abort, not as an engine + // failure: the store drops it silently instead of toasting the user + // about something they asked for. + if (options?.signal?.aborted) throw new DOMException("Aborted", "AbortError"); + throw error; + }) .finally(() => { + options?.signal?.removeEventListener("abort", onAbort); unsubscribe?.(); }); }