diff --git a/CLAUDE.md b/CLAUDE.md index df1fe96..37e8a05 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -267,7 +267,22 @@ no editing logic in the adapter. **animation** — `Clip::transform_at` interpolates it, the engine renders the motion). Text titles / lower-thirds / captions live on the timeline itself as `Timeline.overlays: Vec` (each with its own `TextKeyframe` - animation); `transcript_to_srt` serializes a transcript to SubRip. A `Track` + animation); `transcript_to_srt` serializes a transcript to SubRip. + **Captions are timeline math, not a transcript dump**, and pure + + unit-tested: a transcript is in *source* time and an overlay is in *timeline* + time, so `Timeline::captions` projects each segment through the clips that + actually show its footage (`Clip::source_span_to_timeline`, honoring trim / + speed / reverse) — captions land on the words that survived the cut and words + that were cut out get none. It reads through `for_render`, so a muted track is + as uncaptioned as it is unheard; it chunks a sentence to `CaptionOptions` + (4 words / 28 chars by default — a speech model emits whole sentences and a + whole sentence does not fit a 9:16 frame), timing lines by *character share* + because neither speech backend reports word timings; lines too short to read + merge back into a neighbour instead of flashing; and no two lines are ever on + screen at once (captions are one lane of text, and the same footage reaching + the cut twice would otherwise collide with itself). `TextOverlay.generated` + marks what it wrote, so regenerating replaces its own set and leaves a typed + title alone. A `Track` carries a `duck` flag (sidechain-ducked under the rest of the mix on export). `Fit` and `Delivery` live here (the domain owns the delivery shape; `engine::cli` re-exports `Fit`), and `Timeline.format` is the frame the project is cut for. @@ -415,7 +430,11 @@ proposal appears for review, not that the cut changes: the read tools `timeline_summary` carries `staged_changes` so it cannot mistake one for the other. `smart_crop` frames each shot for the delivery frame (the server `instructions` pair it with `set_delivery_format`, since reshaping to 9:16 otherwise keeps -whatever was in the middle). +whatever was in the middle). `generate_captions` / `clear_captions` caption the +cut; the `instructions` say to caption **last** and to re-run after any further +edit, because captions are placed in timeline time and a later trim moves the +words out from under them — which an agent has no way to infer from the tool +list. `platform_check` tells it whether the cut is publishable where it is going (and the server `instructions` tell it to run that before reporting a cut finished — an agent that assembles a four-minute Reel has done the work and lost @@ -445,7 +464,8 @@ width/height to clear it), `remove_clip`, `set_volume`, `set_fade`, `set_reframe` / `clear_reframe` / `set_reframe_keyframes` / `add_reframe_keyframe`, `set_asset_projection` (asset-level 360 mark; returns the `Asset`), `add_overlay` / `update_overlay` / `remove_overlay` / `set_overlay_keyframes`, -`captions_from_transcript`, `export_srt`, `remove_silence`, `snap_to_beats`, +`generate_captions` / `clear_captions` (caption the whole cut, in timeline +time), `export_srt`, `remove_silence`, `snap_to_beats`, `smart_crop` (frame each shot for the delivery frame), `extract_audio`, `concatenate` — each returns the refreshed `Timeline`), media (`get_frame` → base64 PNG data URL, `get_waveform`, @@ -568,8 +588,11 @@ at the playhead like Transform — note its `lerpAngle` takes the shortest arc, plain `lerp` would read as a 340° swing across the seam; for a source Kerf did not detect as 360 it instead offers a projection picker that marks the whole asset via `set_asset_projection`), and an always-visible -**Text overlays** section (add titles / lower-thirds, generate captions, edit -text / timing / position / size / color / box / bold). +**Text overlays** section (add titles / lower-thirds, caption the whole cut — +the button relabels to `Recaption` once there are generated captions, since a +later trim moves the words out from under them, with `Clear` beside it taking +only the generated ones — and edit text / timing / position / size / color / +box / bold). **Polish presets** (`src/lib/style-presets.ts`, pure data over the existing surfaces): the Color section leads with one-click **looks** — Punchy / Warm / Cool / Faded / B&W chips (the active one highlights; the sliders @@ -579,7 +602,7 @@ omitted at 0 so old graphs stay byte-identical; plain saturation/gamma can't tint) — and the Text overlays section leads with **Title / Lower third / Caption** style chips that create a styled overlay at the playhead with fade-in/out opacity keyframes; the caption style matches what -`captions_from_transcript` generates, so manual and generated captions look +`generate_captions` generates, so manual and generated captions look alike. Everything is styled with the CSS-variable tokens directly (inline `style`), not Tailwind utilities. The **timeline is a bespoke NLE timeline** that renders **real `editor.timeline` @@ -631,7 +654,11 @@ mirror used **only** by the browser harness, so the panel is drivable under `bun run dev`. `src/lib/smart-crop.ts` is the same arrangement for smart crop: only the *shape* arithmetic is mirrored (bun-tested), because the harness has no decoder to sample with and so lands on the centre window — which part of the shot survives -is the half that only exists with media behind it. The **cover frame** is saved from the preview's context menu +is the half that only exists with media behind it. `src/lib/captions.ts` is the +same arrangement again, but *faithful* rather than approximate — captioning is +arithmetic all the way down, so the harness produces exactly the captions the +backend would (the mirror caught the two-captions-at-once collision the Rust +tests had not). The **cover frame** is saved from the preview's context menu (`Save cover frame…` → `export_cover` at the playhead), and both a finished export and a saved cover offer **Show in folder** in their toast. `Preview` shows the composited frame under the playhead, and during @@ -665,8 +692,9 @@ queue** (status · queue · history · add-task) — Kerf has no in-app chat; a LLM claims tasks over MCP. The queue is `agent` state (`src/lib/agent.svelte.ts`, a third runes singleton) backed by the `tasks` table over Tauri/MCP: the add-task box and preset chips `agent.add(...)` real tasks, and `ready` tasks show Apply/Dismiss (`resolve_task`/`remove_task`). -Four preset chips (`Remove silences` / `Assemble rough cut` / `Frame for the delivery` -/ `Cut to the beat` — which +Five preset chips (`Remove silences` / `Assemble rough cut` / `Frame for the delivery` +/ `Caption the cut` (analyzes whatever is in the cut but not yet transcribed, +then captions it) / `Cut to the beat` — which analyzes whatever is on the audio tracks first, then calls `snap_to_beats`, and says "No cuts were near a beat" instead of claiming an alignment when the grid never reached them) also run the matching local op and diff --git a/crates/kerf-app/src/lib.rs b/crates/kerf-app/src/lib.rs index c990235..d22927d 100644 --- a/crates/kerf-app/src/lib.rs +++ b/crates/kerf-app/src/lib.rs @@ -20,8 +20,9 @@ use std::sync::{Arc, Mutex}; use base64::Engine as _; use kerf_core::{ - Asset, AssetAnalysis, AudioEffect, Delivery, EditSource, ExportOptions, Fit, Keyframe, Project, Projection, ReframeKeyframe, - Revision, StagedEdit, StreamKind, Task, TextKeyframe, Timeline, TimelineDiff, Transition, TransitionKind, VideoEffect, + Asset, AssetAnalysis, AudioEffect, CaptionOptions, Delivery, EditSource, ExportOptions, Fit, Keyframe, Project, Projection, + ReframeKeyframe, Revision, StagedEdit, StreamKind, Task, TextKeyframe, Timeline, TimelineDiff, Transition, TransitionKind, + VideoEffect, }; use serde::Serialize; use tauri::{AppHandle, Emitter, Manager, State}; @@ -887,10 +888,18 @@ fn set_overlay_keyframes(state: State<'_, AppState>, overlay_id: String, keyfram } #[tauri::command(async)] -fn captions_from_transcript(state: State<'_, AppState>, asset_id: String) -> CmdResult { - let id = id(&asset_id)?; +fn generate_captions(state: State<'_, AppState>, options: Option) -> CmdResult { + let project = state.project(); + project + .generate_captions(options.unwrap_or_default()) + .map_err(|e| e.to_string())?; + project.timeline().map_err(|e| e.to_string()) +} + +#[tauri::command(async)] +fn clear_captions(state: State<'_, AppState>) -> CmdResult { let project = state.project(); - project.captions_from_transcript(id).map_err(|e| e.to_string())?; + project.clear_captions().map_err(|e| e.to_string())?; project.timeline().map_err(|e| e.to_string()) } @@ -1604,7 +1613,8 @@ pub fn run() { update_overlay, remove_overlay, set_overlay_keyframes, - captions_from_transcript, + generate_captions, + clear_captions, export_srt, remove_silence, snap_to_beats, diff --git a/crates/kerf-app/src/mcp.rs b/crates/kerf-app/src/mcp.rs index 9af2553..901f3e0 100644 --- a/crates/kerf-app/src/mcp.rs +++ b/crates/kerf-app/src/mcp.rs @@ -15,8 +15,8 @@ use std::sync::{Arc, Mutex, MutexGuard}; use base64::Engine as _; use kerf_core::{ - AudioEffect, Delivery, EditSource, ExportOptions, Fit, Keyframe, Project, Projection, ReframeKeyframe, StreamKind, - TextKeyframe, Transition, TransitionKind, VideoEffect, + AudioEffect, CaptionOptions, Delivery, EditSource, ExportOptions, Fit, Keyframe, Project, Projection, ReframeKeyframe, + StreamKind, TextKeyframe, Transition, TransitionKind, VideoEffect, }; use rmcp::handler::server::wrapper::Parameters; use rmcp::model::{CallToolResult, ContentBlock, Implementation, ServerCapabilities, ServerInfo}; @@ -67,6 +67,18 @@ struct AssetIdParams { asset_id: String, } +#[derive(Debug, Default, serde::Deserialize, schemars::JsonSchema)] +struct CaptionParams { + #[schemars(description = "Most words on one caption line (default 4)")] + max_words: Option, + #[schemars(description = "Most characters on one caption line (default 28); the tighter of the two limits wins")] + max_chars: Option, + #[schemars(description = "Vertical position as a fraction of frame height, 0 = top (default 0.88)")] + pos_y: Option, + #[schemars(description = "Font height as a fraction of frame height (default 0.05)")] + size: Option, +} + #[derive(Debug, serde::Deserialize, schemars::JsonSchema)] struct SpeechModelParams { #[schemars( @@ -1257,16 +1269,38 @@ impl KerfMcp { } #[tool( - description = "Generate caption overlays from an asset's cached transcript (run analyze_asset first), one per segment, low-center with a translucent box. Captions use the transcript's timestamps, so they align when the asset sits at the start of the timeline at normal speed. Returns the overlays created." + description = "Caption the cut: project every clip's cached transcript (run analyze_asset first) through the current edit and write the result as text overlays, replacing any previously generated set. Captions are placed in TIMELINE time, so they follow trims, reorders, speed changes and removed silences, and words that were cut out get no caption. Long sentences are split into readable lines (defaults: 4 words / 28 characters). Hand-made titles and lower-thirds are left alone. Returns the overlays created." )] - fn captions_from_transcript(&self, Parameters(p): Parameters) -> Result { - let id = parse_id(&p.asset_id)?; + fn generate_captions(&self, Parameters(p): Parameters) -> Result { + let mut opts = CaptionOptions::default(); + if let Some(v) = p.max_words { + opts.max_words = v; + } + if let Some(v) = p.max_chars { + opts.max_chars = v; + } + if let Some(v) = p.pos_y { + opts.pos_y = v; + } + if let Some(v) = p.size { + opts.size = v; + } let project = self.lock(); - let out = project.captions_from_transcript(id).map_err(core_err)?; + let out = project.generate_captions(opts).map_err(core_err)?; self.changed(); json(&out) } + #[tool( + description = "Remove the captions generate_captions wrote, leaving hand-made titles and lower-thirds alone. Returns how many were removed." + )] + fn clear_captions(&self) -> Result { + let project = self.lock(); + let removed = project.clear_captions().map_err(core_err)?; + self.changed(); + Ok(format!("removed {removed} generated caption(s)")) + } + #[tool(description = "Write an asset's cached transcript to a SubRip (.srt) subtitle file (run analyze_asset first)")] fn export_srt(&self, Parameters(p): Parameters) -> Result { let id = parse_id(&p.asset_id)?; @@ -1777,8 +1811,12 @@ impl ServerHandler for KerfMcp { the shot is actually about. Add titles, lower-thirds \ and captions with add_overlay / update_overlay / set_overlay_keyframes \ (drawn over the cut; list_fonts lists installed system fonts to pass \ - as update_overlay's font), or captions_from_transcript to caption an \ - analyzed asset in one call; export_srt writes a subtitle file. \ + as update_overlay's font), or generate_captions to caption the whole \ + cut in one call. Caption LAST, after the cutting is done: captions \ + are placed in timeline time, so a later trim or remove_silence moves \ + the words out from under them — re-run generate_captions after any \ + further edit and it replaces its own set, leaving typed titles \ + alone. export_srt writes a subtitle file. \ When the cut is going somewhere vertical, set_delivery_format sets \ the frame it is being made for and smart_crop then frames each shot \ for it — reshaping 16:9 footage to 9:16 throws away most of the \ diff --git a/crates/kerf-core/src/lib.rs b/crates/kerf-core/src/lib.rs index ea74dfb..f9d983c 100644 --- a/crates/kerf-core/src/lib.rs +++ b/crates/kerf-core/src/lib.rs @@ -30,10 +30,10 @@ pub use engine::{ pub use error::{Error, Result}; pub use fonts::list_system_fonts; pub use model::{ - Asset, AssetAnalysis, AudioEffect, Clip, Color, CropFrame, Delivery, DiffEntry, DiffKind, EditSource, Keyframe, Marker, - Projection, Reframe, ReframeKeyframe, ResolvedReframe, Revision, Rhythm, SalienceMap, StagedEdit, StreamInfo, StreamKind, - Task, TaskStatus, TextKeyframe, TextOverlay, TimeRange, Timeline, TimelineDiff, Track, TranscriptSegment, Transform, - Transition, TransitionKind, VideoEffect, + Asset, AssetAnalysis, AudioEffect, CaptionOptions, Clip, Color, CropFrame, Delivery, DiffEntry, DiffKind, EditSource, + Keyframe, Marker, Projection, Reframe, ReframeKeyframe, ResolvedReframe, Revision, Rhythm, SalienceMap, StagedEdit, + StreamInfo, StreamKind, Task, TaskStatus, TextKeyframe, TextOverlay, TimeRange, Timeline, TimelineDiff, Track, + TranscriptSegment, Transform, Transition, TransitionKind, VideoEffect, }; pub use platform::{ check_all as check_platforms, CutSummary, DeliveryCheck, DeliveryIssue, IssueKind, PlatformTarget, Severity, diff --git a/crates/kerf-core/src/model.rs b/crates/kerf-core/src/model.rs index 37bb1db..568f326 100644 --- a/crates/kerf-core/src/model.rs +++ b/crates/kerf-core/src/model.rs @@ -765,6 +765,12 @@ pub struct TextOverlay { /// opacity animate over the overlay's lifetime. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub keyframes: Vec, + /// Written by [`Timeline::captions`] rather than by hand. Regenerating + /// captions replaces these and leaves everything else alone, so re-running + /// after a trim does not stack a second set on top of the first — and does + /// not throw away the title the editor typed. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub generated: bool, } impl TextOverlay { @@ -782,6 +788,7 @@ impl TextOverlay { font: None, bold: false, keyframes: Vec::new(), + generated: false, } } @@ -888,6 +895,229 @@ pub fn transcript_to_srt(segments: &[TranscriptSegment]) -> String { out } +/// Shortest a generated caption line is allowed to stay on screen (seconds). +/// Splitting a fast sentence strictly by character share can hand a two-letter +/// chunk a couple of frames, which reads as a flicker rather than as a word, so +/// chunks below this are merged back into a neighbour instead. +pub const MIN_CAPTION: f64 = 0.45; + +/// How much of a caption line has to survive a cut for it to be kept (seconds). +/// A line whose words were trimmed away leaves a sliver of overlap at the clip +/// edge; showing it would caption footage that is no longer there. +pub const MIN_CAPTION_VISIBLE: f64 = 0.15; + +/// How a transcript is turned into on-screen captions. The defaults are the +/// social shape — a few words at a time, sized to be read on a phone — because a +/// speech model emits whole sentences and a whole sentence does not fit a 9:16 +/// frame. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, schemars::JsonSchema)] +pub struct CaptionOptions { + /// Most words on one caption line. + #[serde(default = "default_caption_words")] + pub max_words: usize, + /// Most characters on one caption line; the tighter of the two limits wins. + #[serde(default = "default_caption_chars")] + pub max_chars: usize, + /// Vertical position as a fraction of frame height. + #[serde(default = "default_caption_y")] + pub pos_y: f64, + /// Font height as a fraction of frame height. + #[serde(default = "default_caption_size")] + pub size: f64, +} + +fn default_caption_words() -> usize { + 4 +} + +fn default_caption_chars() -> usize { + 28 +} + +fn default_caption_y() -> f64 { + 0.88 +} + +fn default_caption_size() -> f64 { + 0.05 +} + +impl Default for CaptionOptions { + fn default() -> Self { + Self { + max_words: default_caption_words(), + max_chars: default_caption_chars(), + pos_y: default_caption_y(), + size: default_caption_size(), + } + } +} + +impl CaptionOptions { + fn sanitized(self) -> Self { + Self { + max_words: self.max_words.max(1), + max_chars: self.max_chars.max(1), + pos_y: if self.pos_y.is_finite() { + self.pos_y.clamp(0.0, 1.0) + } else { + default_caption_y() + }, + size: if self.size.is_finite() { + self.size.clamp(0.005, 0.5) + } else { + default_caption_size() + }, + } + } +} + +/// Break a transcript line into caption-sized groups of words. Greedy: take +/// words until either limit would be exceeded, always at least one (a single +/// word longer than `max_chars` is its own line rather than being cut in half). +fn chunk_words(text: &str, opts: CaptionOptions) -> Vec { + let mut out: Vec = Vec::new(); + let mut current = String::new(); + let mut words = 0usize; + for word in text.split_whitespace() { + let extra = if current.is_empty() { + word.chars().count() + } else { + word.chars().count() + 1 + }; + let fits = words < opts.max_words && current.chars().count() + extra <= opts.max_chars; + if !current.is_empty() && !fits { + out.push(std::mem::take(&mut current)); + words = 0; + } + if !current.is_empty() { + current.push(' '); + } + current.push_str(word); + words += 1; + } + if !current.is_empty() { + out.push(current); + } + out +} + +/// Spread `span` across `chunks` in proportion to how much text each carries, +/// then merge away any line too short to read. Character share is the honest +/// approximation available here: neither speech backend reports word timings +/// (`TranscriptSegment` has only a start and an end), so within a segment the +/// speaker is assumed to be at a steady pace. +fn time_chunks(chunks: Vec, span: TimeRange, min: f64) -> Vec<(TimeRange, String)> { + let mut chunks = chunks; + let duration = (span.end - span.start).max(0.0); + loop { + let weights: Vec = chunks.iter().map(|c| c.chars().count().max(1) as f64).collect(); + let total: f64 = weights.iter().sum(); + let mut timed: Vec<(TimeRange, String)> = Vec::with_capacity(chunks.len()); + let mut at = span.start; + for (i, text) in chunks.iter().enumerate() { + let share = if total > 0.0 { weights[i] / total } else { 1.0 }; + let end = if i + 1 == chunks.len() { + span.end + } else { + at + duration * share + }; + timed.push((TimeRange { start: at, end }, text.clone())); + at = end; + } + // A whole segment shorter than `min` is one line, not a merge loop. + if chunks.len() < 2 { + return timed; + } + let short = timed.iter().position(|(r, _)| r.end - r.start < min); + let Some(i) = short else { return timed }; + // Merge into the shorter neighbour so the joined line stays as close to + // the requested width as the timing allows. + let merge_back = i > 0 && (i + 1 == chunks.len() || chunks[i - 1].chars().count() <= chunks[i + 1].chars().count()); + let into = if merge_back { i - 1 } else { i }; + let moved = chunks.remove(into + 1); + chunks[into] = format!("{}{}{}", chunks[into], ' ', moved); + } +} + +impl Timeline { + /// Caption overlays for the cut as it currently stands. + /// + /// The point of doing this over the timeline rather than over an asset: a + /// transcript is in **source** time, an overlay is in **timeline** time, and + /// between them sit every trim, every reorder, every speed change and every + /// silence the editor removed. Each segment is projected through the clips + /// that actually show its footage ([`Clip::source_span_to_timeline`]), so + /// captions land on the words that survived the cut — and words that did not + /// survive get no caption at all. + /// + /// Reads through [`Timeline::for_render`], so a muted track and a disabled + /// clip are as uncaptioned as they are unheard. + pub fn captions(&self, transcripts: &HashMap>, opts: CaptionOptions) -> Vec { + let opts = opts.sanitized(); + let rendered = self.for_render(); + let mut lines: Vec<(TimeRange, String)> = Vec::new(); + for track in &rendered.tracks { + for clip in &track.clips { + let Some(segments) = transcripts.get(&clip.asset_id) else { + continue; + }; + let (visible_start, visible_end) = (clip.timeline_start, clip.timeline_end()); + for seg in segments { + let text = seg.text.trim(); + if text.is_empty() || seg.end <= seg.start || !clip.covers_source(seg.start, seg.end) { + continue; + } + // Chunk over the segment's *whole* projected span, then clip + // each line to the clip — so a sentence cut in half captions + // only the half that is still in the cut. + let span = clip.source_span_to_timeline(seg.start, seg.end); + for (range, chunk) in time_chunks(chunk_words(text, opts), span, MIN_CAPTION) { + let start = range.start.max(visible_start); + let end = range.end.min(visible_end); + if end - start < MIN_CAPTION_VISIBLE { + continue; + } + lines.push((TimeRange { start, end }, chunk)); + } + } + } + } + lines.sort_by(|a, b| a.0.start.total_cmp(&b.0.start).then_with(|| a.1.cmp(&b.1))); + // The same words can reach two clips — `extract_audio` leaves the picture + // and its detached audio both referencing the asset — and drawing one + // caption twice is drawing it bolder, not twice. + lines.dedup_by(|a, b| a.1 == b.1 && (a.0.start - b.0.start).abs() < 1e-3); + // Captions are one lane of text at one screen position, so two at once is + // two unreadable ones. The same footage reaching the cut twice — a + // callback shot, or a full source parked under the edit — otherwise + // collides with whatever is already on screen. First line in wins the + // slot; the next starts where it ends, or is dropped if nothing readable + // is left of it. + let mut placed: Vec<(TimeRange, String)> = Vec::with_capacity(lines.len()); + for (range, text) in lines { + let start = placed + .last() + .map_or(range.start, |(prev, _): &(TimeRange, String)| range.start.max(prev.end)); + if range.end - start < MIN_CAPTION_VISIBLE { + continue; + } + placed.push((TimeRange { start, end: range.end }, text)); + } + placed + .into_iter() + .map(|(range, text)| { + let mut o = TextOverlay::new(text, range.start.max(0.0), range.end); + o.pos_y = opts.pos_y; + o.size = opts.size; + o.bg = Some("black@0.5".to_string()); + o.generated = true; + o + }) + .collect() + } +} + /// A single non-destructive edit referencing a source range of an asset. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Clip { @@ -1075,6 +1305,28 @@ impl Clip { }; self.timeline_start + offset / self.speed_mag() } + + /// True when any part of the source span `[from, to)` is inside this clip's + /// source window — i.e. whether this clip actually shows that footage. + pub fn covers_source(&self, from: f64, to: f64) -> bool { + let (lo, hi) = if from <= to { (from, to) } else { (to, from) }; + hi.min(self.source_out) > lo.max(self.source_in) + } + + /// Where a source span lands on the timeline, as an ordered range. The ends + /// are mapped through [`Clip::source_to_timeline`] and **not** clamped to the + /// clip, so a span that starts before the in-point maps to a time before + /// `timeline_start` — the caller decides what to do with the part that was + /// trimmed away. A reversed clip swaps the ends, which is why the result is + /// ordered rather than built from `from`/`to` directly. + pub fn source_span_to_timeline(&self, from: f64, to: f64) -> TimeRange { + let a = self.source_to_timeline(from); + let b = self.source_to_timeline(to); + TimeRange { + start: a.min(b), + end: a.max(b), + } + } } /// Tempo estimates below this confidence are ignored when building a beat grid @@ -3024,4 +3276,218 @@ mod tests { assert!(map.crop_for(1920, 1080, f64::NAN).is_none()); assert!(map.crop_for(0, 1080, 1.0).is_none()); } + + fn seg(start: f64, end: f64, text: &str) -> TranscriptSegment { + TranscriptSegment { + start, + end, + text: text.to_string(), + } + } + + fn captioned(timeline: &Timeline, asset: Uuid, segments: Vec) -> Vec<(String, f64, f64)> { + let mut map = HashMap::new(); + map.insert(asset, segments); + timeline + .captions(&map, CaptionOptions::default()) + .into_iter() + .map(|o| (o.text, (o.start * 100.0).round() / 100.0, (o.end * 100.0).round() / 100.0)) + .collect() + } + + fn one_clip(clip: Clip) -> Timeline { + let mut track = Track::new(StreamKind::Video, "V1"); + track.clips = vec![clip]; + Timeline { + tracks: vec![track], + overlays: Vec::new(), + markers: Vec::new(), + format: None, + } + } + + #[test] + fn source_span_maps_through_trim_speed_and_reverse() { + let asset = Uuid::new_v4(); + // Trimmed: the asset's 10s starts at the clip's in-point, placed at 4s. + let mut clip = Clip::new(asset, 10.0, 20.0, 4.0); + let r = clip.source_span_to_timeline(12.0, 14.0); + assert!((r.start - 6.0).abs() < 1e-9, "{r:?}"); + assert!((r.end - 8.0).abs() < 1e-9, "{r:?}"); + + // Double speed halves the distance from the in-point. + clip.speed = 2.0; + let r = clip.source_span_to_timeline(12.0, 14.0); + assert!((r.start - 5.0).abs() < 1e-9, "{r:?}"); + assert!((r.end - 6.0).abs() < 1e-9, "{r:?}"); + + // Reversed: the source's tail is heard first, and the range stays ordered. + clip.speed = -1.0; + let r = clip.source_span_to_timeline(12.0, 14.0); + assert!((r.start - 10.0).abs() < 1e-9, "{r:?}"); + assert!((r.end - 12.0).abs() < 1e-9, "{r:?}"); + assert!(r.end > r.start); + } + + #[test] + fn covers_source_is_the_clips_own_window() { + let clip = Clip::new(Uuid::new_v4(), 10.0, 20.0, 0.0); + assert!(clip.covers_source(12.0, 14.0)); + assert!(clip.covers_source(8.0, 11.0), "straddling the in-point still shows"); + assert!(!clip.covers_source(0.0, 10.0), "ending exactly at the in-point shows nothing"); + assert!(!clip.covers_source(20.0, 25.0)); + } + + #[test] + fn captions_follow_a_trimmed_and_moved_clip() { + let asset = Uuid::new_v4(); + // The interesting case: the transcript says 30s, the cut says 0s. + let timeline = one_clip(Clip::new(asset, 30.0, 34.0, 0.0)); + let lines = captioned(&timeline, asset, vec![seg(30.0, 34.0, "one two three four")]); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].0, "one two three four"); + // Timeline time, not the transcript's 30.0. + assert!(lines[0].1.abs() < 1e-9, "{lines:?}"); + assert!((lines[0].2 - 4.0).abs() < 1e-9, "{lines:?}"); + } + + #[test] + fn a_sentence_cut_in_half_only_captions_what_survived() { + let asset = Uuid::new_v4(); + // A four-word line spoken over 0..4s, but the cut keeps only 0..2s. + let timeline = one_clip(Clip::new(asset, 0.0, 2.0, 0.0)); + let lines = captioned( + &timeline, + asset, + vec![seg(0.0, 4.0, "alpha bravo charlie delta echo foxtrot")], + ); + assert!(!lines.is_empty()); + // Nothing runs past the end of the clip that carries it. + for (text, start, end) in &lines { + assert!(*end <= 2.0 + 1e-9, "{text:?} runs to {end} past the clip"); + assert!(*start >= -1e-9); + } + // The words spoken in the discarded half are gone. + assert!(!lines.iter().any(|(t, _, _)| t.contains("foxtrot")), "{lines:?}"); + } + + #[test] + fn long_segments_split_into_readable_lines() { + let asset = Uuid::new_v4(); + let timeline = one_clip(Clip::new(asset, 0.0, 8.0, 0.0)); + let lines = captioned( + &timeline, + asset, + vec![seg(0.0, 8.0, "Today we are talking about non-destructive editing in Kerf")], + ); + assert!(lines.len() > 1, "a ten-word sentence should not be one caption: {lines:?}"); + for (text, _, _) in &lines { + assert!(text.split_whitespace().count() <= 4, "{text:?} is too many words"); + } + // The lines are contiguous, in order, and cover the segment. + assert!(lines[0].1.abs() < 1e-9); + assert!((lines.last().unwrap().2 - 8.0).abs() < 1e-9); + for pair in lines.windows(2) { + assert!(pair[1].1 >= pair[0].1); + } + // Rejoining the lines gives the sentence back, word for word. + let rejoined = lines.iter().map(|(t, _, _)| t.as_str()).collect::>().join(" "); + assert_eq!(rejoined, "Today we are talking about non-destructive editing in Kerf"); + } + + #[test] + fn fast_speech_merges_rather_than_flickering() { + let asset = Uuid::new_v4(); + // Eight words in 0.9s: split four ways each line would last ~0.22s. + let timeline = one_clip(Clip::new(asset, 0.0, 0.9, 0.0)); + let lines = captioned(&timeline, asset, vec![seg(0.0, 0.9, "a b c d e f g h")]); + for (text, start, end) in &lines { + assert!( + end - start >= MIN_CAPTION - 1e-6 || lines.len() == 1, + "{text:?} flashes for {}s", + end - start + ); + } + // No words were lost to the merging. + let rejoined = lines.iter().map(|(t, _, _)| t.as_str()).collect::>().join(" "); + assert_eq!(rejoined, "a b c d e f g h"); + } + + #[test] + fn captions_are_ordered_by_the_cut_not_by_the_source() { + let asset = Uuid::new_v4(); + // The second half of the source is cut to play first. + let mut track = Track::new(StreamKind::Video, "V1"); + track.clips = vec![Clip::new(asset, 10.0, 12.0, 0.0), Clip::new(asset, 0.0, 2.0, 2.0)]; + let timeline = Timeline { + tracks: vec![track], + overlays: Vec::new(), + markers: Vec::new(), + format: None, + }; + let lines = captioned(&timeline, asset, vec![seg(0.0, 2.0, "first"), seg(10.0, 12.0, "second")]); + assert_eq!(lines.len(), 2, "{lines:?}"); + assert_eq!(lines[0].0, "second", "the reordered cut leads with the later words"); + assert_eq!(lines[1].0, "first"); + assert!(lines[0].1.abs() < 1e-9); + assert!((lines[1].1 - 2.0).abs() < 1e-9); + } + + #[test] + fn two_captions_never_share_the_screen() { + let asset = Uuid::new_v4(); + // The same footage twice in the cut at different offsets — a callback + // shot, or a full source parked under the edit. Both would caption the + // same words on top of each other. + let mut track = Track::new(StreamKind::Video, "V1"); + track.clips = vec![Clip::new(asset, 3.0, 12.0, 0.0), Clip::new(asset, 0.0, 12.0, 0.0)]; + let timeline = Timeline { + tracks: vec![track], + overlays: Vec::new(), + markers: Vec::new(), + format: None, + }; + let lines = captioned( + &timeline, + asset, + vec![seg(0.0, 6.0, "alpha bravo charlie"), seg(6.0, 12.0, "delta echo foxtrot")], + ); + assert!(lines.len() > 1, "{lines:?}"); + for pair in lines.windows(2) { + assert!( + pair[1].1 >= pair[0].2 - 1e-6, + "{:?} starts before {:?} is off screen", + pair[1], + pair[0] + ); + } + } + + #[test] + fn a_muted_track_is_not_captioned() { + let asset = Uuid::new_v4(); + let mut timeline = one_clip(Clip::new(asset, 0.0, 4.0, 0.0)); + assert!(!captioned(&timeline, asset, vec![seg(0.0, 4.0, "heard")]).is_empty()); + timeline.tracks[0].muted = true; + assert!(captioned(&timeline, asset, vec![seg(0.0, 4.0, "heard")]).is_empty()); + } + + #[test] + fn the_same_words_on_two_tracks_are_captioned_once() { + let asset = Uuid::new_v4(); + // What `extract_audio` leaves behind: picture and detached audio, both + // referencing the same asset over the same source window. + let mut video = Track::new(StreamKind::Video, "V1"); + video.clips = vec![Clip::new(asset, 0.0, 3.0, 0.0)]; + let mut audio = Track::new(StreamKind::Audio, "A1"); + audio.clips = vec![Clip::new(asset, 0.0, 3.0, 0.0)]; + let timeline = Timeline { + tracks: vec![video, audio], + overlays: Vec::new(), + markers: Vec::new(), + format: None, + }; + let lines = captioned(&timeline, asset, vec![seg(0.0, 3.0, "only once")]); + assert_eq!(lines.len(), 1, "{lines:?}"); + } } diff --git a/crates/kerf-core/src/project.rs b/crates/kerf-core/src/project.rs index 562f51f..fa9aabf 100644 --- a/crates/kerf-core/src/project.rs +++ b/crates/kerf-core/src/project.rs @@ -13,9 +13,9 @@ use crate::engine::{self, ExportProgress}; use crate::error::{Error, Result}; use crate::model::default_beat_tolerance; use crate::model::{ - Asset, AssetAnalysis, AudioEffect, Clip, CropFrame, Delivery, EditSource, Keyframe, Marker, Projection, Reframe, - ReframeKeyframe, Revision, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, Tempo, TextKeyframe, TextOverlay, TimeRange, - Timeline, TimelineDiff, Track, Transition, VideoEffect, MAX_FOV, MIN_FOV, + Asset, AssetAnalysis, AudioEffect, CaptionOptions, Clip, CropFrame, Delivery, EditSource, Keyframe, Marker, Projection, + Reframe, ReframeKeyframe, Revision, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, Tempo, TextKeyframe, TextOverlay, + TimeRange, Timeline, TimelineDiff, Track, TranscriptSegment, Transition, VideoEffect, MAX_FOV, MIN_FOV, }; /// One clip queued for smart-crop sampling: which media to look at, over which @@ -2811,37 +2811,66 @@ impl Project { }) } - /// Generate caption overlays from an asset's cached transcript — one per - /// segment, low-center with a translucent box. The segments keep the - /// transcript's own timestamps, so they line up when the asset sits at the - /// start of the timeline at normal speed. Returns the overlays created. - pub fn captions_from_transcript(&self, asset_id: Uuid) -> Result> { - let analysis = self - .get_analysis(asset_id)? - .ok_or_else(|| Error::InvalidArgument("no analysis available for asset; run analysis first".to_string()))?; - let overlays: Vec = analysis - .transcript - .iter() - .filter(|s| !s.text.trim().is_empty() && s.end > s.start) - .map(|s| { - let mut o = TextOverlay::new(s.text.trim().to_string(), s.start.max(0.0), s.end); - o.pos_y = 0.88; - o.size = 0.05; - o.bg = Some("black@0.5".to_string()); - o - }) - .collect(); + /// Caption the cut: project every clip's cached transcript through the edit + /// and write the result as text overlays, replacing any previous generated + /// set. + /// + /// This is deliberately timeline-scoped rather than asset-scoped. A + /// transcript is in source time and an overlay is in timeline time, and the + /// two only agree on an untouched asset sitting at zero — which is not a cut. + /// The moment anything is trimmed, reordered, retimed or (most of all) + /// silence-removed, source time and timeline time diverge and every caption + /// is on the wrong word. [`Timeline::captions`] does the projection, so + /// captions follow the cut and words that were cut out get none. + /// + /// Errors when nothing on the timeline has a transcript to caption, rather + /// than quietly writing no overlays. + pub fn generate_captions(&self, opts: CaptionOptions) -> Result> { + let timeline = self.working_timeline()?; + let mut transcripts: HashMap> = HashMap::new(); + let mut analyzed = false; + for track in &timeline.tracks { + for clip in &track.clips { + if transcripts.contains_key(&clip.asset_id) { + continue; + } + let Some(analysis) = self.get_analysis(clip.asset_id)? else { + continue; + }; + analyzed = true; + transcripts.insert(clip.asset_id, analysis.transcript); + } + } + if !analyzed { + return Err(Error::InvalidArgument( + "no analysis available for the clips on the timeline; run analysis first".to_string(), + )); + } + let overlays = timeline.captions(&transcripts, opts); if overlays.is_empty() { - return Err(Error::InvalidArgument("asset has no usable transcript".to_string())); + return Err(Error::InvalidArgument( + "no speech was transcribed for the footage in this cut".to_string(), + )); } let created = overlays.clone(); - self.edit_timeline("Add captions from transcript", move |timeline| { + self.edit_timeline("Generate captions", move |timeline| { + timeline.overlays.retain(|o| !o.generated); timeline.overlays.extend(overlays); Ok(()) })?; Ok(created) } + /// Remove the captions [`Project::generate_captions`] wrote, leaving titles + /// and lower-thirds alone. + pub fn clear_captions(&self) -> Result { + self.edit_timeline("Clear captions", move |timeline| { + let before = timeline.overlays.len(); + timeline.overlays.retain(|o| !o.generated); + Ok(before - timeline.overlays.len()) + }) + } + /// Render an asset's cached transcript as a SubRip (`.srt`) document. pub fn transcript_srt(&self, asset_id: Uuid) -> Result { let analysis = self @@ -4348,4 +4377,59 @@ mod tests { let (ti, ci) = timeline.locate(clip_id).unwrap(); timeline.tracks[ti].clips[ci].transform } + + #[test] + fn captions_survive_the_cut_that_silence_removal_makes() { + let project = Project::sample().unwrap(); + let asset = project.list_assets().unwrap()[0].id; + // The sample's transcript is two lines over 0..12.5s with a silence at + // 12.5..14.0; put the asset on the timeline at a non-zero position and + // trimmed, which is what makes source time and timeline time disagree. + let timeline = project.timeline().unwrap(); + let track = timeline.tracks[0].id; + for t in &timeline.tracks { + for clip in &t.clips { + project.remove(clip.id).unwrap(); + } + } + project + .add_clip_to_timeline(asset, Some(track), 5.5, 12.5, Some(2.0)) + .unwrap(); + + let created = project.generate_captions(CaptionOptions::default()).unwrap(); + assert!(!created.is_empty()); + // Every caption sits inside the clip's timeline span (2.0 .. 9.0), not + // at the transcript's own 5.5 .. 12.5. + for o in &created { + assert!(o.start >= 2.0 - 1e-6 && o.end <= 9.0 + 1e-6, "{o:?} is in source time"); + assert!(o.generated); + } + // The line spoken before the in-point was trimmed away, so it gets none. + assert!( + !created.iter().any(|o| o.text.contains("Welcome")), + "captioned words that are not in the cut: {created:?}" + ); + + // Regenerating replaces rather than stacks, and leaves a hand-made + // title alone. + let title = project.add_overlay("Chapter one".to_string(), 0.0, 1.5).unwrap(); + let again = project.generate_captions(CaptionOptions::default()).unwrap(); + let overlays = project.timeline().unwrap().overlays; + assert_eq!(overlays.iter().filter(|o| o.generated).count(), again.len()); + assert!(overlays.iter().any(|o| o.id == title.id), "the typed title was thrown away"); + + // Clearing takes only the generated ones. + let cleared = project.clear_captions().unwrap(); + assert_eq!(cleared, again.len()); + let overlays = project.timeline().unwrap().overlays; + assert_eq!(overlays.len(), 1); + assert_eq!(overlays[0].id, title.id); + } + + #[test] + fn captions_need_something_transcribed_on_the_timeline() { + let project = Project::open_in_memory().unwrap(); + let err = project.generate_captions(CaptionOptions::default()).unwrap_err(); + assert!(err.to_string().contains("run analysis first"), "{err}"); + } } diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index ee39060..439deaf 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -10,6 +10,7 @@ import type { AssetAnalysis, AssetMetadata, AudioEffect, + CaptionOptions, Clip, Color, Delivery, @@ -34,6 +35,7 @@ import type { Transform, Transition, TranscriptionStatus, + TranscriptSegment, UpdateInfo, VideoEffect } from './types'; @@ -42,6 +44,7 @@ import { alignCutsToBeats, beatGrid, defaultBeatTolerance } from './beats'; import { formatTime as fmtTime } from './diff'; import { checkAll } from './platforms'; import { centeredCrop } from './smart-crop'; +import { captionsForTimeline, CAPTION_DEFAULTS } from './captions'; export function inTauri(): boolean { return typeof window !== 'undefined' && '__TAURI_INTERNALS__' in window; @@ -1192,30 +1195,34 @@ export async function setOverlayKeyframes(overlayId: string, keyframes: TextKeyf return invoke('set_overlay_keyframes', { overlayId, keyframes }); } -/** Generate caption overlays from an asset's cached transcript. */ -export async function captionsFromTranscript(assetId: string): Promise { - if (!inTauri()) { - const segs = sampleAnalysis[assetId]?.transcript ?? []; - const overlays = (devTimeline.overlays ??= []); - for (const s of segs) { - if (!s.text.trim() || s.end <= s.start) continue; - overlays.push({ - id: uid(), - text: s.text.trim(), - start: s.start, - end: s.end, - pos_x: 0.5, - pos_y: 0.88, - size: 0.05, - color: 'white', - bg: 'black@0.5', - bold: false - }); +/** Caption the cut: project every clip's transcript through the current edit and + * write the result as overlays, replacing any previously generated set. */ +export async function generateCaptions(options?: CaptionOptions): Promise { + if (!inTauri()) { + const transcripts: Record = {}; + for (const track of devTimeline.tracks) { + for (const clip of track.clips) { + const segs = sampleAnalysis[clip.asset_id]?.transcript; + if (segs) transcripts[clip.asset_id] = segs; + } } - recordDev('Add captions from transcript'); + const created = captionsForTimeline(devTimeline, transcripts, { ...CAPTION_DEFAULTS, ...options }); + const kept = (devTimeline.overlays ??= []).filter((o) => !o.generated); + devTimeline.overlays = [...kept, ...created.map((o) => ({ ...o, id: uid() }))]; + recordDev('Generate captions'); + return snapshot(); + } + return invoke('generate_captions', { options: options ?? null }); +} + +/** Remove the generated captions, leaving typed titles and lower-thirds alone. */ +export async function clearCaptions(): Promise { + if (!inTauri()) { + devTimeline.overlays = (devTimeline.overlays ?? []).filter((o) => !o.generated); + recordDev('Clear captions'); return snapshot(); } - return invoke('captions_from_transcript', { assetId }); + return invoke('clear_captions'); } /** Write an asset's transcript to a `.srt` file; returns the path. */ diff --git a/frontend/src/lib/captions.test.ts b/frontend/src/lib/captions.test.ts new file mode 100644 index 0000000..cb22b06 --- /dev/null +++ b/frontend/src/lib/captions.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, test } from 'bun:test'; +import { + captionsForTimeline, + chunkWords, + coversSource, + sourceToTimeline, + timeChunks, + CAPTION_DEFAULTS, + MIN_CAPTION +} from './captions'; +import type { Clip, Timeline, TranscriptSegment } from './types'; + +const clip = (over: Partial = {}): Clip => + ({ + id: 'c1', + asset_id: 'a1', + source_in: 0, + source_out: 4, + timeline_start: 0, + volume: 1, + speed: 1, + ...over + }) as Clip; + +const timelineOf = (clips: Clip[]): Timeline => ({ + tracks: [{ id: 't1', kind: 'video', name: 'V1', clips }], + overlays: [], + markers: [] +}); + +const seg = (start: number, end: number, text: string): TranscriptSegment => ({ start, end, text }); + +describe('projection', () => { + test('maps through trim, speed and reverse', () => { + const c = clip({ source_in: 10, source_out: 20, timeline_start: 4 }); + expect(sourceToTimeline(c, 12)).toBeCloseTo(6, 9); + expect(sourceToTimeline({ ...c, speed: 2 }, 12)).toBeCloseTo(5, 9); + // Reversed: the source's tail is heard first. + expect(sourceToTimeline({ ...c, speed: -1 }, 12)).toBeCloseTo(12, 9); + }); + + test('covers_source is the clip window', () => { + const c = clip({ source_in: 10, source_out: 20 }); + expect(coversSource(c, 12, 14)).toBe(true); + expect(coversSource(c, 8, 11)).toBe(true); + expect(coversSource(c, 0, 10)).toBe(false); + expect(coversSource(c, 20, 25)).toBe(false); + }); +}); + +describe('chunking', () => { + test('splits to the word and character limits', () => { + const lines = chunkWords('Today we are talking about non-destructive editing in Kerf', CAPTION_DEFAULTS); + expect(lines.length).toBeGreaterThan(1); + for (const l of lines) { + expect(l.split(' ').length).toBeLessThanOrEqual(CAPTION_DEFAULTS.max_words); + } + expect(lines.join(' ')).toBe('Today we are talking about non-destructive editing in Kerf'); + }); + + test('a word longer than the limit is its own line, not cut in half', () => { + const lines = chunkWords('supercalifragilisticexpialidocious ok', { ...CAPTION_DEFAULTS, max_chars: 10 }); + expect(lines[0]).toBe('supercalifragilisticexpialidocious'); + }); + + test('lines too short to read merge instead of flickering', () => { + const lines = timeChunks(chunkWords('a b c d e f g h', CAPTION_DEFAULTS), 0, 0.9); + for (const l of lines) { + if (lines.length > 1) expect(l.end - l.start).toBeGreaterThanOrEqual(MIN_CAPTION - 1e-6); + } + expect(lines.map((l) => l.text).join(' ')).toBe('a b c d e f g h'); + }); +}); + +describe('captions follow the cut', () => { + test('a trimmed clip captions in timeline time', () => { + const out = captionsForTimeline(timelineOf([clip({ source_in: 30, source_out: 34 })]), { + a1: [seg(30, 34, 'one two three four')] + }); + expect(out).toHaveLength(1); + expect(out[0].start).toBeCloseTo(0, 9); + expect(out[0].end).toBeCloseTo(4, 9); + expect(out[0].generated).toBe(true); + }); + + test('words cut out get no caption', () => { + const out = captionsForTimeline(timelineOf([clip({ source_in: 0, source_out: 2 })]), { + a1: [seg(0, 4, 'alpha bravo charlie delta echo foxtrot')] + }); + expect(out.length).toBeGreaterThan(0); + for (const o of out) expect(o.end).toBeLessThanOrEqual(2 + 1e-9); + expect(out.some((o) => o.text.includes('foxtrot'))).toBe(false); + }); + + test('a reordered cut captions in the order it plays', () => { + const out = captionsForTimeline( + timelineOf([ + clip({ id: 'c1', source_in: 10, source_out: 12, timeline_start: 0 }), + clip({ id: 'c2', source_in: 0, source_out: 2, timeline_start: 2 }) + ]), + { a1: [seg(0, 2, 'first'), seg(10, 12, 'second')] } + ); + expect(out.map((o) => o.text)).toEqual(['second', 'first']); + }); + + test('two captions never share the screen', () => { + // The same footage twice in the cut — a callback shot, or a full source + // parked under the edit — would caption the same words on top of itself. + const out = captionsForTimeline( + timelineOf([ + clip({ id: 'c1', source_in: 3, source_out: 12 }), + clip({ id: 'c2', source_in: 0, source_out: 12 }) + ]), + { a1: [seg(0, 6, 'alpha bravo charlie'), seg(6, 12, 'delta echo foxtrot')] } + ); + expect(out.length).toBeGreaterThan(1); + for (let i = 1; i < out.length; i++) { + expect(out[i].start).toBeGreaterThanOrEqual(out[i - 1].end - 1e-6); + } + }); + + test('a muted track is not captioned', () => { + const tl = timelineOf([clip()]); + expect(captionsForTimeline(tl, { a1: [seg(0, 4, 'heard')] })).toHaveLength(1); + tl.tracks[0].muted = true; + expect(captionsForTimeline(tl, { a1: [seg(0, 4, 'heard')] })).toHaveLength(0); + }); +}); diff --git a/frontend/src/lib/captions.ts b/frontend/src/lib/captions.ts new file mode 100644 index 0000000..bdda44e --- /dev/null +++ b/frontend/src/lib/captions.ts @@ -0,0 +1,163 @@ +/** The caption arithmetic, mirrored from `kerf_core::model` so the browser dev + * harness produces the same captions the backend would. + * + * Only the *timing* math lives here — projecting a transcript's source time + * onto timeline time through each clip's trim / speed / reverse, and splitting + * a sentence into readable lines. That is the whole feature: which words a + * caption carries and when it appears. Kerf-core stays the authority; this is + * the same arrangement as `platforms.ts` and `smart-crop.ts`. + */ + +import type { Clip, TextOverlay, Timeline, TranscriptSegment } from './types'; + +/** Shortest a generated line stays on screen; below this it reads as a flicker. */ +export const MIN_CAPTION = 0.45; +/** How much of a line has to survive a cut for it to be kept. */ +export const MIN_CAPTION_VISIBLE = 0.15; + +export interface CaptionOpts { + max_words: number; + max_chars: number; + pos_y: number; + size: number; +} + +export const CAPTION_DEFAULTS: CaptionOpts = { max_words: 4, max_chars: 28, pos_y: 0.88, size: 0.05 }; + +const MIN_SPEED = 0.01; +const speedMag = (c: Clip) => Math.max(Math.abs(c.speed ?? 1), MIN_SPEED); +const reversed = (c: Clip) => (c.speed ?? 1) < 0; + +export function clipDuration(c: Clip): number { + return Math.max(c.source_out - c.source_in, 0) / speedMag(c); +} + +/** Where a source timestamp of this clip lands on the timeline. */ +export function sourceToTimeline(c: Clip, source: number): number { + const offset = reversed(c) ? c.source_out - source : source - c.source_in; + return c.timeline_start + offset / speedMag(c); +} + +/** Whether any of the source span `[from, to)` is inside the clip's window. */ +export function coversSource(c: Clip, from: number, to: number): boolean { + const lo = Math.min(from, to); + const hi = Math.max(from, to); + return Math.min(hi, c.source_out) > Math.max(lo, c.source_in); +} + +/** Break a line into caption-sized groups of words; always at least one word, + * so a single word longer than `max_chars` is its own line rather than cut. */ +export function chunkWords(text: string, opts: CaptionOpts): string[] { + const out: string[] = []; + let current = ''; + let words = 0; + for (const word of text.split(/\s+/).filter(Boolean)) { + const extra = current ? word.length + 1 : word.length; + const fits = words < opts.max_words && current.length + extra <= opts.max_chars; + if (current && !fits) { + out.push(current); + current = ''; + words = 0; + } + current = current ? `${current} ${word}` : word; + words += 1; + } + if (current) out.push(current); + return out; +} + +/** Spread a span across lines by character share, merging away any line too + * short to read. Character share is the approximation available: neither + * speech backend reports word timings. */ +export function timeChunks( + chunks: string[], + start: number, + end: number, + min = MIN_CAPTION +): { start: number; end: number; text: string }[] { + let lines = [...chunks]; + const duration = Math.max(end - start, 0); + for (;;) { + const weights = lines.map((c) => Math.max(c.length, 1)); + const total = weights.reduce((a, b) => a + b, 0); + const timed: { start: number; end: number; text: string }[] = []; + let at = start; + lines.forEach((text, i) => { + const share = total > 0 ? weights[i] / total : 1; + const to = i + 1 === lines.length ? end : at + duration * share; + timed.push({ start: at, end: to, text }); + at = to; + }); + if (lines.length < 2) return timed; + const short = timed.findIndex((t) => t.end - t.start < min); + if (short < 0) return timed; + const mergeBack = + short > 0 && (short + 1 === lines.length || lines[short - 1].length <= lines[short + 1].length); + const into = mergeBack ? short - 1 : short; + lines = [ + ...lines.slice(0, into), + `${lines[into]} ${lines[into + 1]}`, + ...lines.slice(into + 2) + ]; + } +} + +/** Caption the cut: project each transcript segment through the clips that + * actually show its footage. Mirrors `Timeline::captions`. */ +export function captionsForTimeline( + timeline: Timeline, + transcripts: Record, + opts: CaptionOpts = CAPTION_DEFAULTS +): Omit[] { + const soloed = new Set(timeline.tracks.filter((t) => t.solo).map((t) => t.kind)); + const lines: { start: number; end: number; text: string }[] = []; + for (const track of timeline.tracks) { + if (track.muted || (soloed.has(track.kind) && !track.solo)) continue; + for (const clip of track.clips) { + if (clip.enabled === false) continue; + const segments = transcripts[clip.asset_id]; + if (!segments) continue; + const visibleStart = clip.timeline_start; + const visibleEnd = clip.timeline_start + clipDuration(clip); + for (const seg of segments) { + const text = seg.text.trim(); + if (!text || seg.end <= seg.start || !coversSource(clip, seg.start, seg.end)) continue; + // Chunk over the segment's whole projected span, then clip each + // line — so a sentence cut in half captions only the surviving half. + const a = sourceToTimeline(clip, seg.start); + const b = sourceToTimeline(clip, seg.end); + for (const line of timeChunks(chunkWords(text, opts), Math.min(a, b), Math.max(a, b))) { + const start = Math.max(line.start, visibleStart); + const end = Math.min(line.end, visibleEnd); + if (end - start < MIN_CAPTION_VISIBLE) continue; + lines.push({ start, end, text: line.text }); + } + } + } + } + lines.sort((x, y) => x.start - y.start || x.text.localeCompare(y.text)); + const deduped = lines.filter( + (l, i) => i === 0 || l.text !== lines[i - 1].text || Math.abs(l.start - lines[i - 1].start) >= 1e-3 + ); + // Captions are one lane of text at one screen position, so two at once is two + // unreadable ones. First line in wins the slot; the next starts where it ends, + // or is dropped if nothing readable is left of it. + const placed: typeof deduped = []; + for (const l of deduped) { + const start = placed.length ? Math.max(l.start, placed[placed.length - 1].end) : l.start; + if (l.end - start < MIN_CAPTION_VISIBLE) continue; + placed.push({ ...l, start }); + } + return placed.map((l) => ({ + text: l.text, + start: Math.max(l.start, 0), + end: l.end, + pos_x: 0.5, + pos_y: opts.pos_y, + size: opts.size, + color: 'white', + bg: 'black@0.5', + bold: false, + generated: true + })); +} diff --git a/frontend/src/lib/components/editor/AgentPanel.svelte b/frontend/src/lib/components/editor/AgentPanel.svelte index 0ee08f2..72ddf1a 100644 --- a/frontend/src/lib/components/editor/AgentPanel.svelte +++ b/frontend/src/lib/components/editor/AgentPanel.svelte @@ -205,6 +205,18 @@ // rather than claiming an alignment that never happened. if (cutSignature() === before) toast.info('No cuts were near a beat'); else toast.success('Aligned the cuts to the beat'); + } else if (task && p === 'Caption the cut') { + // Captions read from the clips' transcripts, so analyze whatever is + // in the cut but has not been transcribed yet. + const sources = [...new Set(editor.timeline.tracks.flatMap((t) => t.clips.map((c) => c.asset_id)))]; + if (sources.length === 0) throw new Error('Put a clip on the timeline first'); + for (const id of sources) if (!editor.analysisFor(id)) await ui.runAnalysis(id); + await editor.generateCaptions(); + await agent.resolve(task.id); + const n = (editor.timeline.overlays ?? []).filter((o) => o.generated).length; + toast.success(`Captioned the cut — ${n} line${n === 1 ? '' : 's'}`, { + action: { label: 'Undo', onClick: () => void editor.undo() } + }); } else if (task && p === 'Frame for the delivery') { // Smart crop only matters once the project has a frame to be cut // for; without one the frame follows the footage and every shot diff --git a/frontend/src/lib/components/editor/Inspector.svelte b/frontend/src/lib/components/editor/Inspector.svelte index a6e0cf8..bf40d84 100644 --- a/frontend/src/lib/components/editor/Inspector.svelte +++ b/frontend/src/lib/components/editor/Inspector.svelte @@ -219,13 +219,20 @@ editor.selectedOverlayId = created.id; }); } + /** Captions are placed in timeline time, so they follow the cut — which also + * means a later trim moves the words out from under them. Re-running + * replaces the generated set, so the button stays the same after the first + * press and only its label admits what it is doing. */ + const hasCaptions = $derived(overlays.some((o) => o.generated)); function makeCaptions() { - const id = clip?.asset_id ?? editor.selectedAssetId; - if (!id) { - toast.error('Select a clip or asset with a transcript first'); + if (!editor.timeline.tracks.some((t) => t.clips.length > 0)) { + toast.error('Put a clip on the timeline first'); return; } - void run(() => editor.captionsFromTranscript(id)); + void run(() => editor.generateCaptions()); + } + function dropCaptions() { + void run(() => editor.clearCaptions()); } // While a slider is being dragged, show its live value (keyed by row label) @@ -300,8 +307,11 @@ } items.push( { label: 'Add text overlay', icon: 'captions', action: addOverlayHere }, - { label: 'Generate captions', icon: 'captions', action: makeCaptions } + { label: hasCaptions ? 'Regenerate captions' : 'Generate captions', icon: 'captions', action: makeCaptions } ); + if (hasCaptions) { + items.push({ label: 'Clear captions', icon: 'trash', action: dropCaptions }); + } contextMenu.show(e, items); } @@ -473,11 +483,17 @@
+ Text - Captions + + {hasCaptions ? 'Recaption' : 'Captions'} + + {#if hasCaptions} + Clear + {/if}
{#if overlays.length === 0}
- No titles or captions yet. Add text, or generate captions from an analyzed asset's transcript. + No titles or captions yet. Add text, or caption the whole cut from the transcripts of the clips on + the timeline — captions land on the words that survived your edit.
{/if} {#each overlays as o (o.id)} diff --git a/frontend/src/lib/components/editor/data.ts b/frontend/src/lib/components/editor/data.ts index 17e4842..04d9d1d 100644 --- a/frontend/src/lib/components/editor/data.ts +++ b/frontend/src/lib/components/editor/data.ts @@ -17,6 +17,7 @@ export const PRESETS = [ 'Remove silences', 'Cut to the beat', 'Frame for the delivery', + 'Caption the cut', 'Assemble rough cut', 'Find best 60s', 'Color match' diff --git a/frontend/src/lib/state.svelte.ts b/frontend/src/lib/state.svelte.ts index f776a21..2afe734 100644 --- a/frontend/src/lib/state.svelte.ts +++ b/frontend/src/lib/state.svelte.ts @@ -7,7 +7,8 @@ import { addReframeKeyframe, addOverlay, analyzeAsset, - captionsFromTranscript, + generateCaptions, + clearCaptions, clearKeyframes, clearReframe, concatenate, @@ -78,6 +79,7 @@ import type { AssetAnalysis, AssetMetadata, AudioEffect, + CaptionOptions, Clip, Color, Delivery, @@ -652,8 +654,11 @@ class EditorState { setOverlayKeyframes(overlayId: string, keyframes: TextKeyframe[]) { return this.#apply(setOverlayKeyframes(overlayId, keyframes)); } - captionsFromTranscript(assetId: string) { - return this.#apply(captionsFromTranscript(assetId)); + generateCaptions(options?: CaptionOptions) { + return this.#apply(generateCaptions(options)); + } + clearCaptions() { + return this.#apply(clearCaptions()); } /** Write the asset's transcript to `.srt`; returns the path (no timeline change). */ exportSrt(assetId: string, outputPath: string) { diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 4787eb2..f06d8b2 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -188,6 +188,18 @@ export interface TextOverlay { font?: string | null; bold: boolean; keyframes?: TextKeyframe[]; + /** Written by `generate_captions` rather than by hand. Regenerating replaces + * these and leaves typed titles alone. */ + generated?: boolean; +} + +/** How a transcript is turned into on-screen captions. Omitted fields keep the + * backend's social-video defaults (4 words / 28 chars, low-centre). */ +export interface CaptionOptions { + max_words?: number; + max_chars?: number; + pos_y?: number; + size?: number; } export interface Clip {