From 09acd5f702978f0d6f86341dc3d280d985e0d000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:01:11 +0200 Subject: [PATCH 01/21] caption one word at a time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A subtitle line is read; a punched word is watched. `CaptionStyle` picks between them, and it is one decision rather than four: the word count, the size, the position and the floors that stop a line flickering all move together, so asking for `word_punch` gets the whole look instead of one word left at subtitle size in the subtitle position. Held to `MIN_CAPTION` every short word would merge into a neighbour and the style would collapse back into lines, so word punch gets its own floors — words still merge, just far later. `CaptionOptions` is now style plus overrides: every number is optional and follows the style when omitted, resolved by `resolve()` into the `CaptionLayout` captioning actually works from. `CaptionOptions::default()` is unchanged, so every existing call captions exactly as before. Captions are also fitted to the frame now. `drawtext` neither wraps nor scales, and a 9:16 frame is barely half as wide as it is tall, so a long word — or a 28-character subtitle line, which was already true before this — was drawn off both edges. `fontsize` cannot be an expression over `text_w` (the width is what depends on the size), so `fit_size` estimates it from the character count against the project's delivery aspect. An unframed project assumes 16:9, which is wide enough that the fit never binds: nothing that never picked a frame captions differently than it did. --- crates/kerf-core/src/lib.rs | 6 +- crates/kerf-core/src/model.rs | 304 ++++++++++++++++++++++++++------ crates/kerf-core/src/project.rs | 18 +- 3 files changed, 268 insertions(+), 60 deletions(-) diff --git a/crates/kerf-core/src/lib.rs b/crates/kerf-core/src/lib.rs index f9d983c..b55e5b8 100644 --- a/crates/kerf-core/src/lib.rs +++ b/crates/kerf-core/src/lib.rs @@ -30,9 +30,9 @@ pub use engine::{ pub use error::{Error, Result}; pub use fonts::list_system_fonts; pub use model::{ - Asset, AssetAnalysis, AudioEffect, CaptionOptions, Clip, Color, CropFrame, Delivery, DiffEntry, DiffKind, EditSource, - Keyframe, Marker, Projection, Reframe, ReframeKeyframe, ResolvedReframe, Revision, Rhythm, SalienceMap, StagedEdit, - StreamInfo, StreamKind, Task, TaskStatus, TextKeyframe, TextOverlay, TimeRange, Timeline, TimelineDiff, Track, + Asset, AssetAnalysis, AudioEffect, CaptionLayout, CaptionOptions, CaptionStyle, Clip, Color, CropFrame, Delivery, DiffEntry, + DiffKind, EditSource, Keyframe, Marker, Projection, Reframe, ReframeKeyframe, ResolvedReframe, Revision, Rhythm, SalienceMap, + StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, TextKeyframe, TextOverlay, TimeRange, Timeline, TimelineDiff, Track, TranscriptSegment, Transform, Transition, TransitionKind, VideoEffect, }; pub use platform::{ diff --git a/crates/kerf-core/src/model.rs b/crates/kerf-core/src/model.rs index 568f326..1782d80 100644 --- a/crates/kerf-core/src/model.rs +++ b/crates/kerf-core/src/model.rs @@ -906,76 +906,170 @@ pub const MIN_CAPTION: f64 = 0.45; /// edge; showing it would caption footage that is no longer there. pub const MIN_CAPTION_VISIBLE: f64 = 0.15; -/// How a transcript is turned into on-screen captions. The defaults are the -/// social shape — a few words at a time, sized to be read on a phone — because a -/// speech model emits whole sentences and a whole sentence does not fit a 9:16 -/// frame. -#[derive(Debug, Clone, Copy, Serialize, Deserialize, schemars::JsonSchema)] -pub struct CaptionOptions { +/// The same two floors for [`CaptionStyle::WordPunch`], where a line *is* one +/// word. Held to [`MIN_CAPTION`] every short word would merge into a neighbour +/// and the style would collapse back into [`CaptionStyle::Lines`]; words still +/// merge — a one-letter word's character share is a couple of frames — just far +/// later. +pub const MIN_WORD_CAPTION: f64 = 0.12; +pub const MIN_WORD_VISIBLE: f64 = 0.06; + +/// The shape a generated caption set takes on screen. +/// +/// Two, because they are consumed differently. A subtitle line is *read*: it +/// holds still long enough to take several words in at once. The one-word form +/// is *watched* — each word lands on the beat of the speech, which is the look +/// social captions have converged on and most of why a muted feed video holds +/// attention. It is not a font choice: the word count, the size, the position +/// and the floors that stop a line flickering all move together, so it is one +/// decision rather than four. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CaptionStyle { + /// A few words at a time, held as a subtitle line low in the frame. + #[default] + Lines, + /// One word at a time, large and bold, cut in and out on the word. + WordPunch, +} + +impl CaptionStyle { + /// The style's own numbers, before any per-call override. + fn layout(self) -> CaptionLayout { + match self { + Self::Lines => CaptionLayout { + max_words: 4, + max_chars: 28, + pos_y: 0.88, + size: 0.05, + bold: false, + min_line: MIN_CAPTION, + min_visible: MIN_CAPTION_VISIBLE, + }, + Self::WordPunch => CaptionLayout { + max_words: 1, + max_chars: 28, + // Higher and much larger than a subtitle: one word carries the + // whole frame, and sitting it on the bottom edge would put it + // under the platform's own caption rail. + pos_y: 0.72, + size: 0.11, + bold: true, + min_line: MIN_WORD_CAPTION, + min_visible: MIN_WORD_VISIBLE, + }, + } + } +} + +/// A [`CaptionStyle`]'s numbers with any per-call override applied — what +/// [`Timeline::captions`] actually works from. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct CaptionLayout { /// Most words on one caption line. - #[serde(default = "default_caption_words")] pub max_words: usize, /// Most characters on one caption line; the tighter of the two limits wins. - #[serde(default = "default_caption_chars")] pub max_chars: usize, /// Vertical position as a fraction of frame height. - #[serde(default = "default_caption_y")] pub pos_y: f64, /// Font height as a fraction of frame height. - #[serde(default = "default_caption_size")] pub size: f64, + /// Whether the text is drawn bold. + pub bold: bool, + /// Shortest a line may be before it merges into a neighbour. + pub min_line: f64, + /// Shortest a line clipped by a cut may be before it is dropped. + pub min_visible: f64, } -fn default_caption_words() -> usize { - 4 +/// How a transcript is turned into on-screen captions. Everything but the style +/// is an *override*: omit a field and it follows the style, so asking for +/// [`CaptionStyle::WordPunch`] on its own gets the whole look rather than one +/// word left at subtitle size in the subtitle position. +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, schemars::JsonSchema)] +pub struct CaptionOptions { + /// The look; defaults to [`CaptionStyle::Lines`]. + #[serde(default)] + pub style: CaptionStyle, + /// Most words on one caption line. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_words: Option, + /// Most characters on one caption line; the tighter of the two limits wins. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_chars: Option, + /// Vertical position as a fraction of frame height. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pos_y: Option, + /// Font height as a fraction of frame height. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, } -fn default_caption_chars() -> usize { - 28 -} +impl CaptionOptions { + /// A style with no overrides. + pub fn styled(style: CaptionStyle) -> Self { + Self { + style, + ..Self::default() + } + } -fn default_caption_y() -> f64 { - 0.88 + /// The numbers to caption with: the style's, with any override that is + /// actually usable applied over them. + pub fn resolve(self) -> CaptionLayout { + let base = self.style.layout(); + CaptionLayout { + max_words: self.max_words.map_or(base.max_words, |v| v.max(1)), + max_chars: self.max_chars.map_or(base.max_chars, |v| v.max(1)), + pos_y: overridden(self.pos_y, base.pos_y, 0.0, 1.0), + size: overridden(self.size, base.size, 0.005, 0.5), + ..base + } + } } -fn default_caption_size() -> f64 { - 0.05 -} +/// Roughly how wide one character is as a fraction of the font size, measured +/// off `drawtext`'s default face. Real caption text runs 0.44–0.75 depending on +/// the word; 0.6 sits above the 0.52–0.55 that *long* text averages, and long +/// text is the only kind that ever reaches the cap. +const CHAR_ADVANCE: f64 = 0.6; -impl Default for CaptionOptions { - fn default() -> Self { - Self { - max_words: default_caption_words(), - max_chars: default_caption_chars(), - pos_y: default_caption_y(), - size: default_caption_size(), - } - } +/// How much of the frame width a caption may take. +const CAPTION_WIDTH: f64 = 0.9; + +/// The frame captions assume when the project has not picked one. A timeline +/// cannot see its assets, so it cannot derive the footage default `export_format` +/// would use — and 16:9 is wide enough that the fit below never binds, which is +/// what keeps an unframed project captioned exactly as it was before. +const DEFAULT_CAPTION_ASPECT: f64 = 16.0 / 9.0; + +/// Shrink a caption's size (a fraction of frame height) until its text fits +/// across a frame of `aspect` (width / height). +/// +/// `drawtext` neither wraps nor scales: text wider than the frame is simply +/// drawn off both edges. A 9:16 frame is barely half as wide as it is tall, so +/// the social shape this whole feature is for is exactly where a long word runs +/// off — and `fontsize` cannot be an expression over `text_w`, since the width +/// is what depends on the size. So the fit is estimated here from the character +/// count, which is the only measurement available before the filter runs. +fn fit_size(text: &str, size: f64, aspect: f64) -> f64 { + let chars = text.chars().count().max(1) as f64; + size.min(CAPTION_WIDTH * aspect / (chars * CHAR_ADVANCE)) } -impl CaptionOptions { - fn sanitized(self) -> Self { - Self { - max_words: self.max_words.max(1), - max_chars: self.max_chars.max(1), - pos_y: if self.pos_y.is_finite() { - self.pos_y.clamp(0.0, 1.0) - } else { - default_caption_y() - }, - size: if self.size.is_finite() { - self.size.clamp(0.005, 0.5) - } else { - default_caption_size() - }, - } +/// Apply an optional override, ignoring one that is not a finite number and +/// clamping the rest into range. +fn overridden(v: Option, base: f64, lo: f64, hi: f64) -> f64 { + match v { + Some(v) if v.is_finite() => v.clamp(lo, hi), + _ => base, } } /// Break a transcript line into caption-sized groups of words. Greedy: take /// words until either limit would be exceeded, always at least one (a single /// word longer than `max_chars` is its own line rather than being cut in half). -fn chunk_words(text: &str, opts: CaptionOptions) -> Vec { +fn chunk_words(text: &str, layout: CaptionLayout) -> Vec { let mut out: Vec = Vec::new(); let mut current = String::new(); let mut words = 0usize; @@ -985,7 +1079,7 @@ fn chunk_words(text: &str, opts: CaptionOptions) -> Vec { } else { word.chars().count() + 1 }; - let fits = words < opts.max_words && current.chars().count() + extra <= opts.max_chars; + let fits = words < layout.max_words && current.chars().count() + extra <= layout.max_chars; if !current.is_empty() && !fits { out.push(std::mem::take(&mut current)); words = 0; @@ -1054,7 +1148,10 @@ impl Timeline { /// Reads through [`Timeline::for_render`], so a muted track and a disabled /// clip are as uncaptioned as they are unheard. pub fn captions(&self, transcripts: &HashMap>, opts: CaptionOptions) -> Vec { - let opts = opts.sanitized(); + let layout = opts.resolve(); + let aspect = self + .format + .map_or(DEFAULT_CAPTION_ASPECT, |d| f64::from(d.width) / f64::from(d.height)); let rendered = self.for_render(); let mut lines: Vec<(TimeRange, String)> = Vec::new(); for track in &rendered.tracks { @@ -1072,10 +1169,10 @@ impl Timeline { // each line to the clip — so a sentence cut in half captions // only the half that is still in the cut. let span = clip.source_span_to_timeline(seg.start, seg.end); - for (range, chunk) in time_chunks(chunk_words(text, opts), span, MIN_CAPTION) { + for (range, chunk) in time_chunks(chunk_words(text, layout), span, layout.min_line) { let start = range.start.max(visible_start); let end = range.end.min(visible_end); - if end - start < MIN_CAPTION_VISIBLE { + if end - start < layout.min_visible { continue; } lines.push((TimeRange { start, end }, chunk)); @@ -1099,7 +1196,7 @@ impl Timeline { let start = placed .last() .map_or(range.start, |(prev, _): &(TimeRange, String)| range.start.max(prev.end)); - if range.end - start < MIN_CAPTION_VISIBLE { + if range.end - start < layout.min_visible { continue; } placed.push((TimeRange { start, end: range.end }, text)); @@ -1107,9 +1204,11 @@ impl Timeline { placed .into_iter() .map(|(range, text)| { + let size = fit_size(&text, layout.size, aspect); let mut o = TextOverlay::new(text, range.start.max(0.0), range.end); - o.pos_y = opts.pos_y; - o.size = opts.size; + o.pos_y = layout.pos_y; + o.size = size; + o.bold = layout.bold; o.bg = Some("black@0.5".to_string()); o.generated = true; o @@ -3463,6 +3562,105 @@ mod tests { } } + #[test] + fn word_punch_puts_one_word_on_screen_at_a_time() { + let asset = Uuid::new_v4(); + let timeline = one_clip(Clip::new(asset, 0.0, 5.0, 0.0)); + let mut map = HashMap::new(); + map.insert(asset, vec![seg(0.0, 5.0, "alpha bravo charlie delta echo")]); + let punched = timeline.captions(&map, CaptionOptions::styled(CaptionStyle::WordPunch)); + assert_eq!( + punched.iter().map(|o| o.text.as_str()).collect::>(), + ["alpha", "bravo", "charlie", "delta", "echo"] + ); + // The whole look, not just the word count: the line style would leave + // one word at subtitle size on the bottom edge. + let layout = CaptionStyle::WordPunch.layout(); + assert!(punched + .iter() + .all(|o| o.bold && o.size == layout.size && o.pos_y == layout.pos_y)); + // Each word hands the screen to the next with no gap and no overlap. + for pair in punched.windows(2) { + assert!((pair[1].start - pair[0].end).abs() < 1e-9, "{:?}", (&pair[0], &pair[1])); + } + // The default style is untouched by any of this. + let lines = timeline.captions(&map, CaptionOptions::default()); + assert_eq!(lines.len(), 2, "{lines:?}"); + assert!(lines.iter().all(|o| !o.bold)); + } + + #[test] + fn a_word_too_short_to_read_joins_its_neighbour() { + let asset = Uuid::new_v4(); + let timeline = one_clip(Clip::new(asset, 0.0, 2.0, 0.0)); + let mut map = HashMap::new(); + // "a" is one character of thirty, so its character share is ~0.07s — + // two frames, which is a flicker rather than a word. + map.insert(asset, vec![seg(0.0, 2.0, "a fairly quickly spoken sentence")]); + let punched = timeline.captions(&map, CaptionOptions::styled(CaptionStyle::WordPunch)); + assert!( + punched.iter().all(|o| o.end - o.start >= MIN_WORD_CAPTION - 1e-6), + "{:?}", + punched.iter().map(|o| (&o.text, o.end - o.start)).collect::>() + ); + assert_eq!(punched[0].text, "a fairly", "the flicker merges instead of being dropped"); + } + + #[test] + fn an_override_moves_one_number_and_leaves_the_style_alone() { + let asset = Uuid::new_v4(); + let timeline = one_clip(Clip::new(asset, 0.0, 5.0, 0.0)); + let mut map = HashMap::new(); + map.insert(asset, vec![seg(0.0, 5.0, "alpha bravo charlie delta echo")]); + let opts = CaptionOptions { + size: Some(0.2), + ..CaptionOptions::styled(CaptionStyle::WordPunch) + }; + let punched = timeline.captions(&map, opts); + assert_eq!(punched.len(), 5, "still one word each"); + assert!(punched.iter().all(|o| o.size == 0.2 && o.bold)); + assert!(punched.iter().all(|o| o.pos_y == CaptionStyle::WordPunch.layout().pos_y)); + // An unusable override falls back to the style rather than through it. + let junk = CaptionOptions { + size: Some(f64::NAN), + pos_y: Some(9.0), + ..CaptionOptions::styled(CaptionStyle::WordPunch) + }; + let layout = junk.resolve(); + assert_eq!(layout.size, CaptionStyle::WordPunch.layout().size); + assert_eq!(layout.pos_y, 1.0); + } + + #[test] + fn a_long_word_is_shrunk_to_fit_a_vertical_frame() { + let asset = Uuid::new_v4(); + let mut timeline = one_clip(Clip::new(asset, 0.0, 4.0, 0.0)); + let mut map = HashMap::new(); + map.insert(asset, vec![seg(0.0, 4.0, "non-destructive editing")]); + let opts = CaptionOptions::styled(CaptionStyle::WordPunch); + let full = CaptionStyle::WordPunch.layout().size; + + // Unframed, so 16:9 — wide enough that nothing is shrunk, which is what + // keeps every project that never picked a frame captioned as it was. + let wide = timeline.captions(&map, opts); + assert!(wide.iter().all(|o| o.size == full), "{wide:?}"); + + // 9:16 is barely half as wide as it is tall, and `drawtext` neither + // wraps nor scales: the long word would be drawn off both edges. + timeline.format = Some(Delivery::new(1080, 1920, Fit::Cover)); + let tall = timeline.captions(&map, opts); + let long = tall.iter().find(|o| o.text == "non-destructive").expect("the long word"); + let short = tall.iter().find(|o| o.text == "editing").expect("the short word"); + assert!(long.size < full, "the long word shrinks: {}", long.size); + assert_eq!(short.size, full, "a word that already fits is left alone"); + let aspect = 1080.0 / 1920.0; + assert!( + long.text.chars().count() as f64 * CHAR_ADVANCE * long.size <= CAPTION_WIDTH * aspect + 1e-9, + "still overflows: {}", + long.size + ); + } + #[test] fn a_muted_track_is_not_captioned() { let asset = Uuid::new_v4(); diff --git a/crates/kerf-core/src/project.rs b/crates/kerf-core/src/project.rs index fa9aabf..5d76547 100644 --- a/crates/kerf-core/src/project.rs +++ b/crates/kerf-core/src/project.rs @@ -13,9 +13,9 @@ use crate::engine::{self, ExportProgress}; use crate::error::{Error, Result}; use crate::model::default_beat_tolerance; use crate::model::{ - Asset, AssetAnalysis, AudioEffect, CaptionOptions, Clip, CropFrame, Delivery, EditSource, Keyframe, Marker, Projection, - Reframe, ReframeKeyframe, Revision, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, Tempo, TextKeyframe, TextOverlay, - TimeRange, Timeline, TimelineDiff, Track, TranscriptSegment, Transition, VideoEffect, MAX_FOV, MIN_FOV, + Asset, AssetAnalysis, AudioEffect, CaptionOptions, CaptionStyle, Clip, CropFrame, Delivery, EditSource, Keyframe, Marker, + Projection, Reframe, ReframeKeyframe, Revision, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, Tempo, TextKeyframe, + TextOverlay, TimeRange, Timeline, TimelineDiff, Track, TranscriptSegment, Transition, VideoEffect, MAX_FOV, MIN_FOV, }; /// One clip queued for smart-crop sampling: which media to look at, over which @@ -2823,6 +2823,10 @@ impl Project { /// is on the wrong word. [`Timeline::captions`] does the projection, so /// captions follow the cut and words that were cut out get none. /// + /// `opts.style` picks the look — a held subtitle line, or one word at a + /// time — and everything else in [`CaptionOptions`] is an override on top + /// of it. + /// /// Errors when nothing on the timeline has a transcript to caption, rather /// than quietly writing no overlays. pub fn generate_captions(&self, opts: CaptionOptions) -> Result> { @@ -2853,7 +2857,13 @@ impl Project { )); } let created = overlays.clone(); - self.edit_timeline("Generate captions", move |timeline| { + // Name the style in the edit log: recaptioning in the other one is a + // different edit, and the history is where that has to be visible. + let label = match opts.style { + CaptionStyle::Lines => "Generate captions", + CaptionStyle::WordPunch => "Generate word captions", + }; + self.edit_timeline(label, move |timeline| { timeline.overlays.retain(|o| !o.generated); timeline.overlays.extend(overlays); Ok(()) From 9eb1056ca54b1f41aefe0863c151f607c9073ba8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:01:17 +0200 Subject: [PATCH 02/21] offer the caption style to an agent `generate_captions` takes a `style`, and the instructions say to prefer `word_punch` for a vertical cut: an agent asked for a Reel has no way to know from a tool list that the subtitle shape is not what social captions look like. The other params are documented as overrides on top of the style rather than as defaults, since that is what they now are. --- crates/kerf-app/src/mcp.rs | 41 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/crates/kerf-app/src/mcp.rs b/crates/kerf-app/src/mcp.rs index 901f3e0..530a875 100644 --- a/crates/kerf-app/src/mcp.rs +++ b/crates/kerf-app/src/mcp.rs @@ -15,8 +15,8 @@ use std::sync::{Arc, Mutex, MutexGuard}; use base64::Engine as _; use kerf_core::{ - AudioEffect, CaptionOptions, Delivery, EditSource, ExportOptions, Fit, Keyframe, Project, Projection, ReframeKeyframe, - StreamKind, TextKeyframe, Transition, TransitionKind, VideoEffect, + AudioEffect, CaptionOptions, CaptionStyle, Delivery, EditSource, ExportOptions, Fit, Keyframe, Project, Projection, + ReframeKeyframe, StreamKind, TextKeyframe, Transition, TransitionKind, VideoEffect, }; use rmcp::handler::server::wrapper::Parameters; use rmcp::model::{CallToolResult, ContentBlock, Implementation, ServerCapabilities, ServerInfo}; @@ -69,13 +69,17 @@ struct AssetIdParams { #[derive(Debug, Default, serde::Deserialize, schemars::JsonSchema)] struct CaptionParams { - #[schemars(description = "Most words on one caption line (default 4)")] + #[schemars( + description = "Look: `lines` (default) holds a few words as a subtitle line; `word_punch` puts one large word on screen at a time, the social-video style. Everything below is an override on top of the style — omit them to get the whole look." + )] + style: Option, + #[schemars(description = "Most words on one caption line (lines: 4, word_punch: 1)")] max_words: Option, #[schemars(description = "Most characters on one caption line (default 28); the tighter of the two limits wins")] max_chars: Option, - #[schemars(description = "Vertical position as a fraction of frame height, 0 = top (default 0.88)")] + #[schemars(description = "Vertical position as a fraction of frame height, 0 = top (lines: 0.88, word_punch: 0.72)")] pos_y: Option, - #[schemars(description = "Font height as a fraction of frame height (default 0.05)")] + #[schemars(description = "Font height as a fraction of frame height (lines: 0.05, word_punch: 0.11)")] size: Option, } @@ -1269,22 +1273,16 @@ impl KerfMcp { } #[tool( - description = "Caption the cut: project every clip's cached transcript (run analyze_asset first) through the current edit and write the result as text overlays, replacing any previously generated set. Captions are placed in TIMELINE time, so they follow trims, reorders, speed changes and removed silences, and words that were cut out get no caption. Long sentences are split into readable lines (defaults: 4 words / 28 characters). Hand-made titles and lower-thirds are left alone. Returns the overlays created." + description = "Caption the cut: project every clip's cached transcript (run analyze_asset first) through the current edit and write the result as text overlays, replacing any previously generated set. Captions are placed in TIMELINE time, so they follow trims, reorders, speed changes and removed silences, and words that were cut out get no caption. Long sentences are split into readable lines. Pick the look with `style`: `lines` for subtitles, `word_punch` for one big word at a time (what social captions usually look like — prefer it for a vertical cut). Hand-made titles and lower-thirds are left alone. Returns the overlays created." )] fn generate_captions(&self, Parameters(p): Parameters) -> Result { - let mut opts = CaptionOptions::default(); - if let Some(v) = p.max_words { - opts.max_words = v; - } - if let Some(v) = p.max_chars { - opts.max_chars = v; - } - if let Some(v) = p.pos_y { - opts.pos_y = v; - } - if let Some(v) = p.size { - opts.size = v; - } + let opts = CaptionOptions { + style: p.style.unwrap_or_default(), + max_words: p.max_words, + max_chars: p.max_chars, + pos_y: p.pos_y, + size: p.size, + }; let project = self.lock(); let out = project.generate_captions(opts).map_err(core_err)?; self.changed(); @@ -1812,7 +1810,10 @@ impl ServerHandler for KerfMcp { and captions with add_overlay / update_overlay / set_overlay_keyframes \ (drawn over the cut; list_fonts lists installed system fonts to pass \ as update_overlay's font), or generate_captions to caption the whole \ - cut in one call. Caption LAST, after the cutting is done: captions \ + cut in one call. Its style=word_punch puts one large word on \ + screen at a time instead of a subtitle line — that is what social \ + captions look like, so prefer it for a vertical cut unless the user \ + asked for subtitles. Caption LAST, after the cutting is done: captions \ are placed in timeline time, so a later trim or remove_silence moves \ the words out from under them — re-run generate_captions after any \ further edit and it replaces its own set, leaving typed titles \ From 5f95e750cffc6aabee38a7399479d79c2bcbdb93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:01:25 +0200 Subject: [PATCH 03/21] pick the caption style from the editor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Inspector's Text overlays section leads with a Lines / Word punch choice, and the caption button generates in it. The selection is not derived from the overlays already on the timeline: a caption's style is not recoverable from the text it carries, and guessing it from the word count would flip the chip every time a sentence happened to be short. `captions.ts` mirrors the style table and the frame fit, so the browser harness still produces exactly what the backend would — which is how this was driven end to end: a 9:16 project captions `non-destructive` at 6% and `the` at the style's full 11%. --- frontend/src/lib/api.ts | 6 +- frontend/src/lib/captions.test.ts | 64 +++++++++++++- frontend/src/lib/captions.ts | 83 +++++++++++++++++-- .../lib/components/editor/Inspector.svelte | 18 +++- frontend/src/lib/style-presets.ts | 24 +++++- frontend/src/lib/types.ts | 10 ++- 6 files changed, 187 insertions(+), 18 deletions(-) diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 439deaf..0f55058 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -44,7 +44,7 @@ import { alignCutsToBeats, beatGrid, defaultBeatTolerance } from './beats'; import { formatTime as fmtTime } from './diff'; import { checkAll } from './platforms'; import { centeredCrop } from './smart-crop'; -import { captionsForTimeline, CAPTION_DEFAULTS } from './captions'; +import { captionsForTimeline, resolveCaptions } from './captions'; export function inTauri(): boolean { return typeof window !== 'undefined' && '__TAURI_INTERNALS__' in window; @@ -1206,10 +1206,10 @@ export async function generateCaptions(options?: CaptionOptions): Promise !o.generated); devTimeline.overlays = [...kept, ...created.map((o) => ({ ...o, id: uid() }))]; - recordDev('Generate captions'); + recordDev(options?.style === 'word_punch' ? 'Generate word captions' : 'Generate captions'); return snapshot(); } return invoke('generate_captions', { options: options ?? null }); diff --git a/frontend/src/lib/captions.test.ts b/frontend/src/lib/captions.test.ts index cb22b06..4262179 100644 --- a/frontend/src/lib/captions.test.ts +++ b/frontend/src/lib/captions.test.ts @@ -5,8 +5,11 @@ import { coversSource, sourceToTimeline, timeChunks, + resolveCaptions, CAPTION_DEFAULTS, - MIN_CAPTION + CAPTION_STYLES, + MIN_CAPTION, + MIN_WORD_CAPTION } from './captions'; import type { Clip, Timeline, TranscriptSegment } from './types'; @@ -125,4 +128,63 @@ describe('captions follow the cut', () => { tl.tracks[0].muted = true; expect(captionsForTimeline(tl, { a1: [seg(0, 4, 'heard')] })).toHaveLength(0); }); + + test('word punch puts one word on screen at a time', () => { + const tl = timelineOf([clip({ source_out: 5 })]); + const words = { a1: [seg(0, 5, 'alpha bravo charlie delta echo')] }; + const punched = captionsForTimeline(tl, words, resolveCaptions({ style: 'word_punch' })); + expect(punched.map((o) => o.text)).toEqual(['alpha', 'bravo', 'charlie', 'delta', 'echo']); + // The whole look, not just the word count. + for (const o of punched) { + expect(o.bold).toBe(true); + expect(o.size).toBe(CAPTION_STYLES.word_punch.size); + expect(o.pos_y).toBe(CAPTION_STYLES.word_punch.pos_y); + } + // Each word hands the screen straight to the next. + for (let i = 1; i < punched.length; i++) expect(punched[i].start).toBeCloseTo(punched[i - 1].end, 9); + // The default style is untouched by any of this. + const lines = captionsForTimeline(tl, words); + expect(lines).toHaveLength(2); + expect(lines.every((o) => !o.bold)).toBe(true); + }); + + test('a word too short to read joins its neighbour', () => { + // "a" is one character of thirty, so its character share is two frames. + const punched = captionsForTimeline( + timelineOf([clip({ source_out: 2 })]), + { a1: [seg(0, 2, 'a fairly quickly spoken sentence')] }, + resolveCaptions({ style: 'word_punch' }) + ); + for (const o of punched) expect(o.end - o.start).toBeGreaterThanOrEqual(MIN_WORD_CAPTION - 1e-6); + expect(punched[0].text).toBe('a fairly'); + }); + + test('an override moves one number and leaves the style alone', () => { + const resolved = resolveCaptions({ style: 'word_punch', size: 0.2 }); + expect(resolved.max_words).toBe(1); + expect(resolved.size).toBe(0.2); + expect(resolved.pos_y).toBe(CAPTION_STYLES.word_punch.pos_y); + // An unusable override falls back to the style rather than through it. + expect(resolveCaptions({ style: 'word_punch', size: NaN }).size).toBe(CAPTION_STYLES.word_punch.size); + expect(resolveCaptions({ style: 'word_punch', pos_y: 9 }).pos_y).toBe(1); + // No options at all is the line style. + expect(resolveCaptions()).toEqual(CAPTION_DEFAULTS); + }); + + test('a long word is shrunk to fit a vertical frame', () => { + const tl = timelineOf([clip({ source_out: 4 })]); + const words = { a1: [seg(0, 4, 'non-destructive editing')] }; + const punch = resolveCaptions({ style: 'word_punch' }); + // Unframed, so 16:9 — wide enough that nothing is shrunk. + expect(captionsForTimeline(tl, words, punch).every((o) => o.size === punch.size)).toBe(true); + // 9:16: drawtext neither wraps nor scales, so the long word would be + // drawn off both edges. + tl.format = { width: 1080, height: 1920, fit: 'cover' }; + const tall = captionsForTimeline(tl, words, punch); + const long = tall.find((o) => o.text === 'non-destructive')!; + const short = tall.find((o) => o.text === 'editing')!; + expect(long.size).toBeLessThan(punch.size); + expect(short.size).toBe(punch.size); + expect('non-destructive'.length * 0.6 * long.size).toBeLessThanOrEqual(0.9 * (1080 / 1920) + 1e-9); + }); }); diff --git a/frontend/src/lib/captions.ts b/frontend/src/lib/captions.ts index bdda44e..b6f2288 100644 --- a/frontend/src/lib/captions.ts +++ b/frontend/src/lib/captions.ts @@ -8,21 +8,88 @@ * the same arrangement as `platforms.ts` and `smart-crop.ts`. */ -import type { Clip, TextOverlay, Timeline, TranscriptSegment } from './types'; +import type { CaptionOptions, CaptionStyle, Clip, TextOverlay, Timeline, TranscriptSegment } from './types'; /** Shortest a generated line stays on screen; below this it reads as a flicker. */ export const MIN_CAPTION = 0.45; /** How much of a line has to survive a cut for it to be kept. */ export const MIN_CAPTION_VISIBLE = 0.15; +/** The same two floors where a line *is* one word: held to `MIN_CAPTION` every + * short word would merge into a neighbour and word punch would collapse back + * into lines. */ +export const MIN_WORD_CAPTION = 0.12; +export const MIN_WORD_VISIBLE = 0.06; +/** A style's numbers with any override applied — what captioning works from. */ export interface CaptionOpts { max_words: number; max_chars: number; pos_y: number; size: number; + bold: boolean; + min_line: number; + min_visible: number; } -export const CAPTION_DEFAULTS: CaptionOpts = { max_words: 4, max_chars: 28, pos_y: 0.88, size: 0.05 }; +/** The two looks. A subtitle line is read; a punched word is watched — so the + * word count, the size, the position and the flicker floors move together. */ +export const CAPTION_STYLES: Record = { + lines: { + max_words: 4, + max_chars: 28, + pos_y: 0.88, + size: 0.05, + bold: false, + min_line: MIN_CAPTION, + min_visible: MIN_CAPTION_VISIBLE + }, + word_punch: { + max_words: 1, + max_chars: 28, + pos_y: 0.72, + size: 0.11, + bold: true, + min_line: MIN_WORD_CAPTION, + min_visible: MIN_WORD_VISIBLE + } +}; + +export const CAPTION_DEFAULTS: CaptionOpts = CAPTION_STYLES.lines; + +/** The style's numbers with any usable override applied over them. Mirrors + * `CaptionOptions::resolve`: omitted fields follow the style, so asking for + * `word_punch` alone gets the whole look rather than one word left at subtitle + * size in the subtitle position. */ +export function resolveCaptions(opts?: CaptionOptions): CaptionOpts { + const base = CAPTION_STYLES[opts?.style ?? 'lines'] ?? CAPTION_DEFAULTS; + const num = (v: number | undefined, fallback: number, lo: number, hi: number) => + typeof v === 'number' && Number.isFinite(v) ? Math.min(Math.max(v, lo), hi) : fallback; + return { + ...base, + max_words: typeof opts?.max_words === 'number' ? Math.max(opts.max_words, 1) : base.max_words, + max_chars: typeof opts?.max_chars === 'number' ? Math.max(opts.max_chars, 1) : base.max_chars, + pos_y: num(opts?.pos_y, base.pos_y, 0, 1), + size: num(opts?.size, base.size, 0.005, 0.5) + }; +} + +/** Roughly how wide one character is as a fraction of the font size, measured + * off `drawtext`'s default face; 0.6 sits above the ~0.53 that long text — the + * only kind that reaches the cap — averages. */ +const CHAR_ADVANCE = 0.6; +/** How much of the frame width a caption may take. */ +const CAPTION_WIDTH = 0.9; +/** The frame captions assume when the project has not picked one: wide enough + * that the fit never binds, so an unframed project captions as it always did. */ +const DEFAULT_CAPTION_ASPECT = 16 / 9; + +/** Shrink a caption's size until its text fits across a frame of `aspect`. + * `drawtext` neither wraps nor scales, and a 9:16 frame is barely half as wide + * as it is tall — so the social shape is exactly where a long word runs off + * both edges. Mirrors `fit_size`. */ +export function fitSize(text: string, size: number, aspect: number): number { + return Math.min(size, (CAPTION_WIDTH * aspect) / (Math.max([...text].length, 1) * CHAR_ADVANCE)); +} const MIN_SPEED = 0.01; const speedMag = (c: Clip) => Math.max(Math.abs(c.speed ?? 1), MIN_SPEED); @@ -109,6 +176,8 @@ export function captionsForTimeline( transcripts: Record, opts: CaptionOpts = CAPTION_DEFAULTS ): Omit[] { + const fmt = timeline.format; + const aspect = fmt && fmt.height ? fmt.width / fmt.height : DEFAULT_CAPTION_ASPECT; const soloed = new Set(timeline.tracks.filter((t) => t.solo).map((t) => t.kind)); const lines: { start: number; end: number; text: string }[] = []; for (const track of timeline.tracks) { @@ -126,10 +195,10 @@ export function captionsForTimeline( // line — so a sentence cut in half captions only the surviving half. const a = sourceToTimeline(clip, seg.start); const b = sourceToTimeline(clip, seg.end); - for (const line of timeChunks(chunkWords(text, opts), Math.min(a, b), Math.max(a, b))) { + for (const line of timeChunks(chunkWords(text, opts), Math.min(a, b), Math.max(a, b), opts.min_line)) { const start = Math.max(line.start, visibleStart); const end = Math.min(line.end, visibleEnd); - if (end - start < MIN_CAPTION_VISIBLE) continue; + if (end - start < opts.min_visible) continue; lines.push({ start, end, text: line.text }); } } @@ -145,7 +214,7 @@ export function captionsForTimeline( const placed: typeof deduped = []; for (const l of deduped) { const start = placed.length ? Math.max(l.start, placed[placed.length - 1].end) : l.start; - if (l.end - start < MIN_CAPTION_VISIBLE) continue; + if (l.end - start < opts.min_visible) continue; placed.push({ ...l, start }); } return placed.map((l) => ({ @@ -154,10 +223,10 @@ export function captionsForTimeline( end: l.end, pos_x: 0.5, pos_y: opts.pos_y, - size: opts.size, + size: fitSize(l.text, opts.size, aspect), color: 'white', bg: 'black@0.5', - bold: false, + bold: opts.bold, generated: true })); } diff --git a/frontend/src/lib/components/editor/Inspector.svelte b/frontend/src/lib/components/editor/Inspector.svelte index bf40d84..85ba2fa 100644 --- a/frontend/src/lib/components/editor/Inspector.svelte +++ b/frontend/src/lib/components/editor/Inspector.svelte @@ -7,11 +7,12 @@ import { contextMenu } from '$lib/context-menu.svelte'; import type { MenuItem } from '$lib/context-menu.svelte'; import { clipDuration, DEFAULT_COLOR, DEFAULT_REFRAME, DEFAULT_TRANSFORM } from '$lib/types'; - import { COLOR_LOOKS, TEXT_STYLES, activeLook } from '$lib/style-presets'; + import { CAPTION_LOOKS, COLOR_LOOKS, TEXT_STYLES, activeLook } from '$lib/style-presets'; import { needsCrop } from '$lib/smart-crop'; import type { TextStyle } from '$lib/style-presets'; import type { AudioEffect, + CaptionStyle, Projection, Reframe, Transform, @@ -224,12 +225,17 @@ * replaces the generated set, so the button stays the same after the first * press and only its label admits what it is doing. */ const hasCaptions = $derived(overlays.some((o) => o.generated)); + // Which look the button generates in. Not derived from the overlays already + // on the timeline: a caption's style is not recoverable from the text it + // carries, and guessing it from the word count would flip the selection + // every time a sentence happened to be short. + let captionStyle = $state('lines'); function makeCaptions() { if (!editor.timeline.tracks.some((t) => t.clips.length > 0)) { toast.error('Put a clip on the timeline first'); return; } - void run(() => editor.generateCaptions()); + void run(() => editor.generateCaptions({ style: captionStyle })); } function dropCaptions() { void run(() => editor.clearCaptions()); @@ -481,6 +487,14 @@ {/each} +
+ Caption style + {#each CAPTION_LOOKS as c (c.id)} + + {/each} +
+ Text diff --git a/frontend/src/lib/style-presets.ts b/frontend/src/lib/style-presets.ts index bec24fc..61a1fd6 100644 --- a/frontend/src/lib/style-presets.ts +++ b/frontend/src/lib/style-presets.ts @@ -3,7 +3,7 @@ // result without manual grading or typography, so these are deliberately few // and tasteful rather than deep. -import type { Color, TextOverlay } from './types'; +import type { CaptionStyle, Color, TextOverlay } from './types'; export interface ColorLook { id: string; @@ -48,8 +48,8 @@ export interface TextStyle { } /** Ready-made title / lower-third / caption styles. The caption style matches - * what `captions_from_transcript` generates, so manual and generated captions - * look the same. */ + * what `generate_captions` writes in its `lines` style, so a manual caption and + * a generated one look the same. */ export const TEXT_STYLES: TextStyle[] = [ { id: 'title', @@ -76,3 +76,21 @@ export const TEXT_STYLES: TextStyle[] = [ fade: 0 } ]; + +export interface CaptionLook { + id: CaptionStyle; + label: string; + hint: string; +} + +/** The two looks the caption button can generate in. Labels only — the numbers + * behind them live in `captions.ts`, mirroring kerf-core, so there is one place + * a style is defined. */ +export const CAPTION_LOOKS: CaptionLook[] = [ + { id: 'lines', label: 'Lines', hint: 'A few words held on screen, like subtitles' }, + { + id: 'word_punch', + label: 'Word punch', + hint: 'One large word at a time, cut on the speech — the social-video look' + } +]; diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index f06d8b2..2d10ebc 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -193,9 +193,15 @@ export interface TextOverlay { generated?: boolean; } -/** How a transcript is turned into on-screen captions. Omitted fields keep the - * backend's social-video defaults (4 words / 28 chars, low-centre). */ +/** The look a generated caption set takes: a held subtitle line, or one large + * word at a time — the style social captions have converged on. */ +export type CaptionStyle = 'lines' | 'word_punch'; + +/** How a transcript is turned into on-screen captions. Everything but the style + * is an override: omit a field and it follows the style, so asking for + * `word_punch` alone gets the whole look. */ export interface CaptionOptions { + style?: CaptionStyle; max_words?: number; max_chars?: number; pos_y?: number; From 78e626dfedbc7f99580ae6275061664845aa14a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:01:54 +0200 Subject: [PATCH 04/21] document the caption style --- CLAUDE.md | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 37e8a05..64e3d44 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -275,14 +275,34 @@ no editing logic in the adapter. speed / reverse) — captions land on the words that survived the cut and words that were cut out get none. It reads through `for_render`, so a muted track is as uncaptioned as it is unheard; it chunks a sentence to `CaptionOptions` - (4 words / 28 chars by default — a speech model emits whole sentences and a - whole sentence does not fit a 9:16 frame), timing lines by *character share* + (a speech model emits whole sentences and a whole sentence does not fit a + 9:16 frame), timing lines by *character share* because neither speech backend reports word timings; lines too short to read merge back into a neighbour instead of flashing; and no two lines are ever on screen at once (captions are one lane of text, and the same footage reaching the cut twice would otherwise collide with itself). `TextOverlay.generated` marks what it wrote, so regenerating replaces its own set and leaves a typed - title alone. A `Track` + title alone. + **`CaptionStyle` is the look**, and one decision rather than four: + `Lines` (4 words / 28 chars, 5% of frame height, low in the frame) is the + subtitle shape a line is *read* in; `WordPunch` (one word, 11%, higher, bold) + is the social shape a word is *watched* in, each landing on the beat of the + speech. Word count, size, position and the flicker floors move together + because they have to — held to `MIN_CAPTION` every short word would merge + into a neighbour and word punch would collapse back into lines, so it gets + its own `MIN_WORD_CAPTION` / `MIN_WORD_VISIBLE` and words merge far later. + `CaptionOptions` is that style plus **overrides**: every number is optional + and follows the style when omitted, `resolve()`ing to the `CaptionLayout` + captioning works from — so asking for `word_punch` alone gets the whole look + rather than one word left at subtitle size, and `CaptionOptions::default()` + is unchanged, so every pre-existing call captions identically. `fit_size` + then shrinks a caption to fit the frame: `drawtext` neither wraps nor scales + and a 9:16 frame is barely half as wide as it is tall, so a long word — or a + 28-char subtitle line, already true before word punch — was drawn off both + edges. `fontsize` cannot be an expression over `text_w` (the width is what + depends on the size), so it is estimated from the character count against + `Timeline.format`'s aspect; an unframed project assumes 16:9, wide enough + that the fit never binds, so nothing that never picked a frame moved. A `Track` carries a `duck` flag (sidechain-ducked under the rest of the mix on export). `Fit` and `Delivery` live here (the domain owns the delivery shape; `engine::cli` re-exports `Fit`), and `Timeline.format` is the frame the project is cut for. @@ -431,7 +451,10 @@ proposal appears for review, not that the cut changes: the read tools `smart_crop` frames each shot for the delivery frame (the server `instructions` pair it with `set_delivery_format`, since reshaping to 9:16 otherwise keeps whatever was in the middle). `generate_captions` / `clear_captions` caption the -cut; the `instructions` say to caption **last** and to re-run after any further +cut; its `style` picks `lines` or `word_punch` and the `instructions` say to +prefer the latter for a vertical cut, since nothing in a tool list tells an +agent that the subtitle shape is not what social captions look like. They also +say to caption **last** and to re-run after any further edit, because captions are placed in timeline time and a later trim moves the words out from under them — which an agent has no way to infer from the tool list. @@ -588,7 +611,11 @@ at the playhead like Transform — note its `lerpAngle` takes the shortest arc, plain `lerp` would read as a 340° swing across the seam; for a source Kerf did not detect as 360 it instead offers a projection picker that marks the whole asset via `set_asset_projection`), and an always-visible -**Text overlays** section (add titles / lower-thirds, caption the whole cut — +**Text overlays** section (add titles / lower-thirds, caption the whole cut in +a **Lines / Word punch** style chosen by two chips above the button — the +selection is deliberately *not* derived from the overlays already there, since +a caption's style is not recoverable from its text and guessing it from the +word count would flip the chip whenever a sentence happened to be short — the button relabels to `Recaption` once there are generated captions, since a later trim moves the words out from under them, with `Clear` beside it taking only the generated ones — and edit text / timing / position / size / color / @@ -602,8 +629,9 @@ omitted at 0 so old graphs stay byte-identical; plain saturation/gamma can't tint) — and the Text overlays section leads with **Title / Lower third / Caption** style chips that create a styled overlay at the playhead with fade-in/out opacity keyframes; the caption style matches what -`generate_captions` generates, so manual and generated captions look -alike. +`generate_captions` writes in its `lines` style, so manual and generated +captions look alike (`CAPTION_LOOKS` in the same file is only the two +generate-time labels; their numbers live in `captions.ts`). Everything is styled with the CSS-variable tokens directly (inline `style`), not Tailwind utilities. The **timeline is a bespoke NLE timeline** that renders **real `editor.timeline` state** (ruler + tracks + clips positioned by `timeline_start`/duration at `ui.zoom` From 0eeee84d11bfd1de7b21dfe78f1c845887871eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:03:09 +0200 Subject: [PATCH 05/21] quote animated position expressions in the filtergraph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A piecewise keyframe expression contains commas, and the filtergraph parser reads an unquoted comma as the end of the filter — so `overlay=x=` for a clip with position keyframes, and the `drawtext` x/y of an animated text overlay, both made ffmpeg abort the whole render with `No such filter: '2)'`. Neither is exotic: the Text overlays style chips animate an overlay's opacity, which puts every overlay they create on the keyframed branch, so a project that used one could not be exported at all. Every unit test asserted on the graph string, and the graph string looked right — so the regression test renders one of each and asks ffmpeg to accept the graph. --- crates/kerf-core/src/engine/cli.rs | 95 ++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 4 deletions(-) diff --git a/crates/kerf-core/src/engine/cli.rs b/crates/kerf-core/src/engine/cli.rs index 96d08df..cc0b3fc 100644 --- a/crates/kerf-core/src/engine/cli.rs +++ b/crates/kerf-core/src/engine/cli.rs @@ -3363,8 +3363,11 @@ fn build_filter_complex( let kf = clip.sorted_keyframes(); let xs: Vec<(f64, f64)> = kf.iter().map(|k| (k.time, k.pos_x)).collect(); let ys: Vec<(f64, f64)> = kf.iter().map(|k| (k.time, k.pos_y)).collect(); + // Quoted for the same reason as the animated `drawtext` position: + // the piecewise expression contains commas, and an unquoted comma + // is where the graph parser thinks the filter ends. format!( - "overlay=x=(W-w)/2+({px})*W:y=(H-h)/2+({py})*H:\ + "overlay=x='(W-w)/2+({px})*W':y='(H-h)/2+({py})*H':\ eof_action=pass:enable='between(t,{start},{end})'", px = keyframe_expr(&xs, "t", clip.timeline_start), py = keyframe_expr(&ys, "t", clip.timeline_start), @@ -3788,8 +3791,10 @@ fn drawtext_export(o: &TextOverlay, fmt: &ExportFormat) -> String { let xs: Vec<(f64, f64)> = o.keyframes.iter().map(|k| (k.time, k.pos_x)).collect(); let ys: Vec<(f64, f64)> = o.keyframes.iter().map(|k| (k.time, k.pos_y)).collect(); let al: Vec<(f64, f64)> = o.keyframes.iter().map(|k| (k.time, k.opacity)).collect(); - parts.push(format!("x=(w*({})-text_w/2)", keyframe_expr(&xs, "t", o.start))); - parts.push(format!("y=(h*({})-text_h/2)", keyframe_expr(&ys, "t", o.start))); + // Quoted: a piecewise expression contains commas, and an unquoted comma + // ends the filter as far as the graph parser is concerned. + parts.push(format!("x='(w*({})-text_w/2)'", keyframe_expr(&xs, "t", o.start))); + parts.push(format!("y='(h*({})-text_h/2)'", keyframe_expr(&ys, "t", o.start))); parts.push(format!("alpha='{}'", keyframe_expr(&al, "t", o.start))); } parts.push(format!("enable='between(t,{},{})'", fnum(o.start), fnum(o.end))); @@ -5520,7 +5525,7 @@ mod tests { &plan_inputs(&timeline, &assets, &transition_fx(&timeline, &assets)), ); assert!( - g.filter.contains("overlay=x=(W-w)/2+(if(lt((t-2)"), + g.filter.contains("overlay=x='(W-w)/2+(if(lt((t-2)"), "animated overlay x: {}", g.filter ); @@ -7650,6 +7655,88 @@ mod tests { assert!(bright > 60.0, "cover should fill the top with picture, got luma {bright}"); } + /// A piecewise position expression contains commas, and the filtergraph + /// parser treats an unquoted comma as the end of the filter — so a clip with + /// position keyframes, or an *animated text overlay*, made ffmpeg fail the + /// whole render with `No such filter`. Nothing caught it: every unit test + /// above asserts on the graph string, and the graph string looked right. + /// + /// Both are on the default path — the Text overlays style chips animate an + /// overlay's opacity, which puts it on the keyframed branch — so this renders + /// one of each and only asks that ffmpeg accept the graph. + /// + /// `cargo test -p kerf-core --no-default-features -- --ignored animated_positions` + #[test] + #[ignore = "needs the ffmpeg binary"] + fn animated_positions_survive_the_filtergraph_parser() { + let dir = std::env::temp_dir().join(format!("kerf-anim-parse-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let media = dir.join("src.mp4"); + let ok = command(&ffmpeg_bin()) + .args(["-hide_banner", "-loglevel", "error", "-y"]) + .args(["-f", "lavfi", "-i", "testsrc=size=320x180:rate=30:duration=2"]) + .args(["-c:v", "libx264", "-pix_fmt", "yuv420p"]) + .arg(&media) + .status() + .expect("run ffmpeg"); + assert!(ok.success()); + + let mut asset = av_asset(Uuid::new_v4(), 2.0); + asset.path = media.to_string_lossy().into_owned(); + asset.streams = vec![video_stream(320, 180, 30.0)]; + + let mut clip = make_clip(asset.id, 0.0, 2.0, 0.0); + clip.keyframes = vec![ + crate::model::Keyframe { + time: 0.0, + scale: 1.0, + pos_x: -0.2, + pos_y: 0.0, + rotation: 0.0, + opacity: 1.0, + }, + crate::model::Keyframe { + time: 2.0, + scale: 1.0, + pos_x: 0.2, + pos_y: 0.0, + rotation: 0.0, + opacity: 1.0, + }, + ]; + let mut timeline = single(vec![clip]); + // An overlay animated the way the Text overlays style chips animate one. + let mut overlay = TextOverlay::new("Kerf".to_string(), 0.0, 2.0); + overlay.keyframes = vec![ + crate::model::TextKeyframe { + time: 0.0, + pos_x: 0.5, + pos_y: 0.5, + opacity: 0.0, + }, + crate::model::TextKeyframe { + time: 1.0, + pos_x: 0.5, + pos_y: 0.8, + opacity: 1.0, + }, + ]; + timeline.overlays = vec![overlay]; + + let out = dir.join("anim.mp4"); + let opts = ExportOptions { + container: Container::Mp4, + video_codec: Some("libx264".into()), + include_audio: false, + ..Default::default() + }; + let rendered = render_with(&timeline, &[asset], &out, &opts); + let ok = rendered.is_ok() && out.exists(); + let err = rendered.err().map(|e| e.to_string()).unwrap_or_default(); + let _ = std::fs::remove_dir_all(&dir); + assert!(ok, "an animated clip and overlay must render: {err}"); + } + /// The delivery frame is the point of the whole feature: with it set and /// **no** export resolution at all, a 16:9 source must still render a filled /// 1080x1920 file — the same thing the preview showed while cutting. From 7931544a1d39dc50358651f16c1fe48a8a750fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:03:56 +0200 Subject: [PATCH 06/21] give the engine a transition set worth using MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two options — a dissolve and a dip to black — is where a montage starts to look homemade, and a travel or brand cut is mostly shot-to-shot movement. Adds nine: a dip to white, and slide / push in four directions each. The families are what decide the render, so `TransitionKind` answers for them rather than the engine matching on every variant: a **dip** takes both sides through a solid colour and needs no handle, a **dissolve** mixes them, and a **motion** transition travels the incoming clip in over the outgoing one (slide) or carries the outgoing one out with it (push). No new machinery in the graph. A motion transition is an overlay offset, and an overlay offset is what `keyframe_expr` already writes — so a slide is two keyframes, reusing the tested expression builder rather than a second expression language. `ClipFx` grows the offsets, the existing crossfade rules for borrowing the outgoing clip's source handle apply unchanged (no handle still degrades to a hard cut), and the travel is *added* to whatever position the clip already has, so a picture-in-picture slides in to where it lives. The sound dissolves under a slide either way: the picture moving is no reason for the audio to cut hard. A slide and a push differ only in whether the outgoing clip moves, which no assertion on the graph string can see — so the real-ffmpeg test renders both and looks at where the stripe went. --- crates/kerf-core/src/engine/cli.rs | 376 +++++++++++++++++++++++++---- crates/kerf-core/src/model.rs | 121 ++++++++++ 2 files changed, 454 insertions(+), 43 deletions(-) diff --git a/crates/kerf-core/src/engine/cli.rs b/crates/kerf-core/src/engine/cli.rs index cc0b3fc..e4c4ab7 100644 --- a/crates/kerf-core/src/engine/cli.rs +++ b/crates/kerf-core/src/engine/cli.rs @@ -16,7 +16,7 @@ use super::ProbeResult; use crate::error::{Error, Result}; use crate::model::{ Asset, AudioEffect, Clip, Color, Projection, Reframe, ReframeKeyframe, ResolvedReframe, SalienceMap, StreamInfo, StreamKind, - TextOverlay, TimeRange, Timeline, Transform, TransitionKind, VideoEffect, + TextOverlay, TimeRange, Timeline, Transform, VideoEffect, }; /// A small process-global LRU of decoded single frames. Decoded frames are a @@ -3358,33 +3358,47 @@ fn build_filter_complex( format!("vov{n}") }; let end = clip.timeline_end() + fx[*flat].tail; + // A slide / push adds its travel to whatever position the clip already + // has, so a transition composes with a static offset or an animated one + // instead of overriding it. + let motion = motion_expr(clip, &fx[*flat]); + // A position that varies over time is a piecewise expression, and a + // piecewise expression contains commas — which the graph parser reads + // as the end of the filter unless the value is quoted. A plain static + // offset has none, and stays unquoted. + let quote = |v: String, dynamic: bool| if dynamic { format!("'{v}'") } else { v }; let overlay = if clip.is_animated() { // Animated picture position: per-frame overlay x / y expressions. let kf = clip.sorted_keyframes(); let xs: Vec<(f64, f64)> = kf.iter().map(|k| (k.time, k.pos_x)).collect(); let ys: Vec<(f64, f64)> = kf.iter().map(|k| (k.time, k.pos_y)).collect(); - // Quoted for the same reason as the animated `drawtext` position: - // the piecewise expression contains commas, and an unquoted comma - // is where the graph parser thinks the filter ends. + let px = keyframe_expr(&xs, "t", clip.timeline_start); + let py = keyframe_expr(&ys, "t", clip.timeline_start); + let (px, py) = match &motion { + Some((mx, my)) => (format!("({px})+({mx})"), format!("({py})+({my})")), + None => (px, py), + }; format!( - "overlay=x='(W-w)/2+({px})*W':y='(H-h)/2+({py})*H':\ - eof_action=pass:enable='between(t,{start},{end})'", - px = keyframe_expr(&xs, "t", clip.timeline_start), - py = keyframe_expr(&ys, "t", clip.timeline_start), + "overlay=x={px}:y={py}:eof_action=pass:enable='between(t,{start},{end})'", + px = quote(format!("(W-w)/2+({px})*W"), true), + py = quote(format!("(H-h)/2+({py})*H"), true), start = clip.timeline_start, ) - } else if clip.transform.is_identity() { + } else if clip.transform.is_identity() && motion.is_none() { format!( "overlay=eof_action=pass:enable='between(t,{start},{end})'", start = clip.timeline_start ) } else { let t = &clip.transform; + let (px, py) = match &motion { + Some((mx, my)) => (format!("({})+({mx})", t.pos_x), format!("({})+({my})", t.pos_y)), + None => (t.pos_x.to_string(), t.pos_y.to_string()), + }; format!( - "overlay=x=(W-w)/2+({px})*W:y=(H-h)/2+({py})*H:\ - eof_action=pass:enable='between(t,{start},{end})'", - px = t.pos_x, - py = t.pos_y, + "overlay=x={px}:y={py}:eof_action=pass:enable='between(t,{start},{end})'", + px = quote(format!("(W-w)/2+({px})*W"), motion.is_some()), + py = quote(format!("(H-h)/2+({py})*H"), motion.is_some()), start = clip.timeline_start, ) }; @@ -3481,15 +3495,29 @@ fn build_filter_complex( } /// Per-clip render adjustments derived from transitions. `tail` extends an -/// outgoing clip so it keeps showing under the incoming crossfade; `xfade_in` -/// is the incoming clip's alpha dissolve; `black_in`/`black_out` are the -/// dip-to-black fades on either side of a cut. +/// outgoing clip so it keeps showing under the incoming one; `xfade_in` is the +/// incoming clip's alpha dissolve; `black_in`/`black_out` and `white_in`/ +/// `white_out` are the dip fades on either side of a cut; `move_in`/`move_out` +/// carry a clip across the frame for a slide or a push. #[derive(Clone, Copy, Default)] struct ClipFx { tail: f64, xfade_in: f64, black_in: f64, black_out: f64, + /// Dip-to-white fades: the same shape as `black_in`/`black_out`, through white. + white_in: f64, + white_out: f64, + /// How long the incoming clip's **sound** dissolves up. Equal to `xfade_in` + /// for a crossfade; a motion transition sets it too, because the picture + /// sliding is no reason for the audio to cut hard. + afade_in: f64, + /// Motion transitions, as `(dx, dy, seconds)` with the offsets in frame + /// widths and heights. `move_in` is where the incoming clip starts before + /// travelling to its position; `move_out` is where the outgoing clip is + /// carried to over its tail (a push only — a slide covers it where it sits). + move_in: Option<(f64, f64, f64)>, + move_out: Option<(f64, f64, f64)>, } /// Compute the [`ClipFx`] for every clip (indexed by ffmpeg input index, i.e. @@ -3519,35 +3547,68 @@ fn transition_fx(timeline: &Timeline, assets: &[Asset]) -> Vec { let prev = (w > 0) .then(|| order[w - 1]) .filter(|&pj| (track.clips[pj].timeline_end() - clip.timeline_start).abs() < 1e-3); - match tr.kind { - TransitionKind::Crossfade => match prev { - Some(pj) => { - let p = &track.clips[pj]; - // The tail borrows the outgoing clip's unused source: for a - // forward clip that is the handle past source_out, for a - // reversed clip the handle below source_in. - let avail = if p.is_reversed() { - p.source_in / p.speed_mag() - } else { - asset_dur(p.asset_id).map(|ad| (ad - p.source_out).max(0.0)).unwrap_or(0.0) / p.speed_mag() - }; - // Both sides share the achievable overlap so the dissolve - // length matches the tail (no fade-from-black when there is - // no handle — it just becomes a hard cut). - let overlap = d.min(p.duration()).min(clip.duration()).min(avail.max(0.0)); - fx[base + j].xfade_in = overlap; - fx[base + pj].tail = fx[base + pj].tail.max(overlap); + match tr.kind.dip_color() { + // A dip happens either side of the cut — the two clips never share + // the screen, so neither needs a handle and neither is extended. + Some(color) => { + let white = color == "white"; + let inn = (d / 2.0).min(clip.duration()); + if white { + fx[base + j].white_in = inn; + } else { + fx[base + j].black_in = inn; } - // No adjacent predecessor: dissolve up from black. - None => fx[base + j].xfade_in = d.min(clip.duration()), - }, - TransitionKind::DipToBlack => { - fx[base + j].black_in = (d / 2.0).min(clip.duration()); if let Some(pj) = prev { let p = &track.clips[pj]; let out = (d / 2.0).min(p.duration()); - fx[base + pj].black_out = fx[base + pj].black_out.max(out); + if white { + fx[base + pj].white_out = fx[base + pj].white_out.max(out); + } else { + fx[base + pj].black_out = fx[base + pj].black_out.max(out); + } + } + } + // A dissolve or a motion transition plays both sides at once, so + // the outgoing clip keeps rolling underneath on its unused handle. + None => { + let slide = tr.kind.slide_from(); + let overlap = match prev { + Some(pj) => { + let p = &track.clips[pj]; + // The tail borrows the outgoing clip's unused source: for a + // forward clip that is the handle past source_out, for a + // reversed clip the handle below source_in. + let avail = if p.is_reversed() { + p.source_in / p.speed_mag() + } else { + asset_dur(p.asset_id).map(|ad| (ad - p.source_out).max(0.0)).unwrap_or(0.0) / p.speed_mag() + }; + // Both sides share the achievable overlap so the transition + // length matches the tail (no fade-from-black when there is + // no handle — it just becomes a hard cut). + let overlap = d.min(p.duration()).min(clip.duration()).min(avail.max(0.0)); + fx[base + pj].tail = fx[base + pj].tail.max(overlap); + if overlap > 0.0 && tr.kind.pushes() { + if let Some((dx, dy)) = slide { + // The outgoing clip leaves the way the incoming one + // arrives: at rest, then a whole frame the other way. + fx[base + pj].move_out = Some((-dx, -dy, overlap)); + } + } + overlap + } + // No adjacent predecessor: dissolve up from black, or travel in + // over it. + None => d.min(clip.duration()), + }; + if overlap <= 0.0 { + continue; + } + match slide { + Some((dx, dy)) => fx[base + j].move_in = Some((dx, dy, overlap)), + None => fx[base + j].xfade_in = overlap, } + fx[base + j].afade_in = overlap; } } } @@ -3595,6 +3656,41 @@ fn fnum(v: f64) -> String { } } +/// The overlay offset a motion transition puts on a clip, as `(x, y)` +/// expressions in frame widths and heights over **timeline** time — or `None` +/// when the clip does not move, which is what keeps every non-motion graph +/// byte-identical. +/// +/// Both halves are ordinary keyframes, so this is [`keyframe_expr`] twice over +/// rather than a second expression language: an incoming clip holds its starting +/// offset before the transition and travels to zero, an outgoing clip sits at +/// zero until its own end and then travels away over its tail. +fn motion_expr(clip: &Clip, fx: &ClipFx) -> Option<(String, String)> { + let (mut xs, mut ys): (Vec<(f64, f64)>, Vec<(f64, f64)>) = (Vec::new(), Vec::new()); + if let Some((dx, dy, secs)) = fx.move_in { + xs.push((0.0, dx)); + xs.push((secs, 0.0)); + ys.push((0.0, dy)); + ys.push((secs, 0.0)); + } + if let Some((dx, dy, secs)) = fx.move_out { + let t0 = clip.duration(); + if xs.is_empty() { + xs.push((0.0, 0.0)); + ys.push((0.0, 0.0)); + } + xs.push((t0, 0.0)); + xs.push((t0 + secs, dx)); + ys.push((t0, 0.0)); + ys.push((t0 + secs, dy)); + } + if xs.is_empty() { + return None; + } + let start = clip.timeline_start; + Some((keyframe_expr(&xs, "t", start), keyframe_expr(&ys, "t", start))) +} + /// Build a piecewise-linear ffmpeg expression over **clip-local time** for a /// channel of keyframes. `points` are `(seconds_from_clip_start, value)` and are /// sorted here. `tvar` is the time variable the target filter exposes (`t` for @@ -4121,6 +4217,18 @@ fn video_clip_chain(clip: &Clip, fmt: &ExportFormat, fx: &ClipFx, is_image: bool if fo > 0.0 { p.push(format!("fade=t=out:st={}:d={}", (dur - fo).max(0.0), fo.clamp(0.0, dur))); } + // A dip through white is the same fade with a colour: it lands on the frame + // itself rather than on the alpha plane, so it needs no `format=yuva420p`. + if fx.white_in > 0.0 { + p.push(format!("fade=t=in:st=0:d={}:c=white", fx.white_in.clamp(0.0, dur))); + } + if fx.white_out > 0.0 { + p.push(format!( + "fade=t=out:st={}:d={}:c=white", + (dur - fx.white_out).max(0.0), + fx.white_out.clamp(0.0, dur) + )); + } if fx.xfade_in > 0.0 { // The alpha plane is already established above (xfade implies needs_alpha). p.push(format!("fade=t=in:st=0:d={}:alpha=1", fx.xfade_in.clamp(0.0, dur))); @@ -4561,8 +4669,8 @@ fn audio_clip_chain(clip: &Clip, fmt: &ExportFormat, fx: &ClipFx, layout: &str) let (trim_start, trim_end) = clip_source_window(clip, fx); let seek = clip_seek(trim_start); let delay_ms = (clip.timeline_start * 1000.0).round().max(0.0) as i64; - let fi = clip.fade_in + fx.black_in + fx.xfade_in; - let fo = clip.fade_out + fx.black_out + fx.tail; + let fi = clip.fade_in + fx.black_in + fx.white_in + fx.afade_in; + let fo = clip.fade_out + fx.black_out + fx.white_out + fx.tail; let mut p: Vec = Vec::new(); p.push(format!("atrim=start={}:end={}", trim_start - seek, trim_end - seek)); @@ -4613,7 +4721,7 @@ fn atempo_chain(speed: f64) -> String { #[cfg(test)] mod tests { use super::*; - use crate::model::{Asset, Clip, Delivery, StreamInfo, StreamKind, Timeline, Track}; + use crate::model::{Asset, Clip, Delivery, StreamInfo, StreamKind, Timeline, Track, TransitionKind}; use chrono::Utc; use uuid::Uuid; @@ -6604,6 +6712,98 @@ mod tests { assert!(!r.streams[0].image); } + #[test] + fn dip_to_white_fades_both_sides_through_white() { + let asset = av_asset(Uuid::new_v4(), 20.0); + let a = make_clip(asset.id, 0.0, 10.0, 0.0); + let mut b = make_clip(asset.id, 0.0, 10.0, 10.0); + b.transition_in = Some(crate::model::Transition { + kind: TransitionKind::DipToWhite, + duration: 1.0, + }); + let g = graph_of(&single(vec![a, b]), &[asset]); + assert!(g.contains("fade=t=out:st=9.5:d=0.5:c=white"), "{g}"); + assert!(g.contains("fade=t=in:st=0:d=0.5:c=white"), "{g}"); + // A dip never borrows a handle: neither clip is extended. + assert!(g.contains("trim=start=0:end=10"), "{g}"); + assert!(!g.contains(":alpha=1"), "a dip is not a dissolve: {g}"); + } + + #[test] + fn slide_travels_the_incoming_clip_in_over_the_held_outgoing_one() { + let asset = av_asset(Uuid::new_v4(), 20.0); + let a = make_clip(asset.id, 0.0, 10.0, 0.0); + let mut b = make_clip(asset.id, 0.0, 10.0, 10.0); + b.transition_in = Some(crate::model::Transition { + kind: TransitionKind::SlideLeft, + duration: 1.0, + }); + let g = graph_of(&single(vec![a, b]), &[asset]); + // The outgoing clip keeps playing under the incoming one, as for a dissolve. + assert!(g.contains("trim=start=0:end=11"), "outgoing holds under the slide: {g}"); + // The incoming clip starts a whole frame to the right and travels to 0 over + // the transition; local time is measured from its timeline start. + assert!(g.contains("(t-10)"), "motion is expressed in clip-local time: {g}"); + assert!(g.contains("1+(-1)*((t-10)-0)/(1)"), "one frame of travel over 1s: {g}"); + // A slide is not a dissolve — the picture stays hard-edged. + assert!(!g.contains(":alpha=1"), "{g}"); + // …but the sound still crossfades. + assert!(g.contains("afade=t=in:st=0:d=1"), "{g}"); + } + + #[test] + fn push_carries_the_outgoing_clip_out_of_frame_too() { + let asset = av_asset(Uuid::new_v4(), 20.0); + let a = make_clip(asset.id, 0.0, 10.0, 0.0); + let mut b = make_clip(asset.id, 0.0, 10.0, 10.0); + b.transition_in = Some(crate::model::Transition { + kind: TransitionKind::PushUp, + duration: 1.0, + }); + let g = graph_of(&single(vec![a, b]), &[asset]); + // The outgoing clip sits still until its own end, then leaves upwards over + // its tail: y travels 0 → -1 between local 10s and 11s. + assert!(g.contains("0+(-1)*((t-0)-10)/(1)"), "outgoing is pushed out: {g}"); + // The incoming one arrives from below over the same second. + assert!(g.contains("1+(-1)*((t-10)-0)/(1)"), "incoming arrives: {g}"); + } + + #[test] + fn a_slide_composes_with_the_clip_position_it_already_has() { + let asset = av_asset(Uuid::new_v4(), 20.0); + let a = make_clip(asset.id, 0.0, 10.0, 0.0); + let mut b = make_clip(asset.id, 0.0, 10.0, 10.0); + b.transform = crate::model::Transform { + pos_x: 0.25, + ..Default::default() + }; + b.transition_in = Some(crate::model::Transition { + kind: TransitionKind::SlideLeft, + duration: 1.0, + }); + let g = graph_of(&single(vec![a, b]), &[asset]); + // The travel is added to the offset the clip already has, not substituted + // for it — a picture-in-picture slides in to where it lives. + assert!(g.contains("x='(W-w)/2+((0.25)+(if(lt((t-10)"), "{g}"); + } + + #[test] + fn slide_without_source_handle_is_a_hard_cut() { + let asset = av_asset(Uuid::new_v4(), 20.0); + let a = make_clip(asset.id, 0.0, 20.0, 0.0); // no handle left to borrow + let mut b = make_clip(asset.id, 0.0, 10.0, 20.0); + b.transition_in = Some(crate::model::Transition { + kind: TransitionKind::PushLeft, + duration: 1.0, + }); + let g = graph_of(&single(vec![a, b]), &[asset]); + assert!(g.contains("trim=start=0:end=20"), "outgoing tail must not be extended: {g}"); + assert!( + !g.contains("overlay=x="), + "nothing moves when there is nothing to move over: {g}" + ); + } + #[test] fn dip_to_black_fades_both_sides_of_the_cut() { let asset = av_asset(Uuid::new_v4(), 20.0); @@ -7737,6 +7937,96 @@ mod tests { assert!(ok, "an animated clip and overlay must render: {err}"); } + /// A slide and a push look identical in the graph builder's assertions — + /// both put the incoming clip a frame away and walk it home — and differ + /// only in whether the *outgoing* clip moves. Nothing above can tell them + /// apart, so this renders both and looks at the pixels. + /// + /// The outgoing source carries a white stripe down its left edge. Halfway + /// through the transition a slide has left it where it was (the stripe is + /// still on screen); a push has carried it half a frame left (the stripe has + /// gone with it). + /// + /// `cargo test -p kerf-core --no-default-features -- --ignored slide_and_push` + #[test] + #[ignore = "needs the ffmpeg binary"] + fn slide_and_push_really_move_the_picture() { + let dir = std::env::temp_dir().join(format!("kerf-motion-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + + // Outgoing: black with a white stripe down the left edge. Incoming: gray. + let striped = dir.join("striped.mp4"); + let ok = command(&ffmpeg_bin()) + .args(["-hide_banner", "-loglevel", "error", "-y"]) + .args(["-f", "lavfi", "-i", "color=c=black:s=640x360:r=30:d=4"]) + .args(["-vf", "drawbox=x=0:y=0:w=64:h=360:color=white:t=fill"]) + .args(["-c:v", "libx264", "-pix_fmt", "yuv420p"]) + .arg(&striped) + .status() + .expect("run ffmpeg"); + assert!(ok.success()); + let plain = dir.join("gray.mp4"); + let ok = command(&ffmpeg_bin()) + .args(["-hide_banner", "-loglevel", "error", "-y"]) + .args(["-f", "lavfi", "-i", "color=c=gray:s=640x360:r=30:d=4"]) + .args(["-c:v", "libx264", "-pix_fmt", "yuv420p"]) + .arg(&plain) + .status() + .expect("run ffmpeg"); + assert!(ok.success()); + + let mut out_asset = av_asset(Uuid::new_v4(), 4.0); + out_asset.path = striped.to_string_lossy().into_owned(); + out_asset.streams = vec![video_stream(640, 360, 30.0)]; + let mut in_asset = av_asset(Uuid::new_v4(), 4.0); + in_asset.path = plain.to_string_lossy().into_owned(); + in_asset.streams = vec![video_stream(640, 360, 30.0)]; + + // Mean luma of the leftmost 32 px, one frame into the middle of the + // transition (the cut is at 2.0s, the transition runs a second). + let stripe_luma = |file: &Path| -> f64 { + let raw = dir.join("stripe.raw"); + let ok = command(&ffmpeg_bin()) + .args(["-hide_banner", "-loglevel", "error", "-y"]) + .args(["-ss", "2.5"]) + .arg("-i") + .arg(file) + .args(["-vf", "crop=32:360:0:0", "-frames:v", "1"]) + .args(["-f", "rawvideo", "-pix_fmt", "gray"]) + .arg(&raw) + .status() + .expect("run ffmpeg"); + assert!(ok.success()); + let bytes = std::fs::read(&raw).expect("raw"); + bytes.iter().map(|b| *b as f64).sum::() / bytes.len() as f64 + }; + + let render = |kind: TransitionKind, name: &str| -> PathBuf { + let a = make_clip(out_asset.id, 0.0, 2.0, 0.0); + let mut b = make_clip(in_asset.id, 0.0, 2.0, 2.0); + b.transition_in = Some(crate::model::Transition { kind, duration: 1.0 }); + let timeline = single(vec![a, b]); + let out = dir.join(name); + let opts = ExportOptions { + container: Container::Mp4, + video_codec: Some("libx264".into()), + include_audio: false, + ..Default::default() + }; + render_with(&timeline, &[out_asset.clone(), in_asset.clone()], &out, &opts).expect("export"); + out + }; + let slid = stripe_luma(&render(TransitionKind::SlideLeft, "slide.mp4")); + let pushed = stripe_luma(&render(TransitionKind::PushLeft, "push.mp4")); + let _ = std::fs::remove_dir_all(&dir); + + assert!(slid > 180.0, "a slide leaves the outgoing clip where it was, got luma {slid}"); + assert!( + pushed < 60.0, + "a push carries the outgoing clip out of frame, got luma {pushed}" + ); + } + /// The delivery frame is the point of the whole feature: with it set and /// **no** export resolution at all, a 16:9 source must still render a filled /// 1080x1920 file — the same thing the preview showed while cutting. diff --git a/crates/kerf-core/src/model.rs b/crates/kerf-core/src/model.rs index 1782d80..5d8c3e5 100644 --- a/crates/kerf-core/src/model.rs +++ b/crates/kerf-core/src/model.rs @@ -374,6 +374,18 @@ impl Color { } /// How a clip blends with the preceding clip on its track. +/// +/// Three families, and the family is what decides how the cut is rendered: +/// a **dip** takes both sides through a solid colour, a **dissolve** mixes them, +/// and a **motion** transition slides the incoming clip in over the outgoing one +/// (`Slide*`) or shoves the outgoing one out of frame with it (`Push*`). All of +/// them borrow the outgoing clip's unused source handle to keep it playing under +/// the transition, so a cut with no handle left degrades to a hard cut rather +/// than to a fade from black. +/// +/// The direction in a motion transition names the direction of **travel**, the +/// way an editor says it: `SlideLeft` brings the new shot in from the right edge +/// and moves it left. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum TransitionKind { @@ -381,6 +393,26 @@ pub enum TransitionKind { Crossfade, /// Dip to black: the outgoing clip fades to black, the incoming up from it. DipToBlack, + /// Dip to white — the same shape as [`Self::DipToBlack`], through white. + /// Reads as a brighter, faster beat than black, which is why a montage of + /// daylight footage usually wants it instead. + DipToWhite, + /// The incoming clip travels in from the right edge over the held outgoing one. + SlideLeft, + /// The incoming clip travels in from the left edge. + SlideRight, + /// The incoming clip travels up from the bottom edge. + SlideUp, + /// The incoming clip travels down from the top edge. + SlideDown, + /// Both clips travel left: the incoming pushes the outgoing out of frame. + PushLeft, + /// Both clips travel right. + PushRight, + /// Both clips travel up. + PushUp, + /// Both clips travel down. + PushDown, } impl TransitionKind { @@ -388,6 +420,15 @@ impl TransitionKind { match self { TransitionKind::Crossfade => "crossfade", TransitionKind::DipToBlack => "dip_to_black", + TransitionKind::DipToWhite => "dip_to_white", + TransitionKind::SlideLeft => "slide_left", + TransitionKind::SlideRight => "slide_right", + TransitionKind::SlideUp => "slide_up", + TransitionKind::SlideDown => "slide_down", + TransitionKind::PushLeft => "push_left", + TransitionKind::PushRight => "push_right", + TransitionKind::PushUp => "push_up", + TransitionKind::PushDown => "push_down", } } @@ -395,9 +436,89 @@ impl TransitionKind { match s { "crossfade" => Some(TransitionKind::Crossfade), "dip_to_black" | "diptoblack" => Some(TransitionKind::DipToBlack), + "dip_to_white" | "diptowhite" => Some(TransitionKind::DipToWhite), + "slide_left" => Some(TransitionKind::SlideLeft), + "slide_right" => Some(TransitionKind::SlideRight), + "slide_up" => Some(TransitionKind::SlideUp), + "slide_down" => Some(TransitionKind::SlideDown), + "push_left" => Some(TransitionKind::PushLeft), + "push_right" => Some(TransitionKind::PushRight), + "push_up" => Some(TransitionKind::PushUp), + "push_down" => Some(TransitionKind::PushDown), _ => None, } } + + /// Every kind, in the order a picker should offer them. + pub const ALL: [TransitionKind; 11] = [ + TransitionKind::Crossfade, + TransitionKind::DipToBlack, + TransitionKind::DipToWhite, + TransitionKind::SlideLeft, + TransitionKind::SlideRight, + TransitionKind::SlideUp, + TransitionKind::SlideDown, + TransitionKind::PushLeft, + TransitionKind::PushRight, + TransitionKind::PushUp, + TransitionKind::PushDown, + ]; + + /// The solid colour this transition dips through, if it is a dip. + pub fn dip_color(self) -> Option<&'static str> { + match self { + TransitionKind::DipToBlack => Some("black"), + TransitionKind::DipToWhite => Some("white"), + _ => None, + } + } + + /// Where the incoming clip starts, as an offset from its final position in + /// frame widths and heights, for a motion transition. It travels from here + /// to `(0, 0)` over the transition, so the vector points back along the + /// direction of travel: a `SlideLeft` starts one full frame to the right. + pub fn slide_from(self) -> Option<(f64, f64)> { + match self { + TransitionKind::SlideLeft | TransitionKind::PushLeft => Some((1.0, 0.0)), + TransitionKind::SlideRight | TransitionKind::PushRight => Some((-1.0, 0.0)), + TransitionKind::SlideUp | TransitionKind::PushUp => Some((0.0, 1.0)), + TransitionKind::SlideDown | TransitionKind::PushDown => Some((0.0, -1.0)), + _ => None, + } + } + + /// True when the outgoing clip is carried out of frame by the incoming one + /// instead of being covered where it stands. + pub fn pushes(self) -> bool { + matches!( + self, + TransitionKind::PushLeft | TransitionKind::PushRight | TransitionKind::PushUp | TransitionKind::PushDown + ) + } + + /// True when both sides play at once — a dissolve or any motion transition. + /// Such a transition needs the outgoing clip's source handle; a dip does not, + /// because the two halves happen either side of the cut. + pub fn overlaps(self) -> bool { + !matches!(self, TransitionKind::DipToBlack | TransitionKind::DipToWhite) + } + + /// Human name, for a diff line or a picker label. + pub fn label(self) -> &'static str { + match self { + TransitionKind::Crossfade => "Crossfade", + TransitionKind::DipToBlack => "Dip to black", + TransitionKind::DipToWhite => "Dip to white", + TransitionKind::SlideLeft => "Slide left", + TransitionKind::SlideRight => "Slide right", + TransitionKind::SlideUp => "Slide up", + TransitionKind::SlideDown => "Slide down", + TransitionKind::PushLeft => "Push left", + TransitionKind::PushRight => "Push right", + TransitionKind::PushUp => "Push up", + TransitionKind::PushDown => "Push down", + } + } } /// A transition blending the **start** of a clip with the clip that precedes it From 148cf3ce0fba73482e278cb3723f1bec59e07c0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:05:02 +0200 Subject: [PATCH 07/21] offer the transition set on both surfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `TransitionKind::parse` already accepted the new kinds, so what was missing was anyone knowing they exist: both surfaces named "crossfade or dip_to_black" in their errors and the MCP tool description, which is the whole of what an agent has to go on. The expected-kind list is now derived from the enum (`wire_names`), so an error message cannot drift from what parses. The tool description says what the directions mean and that a dissolve or motion transition borrows the outgoing clip's source handle — an agent hitting the hard-cut fallback otherwise has no way to know why nothing happened — and `preview_timeline` stops claiming it approximates a mid-transition blend, since the still path renders the plain cut. --- crates/kerf-app/src/lib.rs | 8 ++++++-- crates/kerf-app/src/mcp.rs | 18 ++++++++++++------ crates/kerf-core/src/model.rs | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/crates/kerf-app/src/lib.rs b/crates/kerf-app/src/lib.rs index d22927d..b3229de 100644 --- a/crates/kerf-app/src/lib.rs +++ b/crates/kerf-app/src/lib.rs @@ -85,8 +85,12 @@ fn parse_transition(kind: Option, duration: Option) -> CmdResult Ok(None), Some(k) => { - let kind = TransitionKind::parse(&k) - .ok_or_else(|| format!("invalid transition kind '{k}'; expected \"crossfade\" or \"dip_to_black\""))?; + let kind = TransitionKind::parse(&k).ok_or_else(|| { + format!( + "invalid transition kind '{k}'; expected one of {}", + TransitionKind::wire_names() + ) + })?; let duration = duration.ok_or("transition duration is required")?; Ok(Some(Transition { kind, duration })) } diff --git a/crates/kerf-app/src/mcp.rs b/crates/kerf-app/src/mcp.rs index 530a875..27e57b7 100644 --- a/crates/kerf-app/src/mcp.rs +++ b/crates/kerf-app/src/mcp.rs @@ -370,7 +370,9 @@ struct ColorParams { struct TransitionParams { #[schemars(description = "UUID of the clip whose start blends with the clip before it on the same track")] clip_id: String, - #[schemars(description = "Transition kind: \"crossfade\" or \"dip_to_black\". Omit to clear the transition")] + #[schemars( + description = "Transition kind. Fades: \"crossfade\", \"dip_to_black\", \"dip_to_white\". Motion, named for the direction of travel: \"slide_left\" / \"slide_right\" / \"slide_up\" / \"slide_down\" brings the new shot in over the old one, \"push_left\" / \"push_right\" / \"push_up\" / \"push_down\" carries the old one out with it. Omit to clear the transition" + )] kind: Option, #[schemars(description = "Transition duration in seconds (required when a kind is given)")] duration: Option, @@ -1092,7 +1094,7 @@ impl KerfMcp { } #[tool( - description = "Set or clear the transition blending a clip's start with the clip before it on the same track. kind is \"crossfade\" or \"dip_to_black\" with a duration in seconds; omit kind to clear." + description = "Set or clear the transition blending a clip's start with the clip before it on the same track, with a duration in seconds; omit kind to clear. A dissolve or a motion transition plays both shots at once, so it borrows the outgoing clip's unused source — a clip trimmed to the very end of its footage has none to lend and the transition falls back to a hard cut. Dips need no handle. Reach for a fade between scenes and a motion transition between shots in a montage; a cut needs no transition at all." )] fn set_transition(&self, Parameters(p): Parameters) -> Result { let clip_id = parse_id(&p.clip_id)?; @@ -1672,7 +1674,7 @@ impl KerfMcp { } #[tool( - description = "Render the assembled timeline at a timeline time into one composite image the model can see — the actual cut on screen at that moment (footage layered in track order, picture-in-picture placement, crop, color; gaps render black). Use to verify an edit you just made. Mid-transition blends (crossfade/dip-to-black) are approximated." + description = "Render the assembled timeline at a timeline time into one composite image the model can see — the actual cut on screen at that moment (footage layered in track order, picture-in-picture placement, crop, color; gaps render black). Use to verify an edit you just made. A moment inside a transition is not: dissolves, dips and slides render as the plain cut." )] async fn preview_timeline(&self, Parameters(p): Parameters) -> Result { let project = self.project.clone(); @@ -1792,8 +1794,9 @@ impl ServerHandler for KerfMcp { over the interview (later video tracks composite on top). Polish \ with set_volume / set_fade (fade-in/out, e.g. to smooth hard cuts), \ set_speed, set_transform (scale / position / rotation / opacity / \ - crop — picture-in-picture), set_color and set_transition (crossfade \ - / dip-to-black). Go further: set_video_effects (blur / sharpen / \ + crop — picture-in-picture), set_color and set_transition (a fade \ + between scenes, a slide or push between the shots of a montage). \ + Go further: set_video_effects (blur / sharpen / \ grayscale / invert / vignette / chroma_key — green-screen so a lower \ track shows through), set_audio_effects (highpass / lowpass / EQ / \ compressor / gate), and animate a clip with set_keyframes / \ @@ -1942,7 +1945,10 @@ fn parse_transition(kind: Option, duration: Option) -> Result { let kind = TransitionKind::parse(&k).ok_or_else(|| { McpError::invalid_params( - format!("invalid transition kind '{k}'; expected \"crossfade\" or \"dip_to_black\""), + format!( + "invalid transition kind '{k}'; expected one of {}", + TransitionKind::wire_names() + ), None, ) })?; diff --git a/crates/kerf-core/src/model.rs b/crates/kerf-core/src/model.rs index 5d8c3e5..6893a09 100644 --- a/crates/kerf-core/src/model.rs +++ b/crates/kerf-core/src/model.rs @@ -464,6 +464,16 @@ impl TransitionKind { TransitionKind::PushDown, ]; + /// Every kind's wire name, quoted and comma-joined — so an error message + /// listing what was expected cannot drift from the enum. + pub fn wire_names() -> String { + Self::ALL + .iter() + .map(|k| format!("\"{}\"", k.as_str())) + .collect::>() + .join(", ") + } + /// The solid colour this transition dips through, if it is a dip. pub fn dip_color(self) -> Option<&'static str> { match self { @@ -3558,6 +3568,30 @@ mod tests { assert!(!clip.covers_source(20.0, 25.0)); } + #[test] + fn every_transition_kind_round_trips_and_knows_its_family() { + for k in TransitionKind::ALL { + assert_eq!(TransitionKind::parse(k.as_str()), Some(k), "{k:?} must survive the wire"); + assert!( + TransitionKind::wire_names().contains(k.as_str()), + "{k:?} must be listed for a caller" + ); + // Exactly one family each: a dip has a colour and never moves, a + // motion transition moves and never dips, a dissolve does neither. + assert!( + !(k.dip_color().is_some() && k.slide_from().is_some()), + "{k:?} cannot both dip and travel" + ); + assert_eq!(k.dip_color().is_some(), !k.overlaps(), "{k:?}: only a dip skips the overlap"); + assert!(!k.pushes() || k.slide_from().is_some(), "{k:?}: a push must have a direction"); + } + // A slide and its push travel the same way; the difference is what + // happens to the outgoing clip, not where the incoming one comes from. + assert_eq!(TransitionKind::SlideLeft.slide_from(), TransitionKind::PushLeft.slide_from()); + assert!(!TransitionKind::SlideLeft.pushes() && TransitionKind::PushLeft.pushes()); + assert_eq!(TransitionKind::parse("nonsense"), None); + } + #[test] fn captions_follow_a_trimmed_and_moved_clip() { let asset = Uuid::new_v4(); From 5d988e62b6b90629a62f87e51e1be1cf70e8696c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:08:28 +0200 Subject: [PATCH 08/21] pick a transition from the editor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Inspector's Type dropdown listed the two kinds it was written against. It now lists all eleven, grouped — because the choice a user actually makes is between a fade, a slide and a push, and only then a direction, and a flat list of eleven names hides that. `src/lib/transitions.ts` holds the list with the labels and the grouping, the same arrangement as delivery-formats.ts; its test pins the ids against `TransitionKind::ALL`, so a kind added on one side of the boundary and not the other fails rather than silently going missing from the picker. Driven through the browser harness: all three groups render and each selection survives the round-trip back into the timeline. --- .../lib/components/editor/Inspector.svelte | 12 ++- frontend/src/lib/transitions.test.ts | 55 ++++++++++++++ frontend/src/lib/transitions.ts | 75 +++++++++++++++++++ frontend/src/lib/types.ts | 13 +++- 4 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 frontend/src/lib/transitions.test.ts create mode 100644 frontend/src/lib/transitions.ts diff --git a/frontend/src/lib/components/editor/Inspector.svelte b/frontend/src/lib/components/editor/Inspector.svelte index 85ba2fa..71e9ef8 100644 --- a/frontend/src/lib/components/editor/Inspector.svelte +++ b/frontend/src/lib/components/editor/Inspector.svelte @@ -9,6 +9,7 @@ import { clipDuration, DEFAULT_COLOR, DEFAULT_REFRAME, DEFAULT_TRANSFORM } from '$lib/types'; import { CAPTION_LOOKS, COLOR_LOOKS, TEXT_STYLES, activeLook } from '$lib/style-presets'; import { needsCrop } from '$lib/smart-crop'; + import { DEFAULT_TRANSITION_SECONDS, TRANSITION_GROUPS } from '$lib/transitions'; import type { TextStyle } from '$lib/style-presets'; import type { AudioEffect, @@ -889,13 +890,18 @@ onchange={(e) => { const k = e.currentTarget.value as '' | TransitionKind; if (!k) void run(() => editor.setTransition(clip.id, null)); - else void run(() => editor.setTransition(clip.id, { kind: k, duration: transition?.duration ?? 0.5 })); + else void run(() => editor.setTransition(clip.id, { kind: k, duration: transition?.duration ?? DEFAULT_TRANSITION_SECONDS })); }} style={selectCss} > - - + {#each TRANSITION_GROUPS as g (g.label)} + + {#each g.options as o (o.id)} + + {/each} + + {/each} {#if transition} diff --git a/frontend/src/lib/transitions.test.ts b/frontend/src/lib/transitions.test.ts new file mode 100644 index 0000000..db3527c --- /dev/null +++ b/frontend/src/lib/transitions.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, test } from 'bun:test'; +import { + needsSourceHandle, + TRANSITION_GROUPS, + TRANSITION_OPTIONS, + transitionLabel +} from './transitions'; +import type { TransitionKind } from './types'; + +describe('the transition list', () => { + test('offers every kind the engine renders, once each', () => { + // Mirrors `TransitionKind::ALL`; a kind added to one side and not the + // other is exactly the drift this asserts against. + const expected: TransitionKind[] = [ + 'crossfade', + 'dip_to_black', + 'dip_to_white', + 'push_down', + 'push_left', + 'push_right', + 'push_up', + 'slide_down', + 'slide_left', + 'slide_right', + 'slide_up' + ]; + expect(TRANSITION_OPTIONS.map((o) => o.id).sort()).toEqual(expected.sort()); + expect(new Set(TRANSITION_OPTIONS.map((o) => o.id)).size).toBe(TRANSITION_OPTIONS.length); + }); + + test('every group is named and non-empty', () => { + for (const g of TRANSITION_GROUPS) { + expect(g.label.length).toBeGreaterThan(0); + expect(g.hint.length).toBeGreaterThan(0); + expect(g.options.length).toBeGreaterThan(0); + } + }); +}); + +describe('transitionLabel', () => { + test('names a known kind and falls back to the wire name', () => { + expect(transitionLabel('push_up')).toBe('Push up'); + expect(transitionLabel('whip_pan' as never)).toBe('whip_pan'); + }); +}); + +describe('needsSourceHandle', () => { + test('only a dip renders without the outgoing clip playing under it', () => { + expect(needsSourceHandle('dip_to_black')).toBe(false); + expect(needsSourceHandle('dip_to_white')).toBe(false); + expect(needsSourceHandle('crossfade')).toBe(true); + expect(needsSourceHandle('slide_up')).toBe(true); + expect(needsSourceHandle('push_left')).toBe(true); + }); +}); diff --git a/frontend/src/lib/transitions.ts b/frontend/src/lib/transitions.ts new file mode 100644 index 0000000..9cc0511 --- /dev/null +++ b/frontend/src/lib/transitions.ts @@ -0,0 +1,75 @@ +/* The transitions kerf-core can render, grouped the way a picker should offer + * them. + * + * Mirrors `TransitionKind` in crates/kerf-core/src/model.rs; the ids are the + * serde wire names, and the grouping is the one distinction a user has to make + * before picking a direction — whether the shot they are leaving stays put or + * is carried out of frame with the one arriving. + * + * A direction names the direction of *travel*, the way an editor says it: + * `slide_left` brings the new shot in from the right edge and moves it left. */ + +import type { TransitionKind } from './types'; + +export interface TransitionOption { + id: TransitionKind; + label: string; +} + +export interface TransitionGroup { + label: string; + hint: string; + options: TransitionOption[]; +} + +export const TRANSITION_GROUPS: TransitionGroup[] = [ + { + label: 'Fade', + hint: 'Between scenes', + options: [ + { id: 'crossfade', label: 'Crossfade' }, + { id: 'dip_to_black', label: 'Dip to black' }, + { id: 'dip_to_white', label: 'Dip to white' } + ] + }, + { + label: 'Slide', + hint: 'The new shot travels in over the old one', + options: [ + { id: 'slide_left', label: 'Slide left' }, + { id: 'slide_right', label: 'Slide right' }, + { id: 'slide_up', label: 'Slide up' }, + { id: 'slide_down', label: 'Slide down' } + ] + }, + { + label: 'Push', + hint: 'The new shot carries the old one out of frame', + options: [ + { id: 'push_left', label: 'Push left' }, + { id: 'push_right', label: 'Push right' }, + { id: 'push_up', label: 'Push up' }, + { id: 'push_down', label: 'Push down' } + ] + } +]; + +/** Every kind, flattened, in picker order. */ +export const TRANSITION_OPTIONS: TransitionOption[] = TRANSITION_GROUPS.flatMap((g) => g.options); + +/** Seconds a transition gets when one is first applied. Long enough to read as + * deliberate, short enough not to eat a short shot whole. */ +export const DEFAULT_TRANSITION_SECONDS = 0.5; + +/** The human label for a kind, falling back to the wire name so a kind added to + * the engine ahead of this list still shows as something rather than blank. */ +export function transitionLabel(kind: TransitionKind): string { + return TRANSITION_OPTIONS.find((o) => o.id === kind)?.label ?? kind; +} + +/** True when the transition needs the outgoing clip's unused source to play + * underneath — which is every one but a dip, and the reason a clip trimmed to + * the very end of its footage falls back to a hard cut. */ +export function needsSourceHandle(kind: TransitionKind): boolean { + return kind !== 'dip_to_black' && kind !== 'dip_to_white'; +} diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 2d10ebc..f2b5785 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -98,7 +98,18 @@ export interface Color { temperature: number; } -export type TransitionKind = 'crossfade' | 'dip_to_black'; +export type TransitionKind = + | 'crossfade' + | 'dip_to_black' + | 'dip_to_white' + | 'slide_left' + | 'slide_right' + | 'slide_up' + | 'slide_down' + | 'push_left' + | 'push_right' + | 'push_up' + | 'push_down'; export interface Transition { kind: TransitionKind; From 9f76c30eba16c4f579d134128b31752b4f32c359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:09:01 +0200 Subject: [PATCH 09/21] document the transition set --- CLAUDE.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 64e3d44..bf55ed8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,7 +94,12 @@ so the feature is **only** activated through these forwards — which is what ma (`highpass`/`lowpass`/`equalizer`/`acompressor`/`agate`) and **transform keyframes** — animated zoom via `scale=eval=frame`, animated position via the `overlay` x/y expr, rotation via `rotate`, opacity via `geq` (all driven by piecewise-linear - `keyframe_expr` over clip-local time). **Text overlays** (`Timeline.overlays`) are + `keyframe_expr` over clip-local time). **Any such expression must be quoted in + the filter value** — it contains commas, and an unquoted comma is where the + graph parser thinks the filter ended; an unquoted `overlay=x=` and `drawtext` + x/y made every animated clip and every animated overlay abort the render with + `No such filter`, invisibly, because the graph *string* looked right and every + unit test asserted on the string. **Text overlays** (`Timeline.overlays`) are `drawtext`'d onto the final composite (animated x/y/alpha exprs when keyframed); the still / preview path samples `Clip::transform_at` and draws overlays statically. **360 footage** is reprojected by `v360`: `StreamInfo.projection` is detected at @@ -265,7 +270,20 @@ no editing logic in the adapter. (`Color`) / `Transition` fields, a clip carries a `Vec` and `Vec` (per-clip filter chains) and a `Vec` (transform **animation** — `Clip::transform_at` interpolates it, the engine renders the - motion). Text titles / lower-thirds / captions live on the timeline itself as + motion). + **`TransitionKind` is three families, and the family decides the render**: a + **dip** (`DipToBlack` / `DipToWhite`) takes both sides through a solid colour + either side of the cut, a **dissolve** (`Crossfade`) mixes them, and a + **motion** transition travels the incoming clip in over the outgoing one + (`Slide*`) or carries the outgoing one out with it (`Push*`), four directions + each — the direction naming the direction of *travel*. The enum answers for + its own family (`dip_color` / `slide_from` / `pushes` / `overlaps`), so the + engine never matches on eleven variants, and `wire_names` derives the + expected-kind list both surfaces put in their errors. A dissolve or a motion + transition plays both shots at once, so it borrows the outgoing clip's unused + source handle: a clip trimmed to the very end of its footage has none to lend + and the transition degrades to a hard cut (a dip needs none). Text titles / + lower-thirds / captions live on the timeline itself as `Timeline.overlays: Vec` (each with its own `TextKeyframe` animation); `transcript_to_srt` serializes a transcript to SubRip. **Captions are timeline math, not a transcript dump**, and pure + @@ -600,7 +618,10 @@ editor-grade workspace under `src/lib/components/editor/` — bespoke atoms (`Bt `IconBtn`, `Badge`, `Icon`, `KerfMark`) plus `TitleBar`, `Toolbar`, `MediaBin`, `Preview`, `Timeline`, `Inspector`, `AgentPanel`, `StatusBar`, composed by `routes/+page.svelte`. The `Inspector` (right panel) edits the selected clip — -trim, volume, fades, speed, transform, color, transition, plus **video / audio +trim, volume, fades, speed, transform, color, **transition** (a grouped picker +over `src/lib/transitions.ts` — fade / slide / push, then a direction, because +that is the order the choice is actually made and a flat list of eleven names +hides it; its bun test pins the ids against `TransitionKind::ALL`), plus **video / audio effect chains** (add / tune / remove), **keyframe animation** (the Transform panel auto-keyframes at the playhead and shows the sampled pose), a **Framing** section (a `Smart crop` button that frames *this* shot for the delivery frame, plus From f5beddcd39aca6b474bcf7810a0aa44a0255c1c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:12:49 +0200 Subject: [PATCH 10/21] give every track a fader and a pan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kerf had a per-clip gain, a duck flag and a loudnorm on the master, and nothing in between — no way to say "the music sits 6 dB under the voice", which is most of what makes a cut sound finished. There was no pan anywhere at all. `Track.volume` and `Track.pan` ride every clip on the track, applied in the clip's own chain *after* its gain and effects — a channel strip, so pulling a music bed down does not change what its compressor was reacting to. Both are omitted at their neutral values, so all 271 existing graph tests stay byte-identical and a project that never touched a fader renders exactly what it always did. The pan is a **balance**, not a constant-power pan: the side you turn towards stays at unity and the other is attenuated away. Constant power would boost the near side 3 dB at the extremes, which is right for placing a mono source and wrong for leaning a finished stereo track — nudging a music bed left should not make it louder. It is dropped entirely on a mono delivery, where there is nowhere to pan to. --- crates/kerf-core/src/engine/cli.rs | 107 ++++++++++++++++++++++++++--- crates/kerf-core/src/model.rs | 67 +++++++++++++++++- crates/kerf-core/src/project.rs | 21 ++++++ 3 files changed, 184 insertions(+), 11 deletions(-) diff --git a/crates/kerf-core/src/engine/cli.rs b/crates/kerf-core/src/engine/cli.rs index e4c4ab7..e46301c 100644 --- a/crates/kerf-core/src/engine/cli.rs +++ b/crates/kerf-core/src/engine/cli.rs @@ -3254,8 +3254,9 @@ fn build_filter_complex( // deduplicated ffmpeg input index (may be shared, so the `[input:v]` source is // fanned out with `split` below). let mut video: Vec<(usize, usize, &crate::model::Clip)> = Vec::new(); - // Audio entries also carry the owning track's `duck` flag for the bus split. - let mut audio: Vec<(usize, usize, &crate::model::Clip, bool)> = Vec::new(); + // Audio entries also carry the owning track's mix — the duck flag for the bus + // split, the fader and the pan for the clip's own chain. + let mut audio: Vec<(usize, usize, &crate::model::Clip, TrackMix)> = Vec::new(); let mut base = 0; for track in &timeline.tracks { let mut order: Vec = (0..track.clips.len()).collect(); @@ -3268,7 +3269,16 @@ fn build_filter_complex( video.push((flat, input, clip)); } if has_audio(clip) { - audio.push((flat, input, clip, track.duck)); + audio.push(( + flat, + input, + clip, + TrackMix { + duck: track.duck, + volume: track.volume, + pan: track.pan_gains(), + }, + )); } } base += track.clips.len(); @@ -3433,7 +3443,7 @@ fn build_filter_complex( // ---- sound: positioned per-clip audio summed with amix ------------------ if has_audio_out { - for (flat, input, clip, _) in &audio { + for (flat, input, clip, mix) in &audio { let src = if acount[*input] > 1 { let k = anext[*input]; anext[*input] += 1; @@ -3443,7 +3453,7 @@ fn build_filter_complex( }; chains.push(format!( "[{src}]{chain}[a{flat}]", - chain = audio_clip_chain(clip, fmt, &fx[*flat], layout) + chain = audio_clip_chain(clip, fmt, &fx[*flat], layout, *mix) )); } // Optional single-pass loudness normalization on the final mix; loudnorm @@ -3454,8 +3464,8 @@ fn build_filter_complex( String::new() }; let pads = |flats: &[usize]| flats.iter().map(|f| format!("[a{f}]")).collect::(); - let ducked: Vec = audio.iter().filter(|(_, _, _, d)| *d).map(|(f, _, _, _)| *f).collect(); - let keyed: Vec = audio.iter().filter(|(_, _, _, d)| !*d).map(|(f, _, _, _)| *f).collect(); + let ducked: Vec = audio.iter().filter(|(_, _, _, m)| m.duck).map(|(f, _, _, _)| *f).collect(); + let keyed: Vec = audio.iter().filter(|(_, _, _, m)| !m.duck).map(|(f, _, _, _)| *f).collect(); if ducked.is_empty() || keyed.is_empty() { // No ducking in play (nothing flagged, or nothing to key from): one // flat sum of every clip, exactly as before. @@ -3494,6 +3504,17 @@ fn build_filter_complex( } } +/// The owning track's mix settings, carried alongside each of its audio clips. +/// The fader rides the *finished* clip — after its own gain and effect chain, +/// the way a channel strip does — so pulling a music bed down does not change +/// what its own compressor was reacting to. +#[derive(Clone, Copy)] +struct TrackMix { + duck: bool, + volume: f32, + pan: (f64, f64), +} + /// Per-clip render adjustments derived from transitions. `tail` extends an /// outgoing clip so it keeps showing under the incoming one; `xfade_in` is the /// incoming clip's alpha dissolve; `black_in`/`black_out` and `white_in`/ @@ -4661,7 +4682,7 @@ fn still_overlay(t: &Transform) -> String { /// The audio filter chain for one clip (between `[i:a]` and `[a{i}]`): trim, /// optional reverse / tempo, gain, fades (including transition cross-fades) and /// delay to the clip's timeline position. Defaults reduce to the original chain. -fn audio_clip_chain(clip: &Clip, fmt: &ExportFormat, fx: &ClipFx, layout: &str) -> String { +fn audio_clip_chain(clip: &Clip, fmt: &ExportFormat, fx: &ClipFx, layout: &str, mix: TrackMix) -> String { let s = clip.speed_mag(); let dur = clip.duration() + fx.tail; // Mirror the video crossfade tail (extends below source_in when reversed) and @@ -4693,6 +4714,18 @@ fn audio_clip_chain(clip: &Clip, fmt: &ExportFormat, fx: &ClipFx, layout: &str) if fo > 0.0 { p.push(format!("afade=t=out:st={}:d={}", (dur - fo).max(0.0), fo.clamp(0.0, dur))); } + // The track fader and its pan, after the clip's own chain. Both are omitted + // at their neutral values, so a project that never touched a fader renders + // the graph it always did. + if (mix.volume - 1.0).abs() > f32::EPSILON { + p.push(format!("volume={}", mix.volume)); + } + let (gl, gr) = mix.pan; + if layout == "stereo" && ((gl - 1.0).abs() > 1e-9 || (gr - 1.0).abs() > 1e-9) { + // A balance, not a mono re-pan: each side keeps its own channel and is + // attenuated, so a stereo music bed leans without collapsing. + p.push(format!("pan=stereo|c0={}*c0|c1={}*c1", fnum(gl), fnum(gr))); + } p.push(format!( "aformat=sample_rates={sr}:channel_layouts={layout}", sr = fmt.sample_rate @@ -4722,6 +4755,16 @@ fn atempo_chain(speed: f64) -> String { mod tests { use super::*; use crate::model::{Asset, Clip, Delivery, StreamInfo, StreamKind, Timeline, Track, TransitionKind}; + + /// A track mix that changes nothing — what every test that is not about the + /// mixer wants. + fn unity_mix() -> TrackMix { + TrackMix { + duck: false, + volume: 1.0, + pan: (1.0, 1.0), + } + } use chrono::Utc; use uuid::Uuid; @@ -5572,7 +5615,7 @@ mod tests { makeup_db: 6.0, }, ]; - let chain = audio_clip_chain(&clip, &fmt, &ClipFx::default(), "stereo"); + let chain = audio_clip_chain(&clip, &fmt, &ClipFx::default(), "stereo", unity_mix()); let vi = chain.find("volume=").expect("gain"); let hi = chain.find("highpass=f=80").expect("highpass"); let ai = chain.find("acompressor=").expect("compressor"); @@ -6478,6 +6521,50 @@ mod tests { assert!(joined.contains("trim=start=0:end=4"), "{joined}"); } + #[test] + fn the_track_fader_and_pan_ride_the_finished_clip() { + let asset = av_asset(Uuid::new_v4(), 20.0); + let mut clip = make_clip(asset.id, 0.0, 10.0, 0.0); + clip.volume = 0.8; + clip.audio = vec![crate::model::AudioEffect::Highpass { hz: 80.0 }]; + let mut timeline = single(vec![clip]); + timeline.tracks[0].volume = 0.5; + timeline.tracks[0].pan = -1.0; + let g = graph_of(&timeline, &[asset]); + // The clip's own gain, then its effects, then the fader: a channel strip, + // so the compressor upstream never sees the fader move. + let clip_gain = g.find("volume=0.8").expect("clip gain"); + let effect = g.find("highpass").expect("clip effect"); + let fader = g.find("volume=0.5").expect("track fader"); + assert!(clip_gain < effect && effect < fader, "fader must come last: {g}"); + // Hard left is the right channel gone and the left untouched. + assert!(g.contains("pan=stereo|c0=1*c0|c1=0*c1"), "{g}"); + } + + #[test] + fn an_untouched_track_mix_emits_nothing() { + let asset = av_asset(Uuid::new_v4(), 20.0); + let timeline = single(vec![make_clip(asset.id, 0.0, 10.0, 0.0)]); + let g = graph_of(&timeline, &[asset]); + // Unity and centre are the historical graph exactly — no fader, no pan. + assert!(!g.contains("pan=stereo"), "{g}"); + assert_eq!(g.matches("volume=").count(), 1, "only the clip's own gain: {g}"); + } + + #[test] + fn a_pan_is_dropped_when_there_is_nowhere_to_pan_to() { + let asset = av_asset(Uuid::new_v4(), 20.0); + let mut timeline = single(vec![make_clip(asset.id, 0.0, 10.0, 0.0)]); + timeline.tracks[0].pan = 1.0; + let opts = ExportOptions { + audio_channels: Some(1), + ..Default::default() + }; + let args = build_export_args(&timeline, &[asset], "out.mp4", &opts).unwrap(); + let g = args.join(" "); + assert!(!g.contains("pan=stereo"), "a mono delivery has no stereo field: {g}"); + } + #[test] fn ducked_track_sidechains_under_the_rest() { let asset = av_asset(Uuid::new_v4(), 20.0); @@ -7488,7 +7575,7 @@ mod tests { // whose chain is auralized cannot drift from the mix it will become. let mut clip = make_clip(Uuid::new_v4(), 0.0, 5.0, 0.0); clip.audio = effects; - let exported = audio_clip_chain(&clip, &ExportFormat::default(), &ClipFx::default(), "stereo"); + let exported = audio_clip_chain(&clip, &ExportFormat::default(), &ClipFx::default(), "stereo", unity_mix()); assert!(exported.contains(&chain), "export chain {exported} must contain {chain}"); } diff --git a/crates/kerf-core/src/model.rs b/crates/kerf-core/src/model.rs index 6893a09..95f0827 100644 --- a/crates/kerf-core/src/model.rs +++ b/crates/kerf-core/src/model.rs @@ -1612,14 +1612,39 @@ pub struct Track { /// renders; the GUI refuses to drag, trim or razor its clips. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub locked: bool, + /// The track fader: a linear gain riding every clip on the track *after* its + /// own volume and effect chain, the way a channel strip works — so pulling a + /// music bed down does not change what its compressor was reacting to. + /// 1.0 is unity. Defaulted, so a project written before there was a fader + /// reads back at unity and renders identically. + #[serde(default = "unity_gain", skip_serializing_if = "is_unity_gain")] + pub volume: f32, + /// Stereo placement, -1 (hard left) to 1 (hard right); 0 is centre. Applied + /// as a constant-power balance, so panning a track does not change how loud + /// it is — and it is a no-op on a mono delivery, where there is nowhere to + /// pan to. + #[serde(default, skip_serializing_if = "is_centred")] + pub pan: f32, pub kind: StreamKind, pub name: String, #[serde(default)] pub clips: Vec, } +fn unity_gain() -> f32 { + 1.0 +} + +fn is_unity_gain(v: &f32) -> bool { + (*v - 1.0).abs() < f32::EPSILON +} + +fn is_centred(v: &f32) -> bool { + v.abs() < f32::EPSILON +} + impl Track { - /// An empty track: not ducked, muted, soloed or locked. + /// An empty track: not ducked, muted, soloed or locked, at unity and centred. pub fn new(kind: StreamKind, name: impl Into) -> Self { Self { id: Uuid::new_v4(), @@ -1627,12 +1652,31 @@ impl Track { muted: false, solo: false, locked: false, + volume: 1.0, + pan: 0.0, kind, name: name.into(), clips: Vec::new(), } } + /// The left / right gains for this track's `pan`, as a fraction of unity. + /// + /// A **balance**, not a constant-power pan: the side you turn towards stays + /// at unity and the other is attenuated away. A constant-power law would + /// boost the near side by 3 dB at the extremes, which is right for placing a + /// mono source in a field and wrong for leaning a finished stereo track — + /// nudging a music bed left should not make it louder. Centre is exactly + /// `(1, 1)`, so an untouched track is bit-for-bit what it always was. + pub fn pan_gains(&self) -> (f64, f64) { + let p = self.pan.clamp(-1.0, 1.0) as f64; + if p < 0.0 { + (1.0, 1.0 + p) + } else { + (1.0 - p, 1.0) + } + } + /// End time of the last clip on this track (seconds). pub fn end(&self) -> f64 { self.clips.iter().map(Clip::timeline_end).fold(0.0, f64::max) @@ -3568,6 +3612,27 @@ mod tests { assert!(!clip.covers_source(20.0, 25.0)); } + #[test] + fn a_track_pan_is_a_balance_and_centre_is_exactly_unity() { + let mut t = Track::new(StreamKind::Audio, "A1"); + assert_eq!(t.pan_gains(), (1.0, 1.0), "an untouched track must not be touched"); + t.pan = -1.0; + assert_eq!(t.pan_gains(), (1.0, 0.0), "hard left keeps the left at unity"); + t.pan = 1.0; + assert_eq!(t.pan_gains(), (0.0, 1.0)); + t.pan = -0.5; + assert_eq!(t.pan_gains(), (1.0, 0.5)); + // Never a boost: leaning a finished stereo track must not make it louder. + for p in [-1.0, -0.5, 0.0, 0.25, 1.0] { + t.pan = p; + let (l, r) = t.pan_gains(); + assert!(l <= 1.0 && r <= 1.0, "pan {p} boosted to ({l}, {r})"); + } + // Out of range is clamped rather than inverted. + t.pan = 9.0; + assert_eq!(t.pan_gains(), (0.0, 1.0)); + } + #[test] fn every_transition_kind_round_trips_and_knows_its_family() { for k in TransitionKind::ALL { diff --git a/crates/kerf-core/src/project.rs b/crates/kerf-core/src/project.rs index 5d76547..a6956e5 100644 --- a/crates/kerf-core/src/project.rs +++ b/crates/kerf-core/src/project.rs @@ -1493,6 +1493,27 @@ impl Project { }) } + /// Set a track's fader, the gain riding every clip on it. Clamped to + /// `0..=4` (+12 dB), which is as far up as a fader has any business going. + pub fn set_track_volume(&self, track_id: Uuid, volume: f32) -> Result { + let volume = volume.clamp(0.0, 4.0); + self.edit_timeline("Set track level", |timeline| { + let track = timeline.track_mut(track_id).ok_or(Error::TrackNotFound(track_id))?; + track.volume = volume; + Ok(track.clone()) + }) + } + + /// Set a track's stereo placement, -1 (hard left) to 1 (hard right). + pub fn set_track_pan(&self, track_id: Uuid, pan: f32) -> Result { + let pan = pan.clamp(-1.0, 1.0); + self.edit_timeline("Set track pan", |timeline| { + let track = timeline.track_mut(track_id).ok_or(Error::TrackNotFound(track_id))?; + track.pan = pan; + Ok(track.clone()) + }) + } + /// Set (or clear) the frame this project is cut for. /// /// The delivery frame decides the shape of every rendered picture — the From ce539766d97a662873567987ea8fa34b95885b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:13:34 +0200 Subject: [PATCH 11/21] offer the track mixer on both surfaces `set_track_volume` / `set_track_pan` on Tauri and MCP. The tool descriptions carry the part an agent cannot infer from a signature: that a music bed belongs on its own track pulled to roughly 0.3 under speech, that a fader is the right tool when a level should simply sit lower where ducking is for dipping and recovering, and that a hard pan sounds broken on the phone speaker most of this footage is watched on. --- crates/kerf-app/src/lib.rs | 20 +++++++++++++++ crates/kerf-app/src/mcp.rs | 51 +++++++++++++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/crates/kerf-app/src/lib.rs b/crates/kerf-app/src/lib.rs index b3229de..acdc2e2 100644 --- a/crates/kerf-app/src/lib.rs +++ b/crates/kerf-app/src/lib.rs @@ -513,6 +513,24 @@ fn set_track_duck(state: State<'_, AppState>, track_id: String, duck: bool) -> C project.timeline().map_err(|e| e.to_string()) } +/// Set a track's fader — the gain riding every clip on the track. +#[tauri::command(async)] +fn set_track_volume(state: State<'_, AppState>, track_id: String, volume: f32) -> CmdResult { + let id = id(&track_id)?; + let project = state.project(); + project.set_track_volume(id, volume).map_err(|e| e.to_string())?; + project.timeline().map_err(|e| e.to_string()) +} + +/// Set a track's stereo placement, -1 (hard left) to 1 (hard right). +#[tauri::command(async)] +fn set_track_pan(state: State<'_, AppState>, track_id: String, pan: f32) -> CmdResult { + let id = id(&track_id)?; + let project = state.project(); + project.set_track_pan(id, pan).map_err(|e| e.to_string())?; + project.timeline().map_err(|e| e.to_string()) +} + /// Set the frame the project is cut for, or clear it back to the source shape. /// The preview, the scrubbed still and the export all read it, so the vertical /// crop is visible while cutting instead of only in the rendered file. @@ -1586,6 +1604,8 @@ pub fn run() { add_track, remove_track, set_track_duck, + set_track_volume, + set_track_pan, set_delivery_format, set_track_muted, set_track_solo, diff --git a/crates/kerf-app/src/mcp.rs b/crates/kerf-app/src/mcp.rs index 27e57b7..8781925 100644 --- a/crates/kerf-app/src/mcp.rs +++ b/crates/kerf-app/src/mcp.rs @@ -212,6 +212,22 @@ struct SetTrackDuckParams { duck: bool, } +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +struct SetTrackVolumeParams { + #[schemars(description = "UUID of the track to set the level of")] + track_id: String, + #[schemars(description = "Track fader as a linear gain: 1.0 is unity, 0.5 is -6 dB, 0 is silent. Clamped to 0..4")] + volume: f32, +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +struct SetTrackPanParams { + #[schemars(description = "UUID of the track to place")] + track_id: String, + #[schemars(description = "Stereo placement: -1 hard left, 0 centre, 1 hard right")] + pan: f32, +} + #[derive(Debug, serde::Deserialize, schemars::JsonSchema)] struct SetDeliveryFormatParams { #[schemars(description = "Delivery frame width in pixels, e.g. 1080. Omit (with height) to clear the \ @@ -903,6 +919,35 @@ impl KerfMcp { json(&track) } + #[tool( + description = "Set a track's fader — one linear gain riding every clip on the track, applied after \ + each clip's own volume and effects. This is how a music bed is balanced against a \ + voiceover: put the music on its own track and pull it to roughly 0.25-0.4 under \ + speech. Prefer it to editing every clip's volume, and to set_track_duck when the \ + level should simply sit lower rather than dip and recover." + )] + fn set_track_volume(&self, Parameters(p): Parameters) -> Result { + let track_id = parse_id(&p.track_id)?; + let project = self.lock(); + let track = project.set_track_volume(track_id, p.volume).map_err(core_err)?; + self.changed(); + json(&track) + } + + #[tool( + description = "Place a track in the stereo field, -1 hard left to 1 hard right. A balance, so a \ + panned track never gets louder, and a no-op on a mono delivery. Use it sparingly — \ + a hard-panned music bed sounds broken on a phone speaker, which is what most of \ + this footage is watched on." + )] + fn set_track_pan(&self, Parameters(p): Parameters) -> Result { + let track_id = parse_id(&p.track_id)?; + let project = self.lock(); + let track = project.set_track_pan(track_id, p.pan).map_err(core_err)?; + self.changed(); + json(&track) + } + #[tool( description = "Set the frame this project is cut for — the shape of the delivered video, e.g. \ 1080x1920 for a vertical Reel or 1080x1080 for a square feed post. Everything \ @@ -1799,7 +1844,11 @@ impl ServerHandler for KerfMcp { Go further: set_video_effects (blur / sharpen / \ grayscale / invert / vignette / chroma_key — green-screen so a lower \ track shows through), set_audio_effects (highpass / lowpass / EQ / \ - compressor / gate), and animate a clip with set_keyframes / \ + compressor / gate). Mix with set_track_volume — a music bed belongs \ + on its own track pulled well under the speech (~0.3), which is most \ + of what makes a cut sound finished — plus set_track_duck to dip it \ + further under dialogue and set_track_pan to place it. Animate a clip \ + with set_keyframes / \ add_keyframe (scale / position / rotation / opacity over time — a Ken \ Burns zoom, a moving picture-in-picture). 360 footage (Insta360 \ .insv, equirect exports) is detected on import and clips cut from it \ From afe27c22e80e4f73e33a1347afcd33ed1eca0cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:17:20 +0200 Subject: [PATCH 12/21] mix tracks from the timeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each track header grows a mixer strip: a level fader and a pan, on any track that can actually be heard — an audio track, or a video track whose clips carry sound. A silent track gets none; a mixer strip on a silent track is furniture, not a control. Double-click returns either to its neutral value, and the tooltip reads in dB and L/R, the units a level is actually judged in. Preview playback follows: the fader multiplies the clip gain exactly as the export does, and the pan is rendered as the *same balance*, not a StereoPannerNode — its constant-power law would quietly disagree with the file. `get_audio` hands back mono, so the two legs are the stereo pair. `src/lib/mixer.ts` is the faithful TS mirror of `Track::pan_gains`, with the same assertions as the Rust test on both sides of the boundary: centre is exactly unity, and no pan is ever a boost. --- frontend/src/lib/api.ts | 24 +++++++++ frontend/src/lib/audio.ts | 37 +++++++++++--- .../src/lib/components/editor/Timeline.svelte | 50 +++++++++++++++++- frontend/src/lib/mixer.test.ts | 51 +++++++++++++++++++ frontend/src/lib/mixer.ts | 32 ++++++++++++ frontend/src/lib/state.svelte.ts | 8 +++ frontend/src/lib/types.ts | 4 ++ 7 files changed, 198 insertions(+), 8 deletions(-) create mode 100644 frontend/src/lib/mixer.test.ts create mode 100644 frontend/src/lib/mixer.ts diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 0f55058..3088d50 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -754,6 +754,30 @@ export async function setTrackDuck(trackId: string, duck: boolean): Promise
diff --git a/frontend/src/lib/mixer.test.ts b/frontend/src/lib/mixer.test.ts new file mode 100644 index 0000000..6d44734 --- /dev/null +++ b/frontend/src/lib/mixer.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, test } from 'bun:test'; +import { gainLabel, isUnityMix, panGains, panLabel } from './mixer'; + +describe('panGains', () => { + test('centre is exactly unity on both sides', () => { + // The same assertion as the Rust test: an untouched track must not be + // touched, or every existing mix comes back changed. + expect(panGains(0)).toEqual([1, 1]); + expect(panGains(undefined as never)).toEqual([1, 1]); + }); + + test('is a balance, never a boost', () => { + expect(panGains(-1)).toEqual([1, 0]); + expect(panGains(1)).toEqual([0, 1]); + expect(panGains(-0.5)).toEqual([1, 0.5]); + for (const p of [-1, -0.5, 0, 0.25, 1]) { + const [l, r] = panGains(p); + expect(l).toBeLessThanOrEqual(1); + expect(r).toBeLessThanOrEqual(1); + } + }); + + test('clamps rather than inverting out of range', () => { + expect(panGains(9)).toEqual([0, 1]); + expect(panGains(-9)).toEqual([1, 0]); + }); +}); + +describe('labels', () => { + test('a fader reads in dB, silence included', () => { + expect(gainLabel(1)).toBe('0.0 dB'); // unity reads as 0, the way a mixer shows it + expect(gainLabel(0.5)).toBe('-6.0 dB'); + expect(gainLabel(0)).toBe('−∞ dB'); + expect(gainLabel(2)).toBe('+6.0 dB'); + }); + + test('a pan reads as a mixer shows it', () => { + expect(panLabel(0)).toBe('centre'); + expect(panLabel(-1)).toBe('L100'); + expect(panLabel(0.3)).toBe('R30'); + }); +}); + +describe('isUnityMix', () => { + test('an unset mix is unity', () => { + expect(isUnityMix(undefined, undefined)).toBe(true); + expect(isUnityMix(1, 0)).toBe(true); + expect(isUnityMix(0.5, 0)).toBe(false); + expect(isUnityMix(1, -0.2)).toBe(false); + }); +}); diff --git a/frontend/src/lib/mixer.ts b/frontend/src/lib/mixer.ts new file mode 100644 index 0000000..15472fc --- /dev/null +++ b/frontend/src/lib/mixer.ts @@ -0,0 +1,32 @@ +/* The track mixer's arithmetic, shared by the timeline's mixer strip and the + * Web Audio preview. + * + * `panGains` is the TS mirror of `Track::pan_gains` in + * crates/kerf-core/src/model.rs — faithful, not approximate, because preview + * playback is meant to be what the export sounds like. It is a **balance**: the + * side you turn towards stays at unity and the other is attenuated away, so + * leaning a track never makes it louder. */ + +/** Left / right gains for a pan position, -1 (hard left) to 1 (hard right). */ +export function panGains(pan: number): [number, number] { + const p = Math.min(1, Math.max(-1, pan || 0)); + return p < 0 ? [1, 1 + p] : [1 - p, 1]; +} + +/** A track fader as dB — the unit a level is actually judged in. */ +export function gainLabel(v: number): string { + if (v <= 0.0001) return '−∞ dB'; + const db = 20 * Math.log10(v); + return `${db > 0 ? '+' : ''}${db.toFixed(1)} dB`; +} + +/** A pan position as the L/R reading a mixer shows. */ +export function panLabel(p: number): string { + if (Math.abs(p) < 0.005) return 'centre'; + return `${p < 0 ? 'L' : 'R'}${Math.round(Math.abs(p) * 100)}`; +} + +/** True when a track's mix is untouched, and so contributes nothing to render. */ +export function isUnityMix(volume: number | undefined, pan: number | undefined): boolean { + return (volume ?? 1) === 1 && (pan ?? 0) === 0; +} diff --git a/frontend/src/lib/state.svelte.ts b/frontend/src/lib/state.svelte.ts index 2afe734..535f77f 100644 --- a/frontend/src/lib/state.svelte.ts +++ b/frontend/src/lib/state.svelte.ts @@ -44,6 +44,8 @@ import { smartCrop, removeTrack, setTrackDuck, + setTrackVolume, + setTrackPan, setDeliveryFormat, setTrackMuted, setTrackSolo, @@ -554,6 +556,12 @@ class EditorState { setTrackDuck(trackId: string, duck: boolean) { return this.#apply(setTrackDuck(trackId, duck)); } + setTrackVolume(trackId: string, volume: number) { + return this.#apply(setTrackVolume(trackId, volume)); + } + setTrackPan(trackId: string, pan: number) { + return this.#apply(setTrackPan(trackId, pan)); + } /** The frame this project is cut for; `null` follows the footage's shape. */ setDeliveryFormat(format: Delivery | null) { return this.#apply(setDeliveryFormat(format)); diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index f2b5785..b94520b 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -281,6 +281,10 @@ export interface Track { solo?: boolean; /** Guarded against editing. A locked track still renders. */ locked?: boolean; + /** Track fader: a linear gain over every clip on the track. 1 is unity. */ + volume?: number; + /** Stereo placement, -1 hard left to 1 hard right. 0 is centre. */ + pan?: number; } /** A named point on the timeline. Renders nothing — it is shared vocabulary From 43fba58d563b6da065dd33229c63fe57cfdef460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:17:33 +0200 Subject: [PATCH 13/21] document the track mixer --- CLAUDE.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index bf55ed8..5021cca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,7 +82,16 @@ so the feature is **only** activated through these forwards — which is what ma `format` through so range export and playback keep it. `still_clip_chain` honors `fit` too (it used to letterbox unconditionally, so the one frame you looked at while cutting was the one shape you were never going to ship). - Tracks flagged `Track.duck` are mixed into their own bus and + Every track carries a **mixer strip**: `Track.volume` (the fader) and + `Track.pan`. The fader rides each clip *after* its own gain and effect chain — + a channel strip, so pulling a music bed down does not change what its + compressor was reacting to — and the pan is a **balance** (`Track::pan_gains`, + pure + unit-tested), not a constant-power law: the side you turn towards stays + at unity and the other is attenuated away, because leaning a finished stereo + track should not make it louder. Both are omitted from the graph at their + neutral values, so every pre-existing mix is byte-identical, and the pan is + dropped entirely on a mono delivery. Tracks flagged `Track.duck` are mixed + into their own bus and `sidechaincompress`'d against the rest before the final sum (music dips under dialogue); `ExportOptions.loudnorm` appends a single-pass `loudnorm` to -14 LUFS on the final mix, and `ExportOptions.range` renders only a span by building the @@ -498,7 +507,7 @@ op (`cut_clip`, `add_clip`, `split_clip`, `trim_clip` (optional `timeline_start` left-edge trim keeps the right edge put, atomically), `reorder_clip`, `move_clip`, `ripple_delete`, `cut_clip_range` (remove a **source-time** span from a clip and ripple closed — the transcript-editing primitive), `add_track`, `remove_track`, -`set_track_duck`, `set_delivery_format` (the project's delivery frame; omit +`set_track_duck`, `set_track_volume` / `set_track_pan`, `set_delivery_format` (the project's delivery frame; omit width/height to clear it), `remove_clip`, `set_volume`, `set_fade`, `set_speed`, `set_transform`, `set_color`, `set_transition`, `set_video_effects`, `set_audio_effects`, `set_keyframes` / `add_keyframe` / `clear_keyframes`, @@ -682,7 +691,14 @@ playhead follows the audio clock — edits mid-playback re-anchor via `ui.resync()` from a `+page.svelte` effect. The timeline toolbar's `+ V` / `+ A` add tracks and each track header has a `×` to remove one (`add_track` / `remove_track`) and, on audio tracks, a **DUCK toggle** -(`set_track_duck`); the timeline is genuinely **multi-track**. The old +(`set_track_duck`); the timeline is genuinely **multi-track**. Any track that can +actually be heard — an audio track, or a video track whose clips carry sound — +also gets a **mixer strip** (level fader + pan, double-click to return either to +neutral, tooltips in dB and L/R); a silent track gets none. `src/lib/mixer.ts` is +the *faithful* mirror of `Track::pan_gains`, because preview playback renders the +pan as the same balance the export does — a `StereoPannerNode`'s constant-power +law would quietly disagree with the file, and `get_audio` hands back mono, so the +two gain legs into a merger *are* the stereo pair. The old `@xyflow/svelte` `TimelineCanvas`/`clip-node` scaffold was removed (the dep is still in `package.json`, now unused). The toolbar carries a **delivery frame picker** (Source / 16:9 / 9:16 / 1:1 / 4:5, from `src/lib/delivery-formats.ts`, bun-tested) that sets `Timeline.format` — the From 5fe06d755bab72ee1878fd8d621dd22bf729fb72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:21:51 +0200 Subject: [PATCH 14/21] cut a clip to a shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `Clip.mask`: a rectangle or ellipse, positioned and sized in fractions of the rendered frame, feathered, optionally inverted. Outside the shape the clip goes transparent and whatever is on a lower track shows through. Deliberately one primitive rather than a masking mode per use, because the timeline already stacks tracks: blurring a face is a copy of the shot on the track above, blurred and masked to the face; a rounded-off picture-in-picture is a mask on the upper clip; a region grade is a masked copy with its own colour. One thing to learn, composing with what is there, instead of a second compositor. Which is also what keeps it inside the existing per-clip chain. The mask only rewrites the alpha plane, so it is one filter in the linear chain the graph builder already emits — no branch, no second pass over the export graph. One expression covers both shapes: each axis is scaled so the edge sits at distance 1, and `max` gives a rectangle where `hypot` gives an ellipse. Fields are clamped, since dragging a handle too far would otherwise blank the clip. `geq` is per-pixel and slow, the same cost keyframed opacity already pays. The real-ffmpeg test renders black over white through a hard ellipse and checks the middle is black and the corners white — the only way to know a lower track really shows through. --- crates/kerf-core/src/engine/cli.rs | 208 ++++++++++++++++++++++++++++- crates/kerf-core/src/model.rs | 111 +++++++++++++++ crates/kerf-core/src/project.rs | 19 ++- 3 files changed, 329 insertions(+), 9 deletions(-) diff --git a/crates/kerf-core/src/engine/cli.rs b/crates/kerf-core/src/engine/cli.rs index e46301c..1278473 100644 --- a/crates/kerf-core/src/engine/cli.rs +++ b/crates/kerf-core/src/engine/cli.rs @@ -15,8 +15,8 @@ use std::sync::{Mutex, OnceLock}; use super::ProbeResult; use crate::error::{Error, Result}; use crate::model::{ - Asset, AudioEffect, Clip, Color, Projection, Reframe, ReframeKeyframe, ResolvedReframe, SalienceMap, StreamInfo, StreamKind, - TextOverlay, TimeRange, Timeline, Transform, VideoEffect, + Asset, AudioEffect, Clip, Color, Mask, MaskShape, Projection, Reframe, ReframeKeyframe, ResolvedReframe, SalienceMap, + StreamInfo, StreamKind, TextOverlay, TimeRange, Timeline, Transform, VideoEffect, }; /// A small process-global LRU of decoded single frames. Decoded frames are a @@ -3712,6 +3712,36 @@ fn motion_expr(clip: &Clip, fx: &ClipFx) -> Option<(String, String)> { Some((keyframe_expr(&xs, "t", start), keyframe_expr(&ys, "t", start))) } +/// The `geq` that cuts a clip to its [`Mask`]: the picture is passed through +/// untouched and only the alpha plane is rewritten, so what is outside the shape +/// (or inside it, `inverted`) becomes transparent and a lower track shows +/// through. +/// +/// One expression covers both shapes. Each axis is scaled so the shape's edge +/// sits at distance 1, and the shapes differ only in how the two axes combine — +/// `max` gives a rectangle, `hypot` an ellipse. Feathering is then a ramp over +/// the last `feather` of that distance, measured *inside* the edge so a softened +/// mask never grows beyond the shape that was drawn. +/// +/// `geq` is per-pixel and therefore slow, the same cost keyframed opacity +/// already pays; a mask is worth it and a full-frame one is simply not written. +fn mask_filter(mask: &Mask) -> String { + let m = mask.normalized(); + let dx = format!("(X-{cx}*W)/({rw}*W)", cx = fnum(m.x), rw = fnum(m.width / 2.0)); + let dy = format!("(Y-{cy}*H)/({rh}*H)", cy = fnum(m.y), rh = fnum(m.height / 2.0)); + let d = match m.shape { + MaskShape::Rect => format!("max(abs({dx})\\,abs({dy}))"), + MaskShape::Ellipse => format!("hypot({dx}\\,{dy})"), + }; + let inside = if m.feather <= 1e-6 { + format!("lte({d}\\,1)") + } else { + format!("clip((1-{d})/{f}\\,0\\,1)", f = fnum(m.feather)) + }; + let keep = if m.inverted { format!("(1-{inside})") } else { inside }; + format!("geq=lum='lum(X,Y)':cb='cb(X,Y)':cr='cr(X,Y)':a='({keep})*alpha(X,Y)'") +} + /// Build a piecewise-linear ffmpeg expression over **clip-local time** for a /// channel of keyframes. `points` are `(seconds_from_clip_start, value)` and are /// sorted here. `tvar` is the time variable the target filter exposes (`t` for @@ -4068,7 +4098,7 @@ fn video_clip_chain(clip: &Clip, fmt: &ExportFormat, fx: &ClipFx, is_image: bool // Alpha is needed for static opacity/rotation, animated opacity/rotation, a // chroma key, or a crossfade dissolve. let transform_alpha = (!anim && !t.is_identity() && t.needs_alpha()) || anim_rotation || anim_opacity || chroma; - let needs_alpha = transform_alpha || fx.xfade_in > 0.0; + let needs_alpha = transform_alpha || fx.xfade_in > 0.0 || clip.mask.is_some(); let dur = clip.duration() + fx.tail; // A crossfade tail borrows unused source: forward clips extend past source_out, // reversed clips extend below source_in (reverse plays high->low, so the visible @@ -4201,6 +4231,11 @@ fn video_clip_chain(clip: &Clip, fmt: &ExportFormat, fx: &ClipFx, is_image: bool p.push(f); } } + // The shape mask, once alpha exists: it only rewrites the alpha plane, so it + // composes with a chroma key above it and with the opacity below. + if let Some(mask) = &clip.mask { + p.push(mask_filter(mask)); + } // Opacity: animated via a per-frame geq alpha (geq's time var is `T`), else a // constant alpha mix. if anim_opacity { @@ -4549,7 +4584,7 @@ fn build_still_args( let rf = clip.reframe_at(local); chains.push(format!( "[{n}:v]{chain}[v{n}]", - chain = still_clip_chain(&tf, &clip.color, &clip.effects, rf.as_ref(), &canvas) + chain = still_clip_chain(&tf, &clip.color, &clip.effects, rf.as_ref(), &canvas, clip.mask.as_ref()) )); let out = format!("ov{n}"); chains.push(format!("[{cur}][v{n}]{overlay}[{out}]", overlay = still_overlay(&tf))); @@ -4600,11 +4635,12 @@ fn still_clip_chain( effects: &[VideoEffect], reframe: Option<&ResolvedReframe>, canvas: &StillCanvas, + mask: Option<&Mask>, ) -> String { let StillCanvas { w: ow, h: oh, fit, sf } = canvas; let (ow, oh, fit) = (*ow, *oh, *fit); let chroma = effects.iter().any(|e| e.produces_alpha()); - let needs_alpha = (!tf.is_identity() && tf.needs_alpha()) || chroma; + let needs_alpha = (!tf.is_identity() && tf.needs_alpha()) || chroma || mask.is_some(); let mut p: Vec = vec!["trim=end_frame=1".to_string(), "setpts=PTS-STARTPTS".to_string()]; let crop = tf.has_crop().then(|| { let cw = (1.0 - tf.crop_left - tf.crop_right).max(0.0); @@ -4658,6 +4694,9 @@ fn still_clip_chain( p.push(f); } } + if let Some(mask) = mask { + p.push(mask_filter(mask)); + } if tf.opacity < 1.0 { p.push(format!("colorchannelmixer=aa={}", tf.opacity)); } @@ -6521,6 +6560,77 @@ mod tests { assert!(joined.contains("trim=start=0:end=4"), "{joined}"); } + #[test] + fn a_mask_cuts_the_clip_to_its_shape() { + let asset = av_asset(Uuid::new_v4(), 20.0); + let mut clip = make_clip(asset.id, 0.0, 10.0, 0.0); + clip.mask = Some(crate::model::Mask { + shape: crate::model::MaskShape::Ellipse, + x: 0.25, + y: 0.5, + width: 0.4, + height: 0.6, + feather: 0.2, + inverted: false, + }); + let g = graph_of(&single(vec![clip.clone()]), &[asset.clone()]); + // A mask needs an alpha plane, and it must be established before the geq. + let alpha = g.find("format=yuva420p").expect("alpha"); + let geq = g.find("geq=lum=").expect("mask"); + assert!(alpha < geq, "alpha must precede the mask: {g}"); + // An ellipse combines the axes with hypot; the edge sits at distance 1. + assert!(g.contains("hypot((X-0.25*W)/(0.2*W)"), "{g}"); + assert!(g.contains("clip((1-hypot"), "feathered edge: {g}"); + + // A rectangle is the same expression with max instead of hypot… + let mut rect = clip.clone(); + rect.mask = Some(crate::model::Mask { + shape: crate::model::MaskShape::Rect, + feather: 0.0, + ..Default::default() + }); + let g = graph_of(&single(vec![rect.clone()]), &[asset.clone()]); + assert!(g.contains("max(abs("), "{g}"); + // …and no feather is a hard test rather than a ramp. + assert!(g.contains("lte(max(abs("), "{g}"); + assert!(!g.contains("clip((1-"), "{g}"); + + // Inverted keeps what is outside the shape. + let mut inv = rect; + inv.mask = Some(crate::model::Mask { + inverted: true, + feather: 0.0, + ..Default::default() + }); + let g = graph_of(&single(vec![inv]), &[asset]); + assert!(g.contains("a='((1-lte("), "{g}"); + } + + #[test] + fn an_unmasked_clip_writes_no_mask() { + let asset = av_asset(Uuid::new_v4(), 20.0); + let g = graph_of(&single(vec![make_clip(asset.id, 0.0, 10.0, 0.0)]), &[asset]); + assert!(!g.contains("geq="), "{g}"); + assert!(!g.contains("format=yuva420p"), "no mask, no alpha: {g}"); + } + + #[test] + fn a_degenerate_mask_is_clamped_rather_than_blanking_the_clip() { + // A zero-width shape would make the whole clip transparent, which is + // never what was meant by dragging a handle too far. + let m = crate::model::Mask { + width: 0.0, + height: -3.0, + x: 9.0, + feather: f64::NAN, + ..Default::default() + } + .normalized(); + assert!(m.width >= 0.01 && m.height >= 0.01); + assert_eq!(m.x, 1.0); + assert!((0.0..=1.0).contains(&m.feather)); + } + #[test] fn the_track_fader_and_pan_ride_the_finished_clip() { let asset = av_asset(Uuid::new_v4(), 20.0); @@ -7538,11 +7648,18 @@ mod tests { fit, sf: String::new(), }; - let contained = still_clip_chain(&Transform::default(), &Color::default(), &[], None, &canvas(Fit::Contain)); + let contained = still_clip_chain( + &Transform::default(), + &Color::default(), + &[], + None, + &canvas(Fit::Contain), + None, + ); assert!(contained.contains("force_original_aspect_ratio=decrease")); assert!(contained.contains("pad=1080:1920"), "contain letterboxes: {contained}"); - let covered = still_clip_chain(&Transform::default(), &Color::default(), &[], None, &canvas(Fit::Cover)); + let covered = still_clip_chain(&Transform::default(), &Color::default(), &[], None, &canvas(Fit::Cover), None); assert!(covered.contains("force_original_aspect_ratio=increase")); assert!(covered.contains("crop=1080:1920"), "cover crops: {covered}"); assert!(!covered.contains("pad="), "and never letterboxes: {covered}"); @@ -8024,6 +8141,83 @@ mod tests { assert!(ok, "an animated clip and overlay must render: {err}"); } + /// A mask is only worth anything if a lower track really shows through it, + /// which no assertion on the graph string can establish. Black over white, + /// masked to a hard-edged ellipse: the middle must come out black and the + /// corners white. + /// + /// `cargo test -p kerf-core --no-default-features -- --ignored a_mask_really` + #[test] + #[ignore = "needs the ffmpeg binary"] + fn a_mask_really_lets_the_track_below_show_through() { + let dir = std::env::temp_dir().join(format!("kerf-mask-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let mk = |name: &str, color: &str| -> PathBuf { + let out = dir.join(name); + let ok = command(&ffmpeg_bin()) + .args(["-hide_banner", "-loglevel", "error", "-y"]) + .args(["-f", "lavfi", "-i", &format!("color=c={color}:s=320x180:r=30:d=2")]) + .args(["-c:v", "libx264", "-pix_fmt", "yuv420p"]) + .arg(&out) + .status() + .expect("run ffmpeg"); + assert!(ok.success()); + out + }; + let (white, black) = (mk("white.mp4", "white"), mk("black.mp4", "black")); + let mut lower = av_asset(Uuid::new_v4(), 2.0); + lower.path = white.to_string_lossy().into_owned(); + lower.streams = vec![video_stream(320, 180, 30.0)]; + let mut upper = av_asset(Uuid::new_v4(), 2.0); + upper.path = black.to_string_lossy().into_owned(); + upper.streams = vec![video_stream(320, 180, 30.0)]; + + let mut top = make_clip(upper.id, 0.0, 2.0, 0.0); + top.mask = Some(crate::model::Mask { + shape: crate::model::MaskShape::Ellipse, + x: 0.5, + y: 0.5, + width: 0.5, + height: 0.5, + feather: 0.0, + inverted: false, + }); + let timeline = timeline_of(vec![ + video_track(vec![make_clip(lower.id, 0.0, 2.0, 0.0)]), + video_track(vec![top]), + ]); + + let out = dir.join("masked.mp4"); + let opts = ExportOptions { + container: Container::Mp4, + video_codec: Some("libx264".into()), + include_audio: false, + ..Default::default() + }; + render_with(&timeline, &[lower, upper], &out, &opts).expect("export"); + + // Mean luma of a 20x20 patch at (x, y) of the first frame. + let patch = |x: u32, y: u32| -> f64 { + let raw = dir.join("patch.raw"); + let ok = command(&ffmpeg_bin()) + .args(["-hide_banner", "-loglevel", "error", "-y"]) + .arg("-i") + .arg(&out) + .args(["-vf", &format!("crop=20:20:{x}:{y}"), "-frames:v", "1"]) + .args(["-f", "rawvideo", "-pix_fmt", "gray"]) + .arg(&raw) + .status() + .expect("run ffmpeg"); + assert!(ok.success()); + let bytes = std::fs::read(&raw).expect("raw"); + bytes.iter().map(|b| *b as f64).sum::() / bytes.len() as f64 + }; + let (middle, corner) = (patch(150, 80), patch(0, 0)); + let _ = std::fs::remove_dir_all(&dir); + assert!(middle < 60.0, "inside the mask the upper clip is kept, got {middle}"); + assert!(corner > 180.0, "outside it the lower track shows through, got {corner}"); + } + /// A slide and a push look identical in the graph builder's assertions — /// both put the incoming clip a frame away and walk it home — and differ /// only in whether the *outgoing* clip moves. Nothing above can tell them diff --git a/crates/kerf-core/src/model.rs b/crates/kerf-core/src/model.rs index 95f0827..aaf51c0 100644 --- a/crates/kerf-core/src/model.rs +++ b/crates/kerf-core/src/model.rs @@ -540,6 +540,112 @@ pub struct Transition { pub duration: f64, } +/// The outline of a [`Mask`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum MaskShape { + /// An axis-aligned rectangle — a sign, a screen, a lower band of the frame. + #[default] + Rect, + /// An ellipse — a face, a spotlight. + Ellipse, +} + +impl MaskShape { + pub fn as_str(self) -> &'static str { + match self { + MaskShape::Rect => "rect", + MaskShape::Ellipse => "ellipse", + } + } + + pub fn parse(s: &str) -> Option { + match s { + "rect" | "rectangle" => Some(MaskShape::Rect), + "ellipse" | "circle" | "oval" => Some(MaskShape::Ellipse), + _ => None, + } + } +} + +/// A shape cut out of a clip: inside the shape the clip is kept, outside it goes +/// transparent (or the other way round, `inverted`). Everything is a **fraction +/// of the clip's rendered frame**, so a mask does not have to be redone when the +/// delivery frame changes. +/// +/// Deliberately one primitive rather than a masking *mode* per use. A mask makes +/// a clip see-through, and the timeline already stacks tracks — so blurring a +/// face is a copy of the shot on the track above, blurred, masked to the face; +/// a picture-in-picture vignette is a mask on the upper clip; a region grade is +/// a masked copy with its own colour. One thing to learn, and it composes with +/// what is already there instead of adding a second compositor. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct Mask { + #[serde(default)] + pub shape: MaskShape, + /// Centre of the shape, as a fraction of the frame. 0.5, 0.5 is the middle. + #[serde(default = "half")] + pub x: f64, + #[serde(default = "half")] + pub y: f64, + /// Size of the shape as a fraction of the frame (its full width / height, + /// not a radius). + #[serde(default = "half")] + pub width: f64, + #[serde(default = "half")] + pub height: f64, + /// Softness of the edge, as a fraction of the shape's own half-size. 0 is a + /// hard cut — which on a face reads as a sticker, so the default is soft. + #[serde(default = "default_feather")] + pub feather: f64, + /// Keep what is *outside* the shape instead of inside it. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub inverted: bool, +} + +fn default_feather() -> f64 { + 0.15 +} + +impl Default for Mask { + fn default() -> Self { + Self { + shape: MaskShape::default(), + x: 0.5, + y: 0.5, + width: 0.5, + height: 0.5, + feather: default_feather(), + inverted: false, + } + } +} + +impl Mask { + /// The mask with every field clamped into range: sizes to a visible minimum, + /// the centre to the frame, feather to 0..1. A zero-width mask would blank + /// the clip entirely, which is never what was meant. + pub fn normalized(self) -> Self { + Self { + shape: self.shape, + x: clamp01(self.x), + y: clamp01(self.y), + width: self.width.clamp(0.01, 2.0), + height: self.height.clamp(0.01, 2.0), + feather: clamp01(self.feather), + inverted: self.inverted, + } + } +} + +fn clamp01(v: f64) -> f64 { + if v.is_finite() { + v.clamp(0.0, 1.0) + } else { + 0.5 + } +} + /// A per-clip video effect, realized as a filter inserted into the clip's video /// chain at export (after color correction). The order in `Clip::effects` is the /// order they are applied. `ChromaKey` is the one effect that establishes an @@ -1398,6 +1504,10 @@ pub struct Clip { /// explicitly un-reframed, to work in the raw projection). #[serde(default, skip_serializing_if = "Option::is_none")] pub reframe: Option, + /// A shape cut out of this clip, making the rest transparent so a lower + /// track shows through. `None` is the whole frame. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mask: Option, /// Whether this clip renders. A disabled clip keeps its place on the /// timeline (and its trims, effects and keyframes) but is dropped before the /// render graph is built — the per-clip counterpart of muting a track. @@ -1437,6 +1547,7 @@ impl Clip { audio: Vec::new(), keyframes: Vec::new(), reframe: None, + mask: None, enabled: true, } } diff --git a/crates/kerf-core/src/project.rs b/crates/kerf-core/src/project.rs index a6956e5..18bb63f 100644 --- a/crates/kerf-core/src/project.rs +++ b/crates/kerf-core/src/project.rs @@ -14,8 +14,9 @@ use crate::error::{Error, Result}; use crate::model::default_beat_tolerance; use crate::model::{ Asset, AssetAnalysis, AudioEffect, CaptionOptions, CaptionStyle, Clip, CropFrame, Delivery, EditSource, Keyframe, Marker, - Projection, Reframe, ReframeKeyframe, Revision, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, Tempo, TextKeyframe, - TextOverlay, TimeRange, Timeline, TimelineDiff, Track, TranscriptSegment, Transition, VideoEffect, MAX_FOV, MIN_FOV, + Mask, Projection, Reframe, ReframeKeyframe, Revision, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, Tempo, + TextKeyframe, TextOverlay, TimeRange, Timeline, TimelineDiff, Track, TranscriptSegment, Transition, VideoEffect, MAX_FOV, + MIN_FOV, }; /// One clip queued for smart-crop sampling: which media to look at, over which @@ -1920,6 +1921,20 @@ impl Project { } /// Set or clear (`None`) the transition that blends a clip's start with the + /// Cut a clip to a shape, or clear the mask. Outside the shape the clip goes + /// transparent, so whatever is on a lower track shows through — which is how + /// a face is blurred (a masked, blurred copy of the shot on the track above), + /// how a picture-in-picture is rounded off, and how one region gets its own + /// grade. Fields are clamped, since a zero-width shape would blank the clip. + pub fn set_mask(&self, clip_id: Uuid, mask: Option) -> Result { + let mask = mask.map(Mask::normalized); + self.edit_timeline(if mask.is_some() { "Mask clip" } else { "Clear mask" }, |timeline| { + let (ti, ci) = timeline.locate(clip_id).ok_or(Error::ClipNotFound(clip_id))?; + timeline.tracks[ti].clips[ci].mask = mask; + Ok(timeline.tracks[ti].clips[ci].clone()) + }) + } + /// clip preceding it on the same track. Realized at export. pub fn set_transition(&self, clip_id: Uuid, transition: Option) -> Result { if transition.is_some_and(|t| !t.duration.is_finite() || t.duration <= 0.0) { From 5aaa5d212b77f5aca0dfacd63ffc3566e80c432a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:22:46 +0200 Subject: [PATCH 15/21] offer masks on both surfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `set_mask` on Tauri and MCP. The tool description carries what a signature cannot: that the mask composes with the track stack rather than replacing it, and therefore *how* to blur a face — duplicate the shot onto the track above, blur the copy, mask the copy to an ellipse. It also says to look at the result with preview_timeline, since the position is a fraction of the frame and nothing but a picture tells you whether it covered the right thing. --- crates/kerf-app/src/lib.rs | 17 ++++++++-- crates/kerf-app/src/mcp.rs | 66 +++++++++++++++++++++++++++++++++++-- crates/kerf-core/src/lib.rs | 6 ++-- 3 files changed, 80 insertions(+), 9 deletions(-) diff --git a/crates/kerf-app/src/lib.rs b/crates/kerf-app/src/lib.rs index acdc2e2..8984ea2 100644 --- a/crates/kerf-app/src/lib.rs +++ b/crates/kerf-app/src/lib.rs @@ -20,9 +20,9 @@ use std::sync::{Arc, Mutex}; use base64::Engine as _; use kerf_core::{ - Asset, AssetAnalysis, AudioEffect, CaptionOptions, Delivery, EditSource, ExportOptions, Fit, Keyframe, Project, Projection, - ReframeKeyframe, Revision, StagedEdit, StreamKind, Task, TextKeyframe, Timeline, TimelineDiff, Transition, TransitionKind, - VideoEffect, + Asset, AssetAnalysis, AudioEffect, CaptionOptions, Delivery, EditSource, ExportOptions, Fit, Keyframe, Mask, Project, + Projection, ReframeKeyframe, Revision, StagedEdit, StreamKind, Task, TextKeyframe, Timeline, TimelineDiff, Transition, + TransitionKind, VideoEffect, }; use serde::Serialize; use tauri::{AppHandle, Emitter, Manager, State}; @@ -706,6 +706,16 @@ fn set_transition( project.timeline().map_err(|e| e.to_string()) } +/// Cut a clip to a shape (or clear it with `mask: null`), so a lower track shows +/// through outside it. +#[tauri::command(async)] +fn set_mask(state: State<'_, AppState>, clip_id: String, mask: Option) -> CmdResult { + let id = id(&clip_id)?; + let project = state.project(); + project.set_mask(id, mask).map_err(|e| e.to_string())?; + project.timeline().map_err(|e| e.to_string()) +} + #[tauri::command(async)] fn set_video_effects(state: State<'_, AppState>, clip_id: String, effects: Vec) -> CmdResult { let id = id(&clip_id)?; @@ -1620,6 +1630,7 @@ pub fn run() { set_transform, set_color, set_transition, + set_mask, set_video_effects, set_audio_effects, set_keyframes, diff --git a/crates/kerf-app/src/mcp.rs b/crates/kerf-app/src/mcp.rs index 8781925..0c4b73d 100644 --- a/crates/kerf-app/src/mcp.rs +++ b/crates/kerf-app/src/mcp.rs @@ -15,8 +15,8 @@ use std::sync::{Arc, Mutex, MutexGuard}; use base64::Engine as _; use kerf_core::{ - AudioEffect, CaptionOptions, CaptionStyle, Delivery, EditSource, ExportOptions, Fit, Keyframe, Project, Projection, - ReframeKeyframe, StreamKind, TextKeyframe, Transition, TransitionKind, VideoEffect, + AudioEffect, CaptionOptions, CaptionStyle, Delivery, EditSource, ExportOptions, Fit, Keyframe, Mask, MaskShape, Project, + Projection, ReframeKeyframe, StreamKind, TextKeyframe, Transition, TransitionKind, VideoEffect, }; use rmcp::handler::server::wrapper::Parameters; use rmcp::model::{CallToolResult, ContentBlock, Implementation, ServerCapabilities, ServerInfo}; @@ -212,6 +212,28 @@ struct SetTrackDuckParams { duck: bool, } +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +struct SetMaskParams { + #[schemars(description = "UUID of the clip to mask")] + clip_id: String, + #[schemars(description = "Shape outline: \"rect\" or \"ellipse\". Omit to CLEAR the mask entirely")] + shape: Option, + #[schemars(description = "Centre of the shape across the frame, 0 = left edge, 1 = right (default 0.5)")] + x: Option, + #[schemars(description = "Centre of the shape down the frame, 0 = top, 1 = bottom (default 0.5)")] + y: Option, + #[schemars(description = "Full width of the shape as a fraction of the frame, not a radius (default 0.5)")] + width: Option, + #[schemars(description = "Full height of the shape as a fraction of the frame (default 0.5)")] + height: Option, + #[schemars( + description = "Edge softness as a fraction of the shape's own half-size, 0 = hard (default 0.15). A hard-edged mask over a face reads as a sticker" + )] + feather: Option, + #[schemars(description = "Keep what is OUTSIDE the shape instead of inside it (default false)")] + inverted: Option, +} + #[derive(Debug, serde::Deserialize, schemars::JsonSchema)] struct SetTrackVolumeParams { #[schemars(description = "UUID of the track to set the level of")] @@ -919,6 +941,41 @@ impl KerfMcp { json(&track) } + #[tool(description = "Cut a clip to a shape: inside the shape the clip is kept, outside it goes \ + transparent and whatever is on a LOWER track shows through. Omit `shape` to \ + clear the mask. This is the one masking primitive, and it composes with the \ + track stack rather than replacing it — to blur a face, duplicate the shot onto \ + the track above with add_clip at the same timeline_start, give the copy a blur \ + with set_video_effects, then mask the copy to an ellipse over the face; to grade \ + one region, do the same with set_color. Check the result with preview_timeline: \ + positions are fractions of the frame, so you have to look to know you covered \ + the right thing.")] + fn set_mask(&self, Parameters(p): Parameters) -> Result { + let clip_id = parse_id(&p.clip_id)?; + let mask = match p.shape { + None => None, + Some(ref s) => { + let shape = MaskShape::parse(s).ok_or_else(|| { + McpError::invalid_params(format!("invalid mask shape '{s}'; expected \"rect\" or \"ellipse\""), None) + })?; + let d = Mask::default(); + Some(Mask { + shape, + x: p.x.unwrap_or(d.x), + y: p.y.unwrap_or(d.y), + width: p.width.unwrap_or(d.width), + height: p.height.unwrap_or(d.height), + feather: p.feather.unwrap_or(d.feather), + inverted: p.inverted.unwrap_or(false), + }) + } + }; + let project = self.lock(); + let clip = project.set_mask(clip_id, mask).map_err(core_err)?; + self.changed(); + json(&clip) + } + #[tool( description = "Set a track's fader — one linear gain riding every clip on the track, applied after \ each clip's own volume and effects. This is how a music bed is balanced against a \ @@ -1843,7 +1900,10 @@ impl ServerHandler for KerfMcp { between scenes, a slide or push between the shots of a montage). \ Go further: set_video_effects (blur / sharpen / \ grayscale / invert / vignette / chroma_key — green-screen so a lower \ - track shows through), set_audio_effects (highpass / lowpass / EQ / \ + track shows through), set_mask (cut a clip to a rectangle or ellipse so \ + a lower track shows through — with a duplicated, blurred copy above, \ + that is how a face or a number plate is blurred), \ + set_audio_effects (highpass / lowpass / EQ / \ compressor / gate). Mix with set_track_volume — a music bed belongs \ on its own track pulled well under the speech (~0.3), which is most \ of what makes a cut sound finished — plus set_track_duck to dip it \ diff --git a/crates/kerf-core/src/lib.rs b/crates/kerf-core/src/lib.rs index b55e5b8..6ba8703 100644 --- a/crates/kerf-core/src/lib.rs +++ b/crates/kerf-core/src/lib.rs @@ -31,9 +31,9 @@ pub use error::{Error, Result}; pub use fonts::list_system_fonts; pub use model::{ Asset, AssetAnalysis, AudioEffect, CaptionLayout, CaptionOptions, CaptionStyle, Clip, Color, CropFrame, Delivery, DiffEntry, - DiffKind, EditSource, Keyframe, Marker, Projection, Reframe, ReframeKeyframe, ResolvedReframe, Revision, Rhythm, SalienceMap, - StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, TextKeyframe, TextOverlay, TimeRange, Timeline, TimelineDiff, Track, - TranscriptSegment, Transform, Transition, TransitionKind, VideoEffect, + DiffKind, EditSource, Keyframe, Marker, Mask, MaskShape, Projection, Reframe, ReframeKeyframe, ResolvedReframe, Revision, + Rhythm, SalienceMap, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, TextKeyframe, TextOverlay, TimeRange, Timeline, + TimelineDiff, Track, TranscriptSegment, Transform, Transition, TransitionKind, VideoEffect, }; pub use platform::{ check_all as check_platforms, CutSummary, DeliveryCheck, DeliveryIssue, IssueKind, PlatformTarget, Severity, From 7c489ee2aad97e0f75536759ebeab71b18304e8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Fri, 28 Aug 2026 01:25:02 +0200 Subject: [PATCH 16/21] mask a clip from the editor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Mask section in the Inspector: None / Rectangle / Ellipse chips, then centre, size, feather and invert. Picking a shape starts from a sensible default rather than a collapsed one, so the first click shows a mask instead of nothing. The caption under it carries the recipe the shape alone does not suggest — that a lower track shows through, and that blurring a face is a duplicated, blurred copy above, masked. Driven through the browser harness: the chips create and clear, the sliders round-trip into the timeline, and the edit lands in history. --- frontend/src/lib/api.ts | 12 ++++ .../lib/components/editor/Inspector.svelte | 55 ++++++++++++++++++- frontend/src/lib/state.svelte.ts | 6 ++ frontend/src/lib/types.ts | 31 +++++++++++ 4 files changed, 103 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 3088d50..4ea2d87 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -12,6 +12,7 @@ import type { AudioEffect, CaptionOptions, Clip, + Mask, Color, Delivery, DeliveryCheck, @@ -754,6 +755,17 @@ export async function setTrackDuck(trackId: string, duck: boolean): Promise