From c378cab451695f2c1e8d40a5d697f5bb1051cfea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Tue, 25 Aug 2026 21:16:35 +0200 Subject: [PATCH 1/7] add cover frames and platform readiness to the engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two halves of the last mile a social-video cut has to travel after the render, both in kerf-core so the GUI and an agent get them together. `export_still` writes the composited timeline at a given time to a real file at full delivery resolution, through the same graph the export renders — so a cover is literally a frame of the video it fronts, at the shape the project is cut for, rather than a screenshot to crop back into agreement. The still arg builder grew a sink (`StillOutput`) instead of a second copy: the preview keeps its MJPEG pipe, a cover gets a JPEG or PNG file, and every existing still test is untouched. An `--ignored` test runs the binary and probes the result, because an arg builder that never produced a file would look identical from here. `platform.rs` answers whether the cut is ready to go somewhere. It keeps two limits apart that are usually conflated: what a platform *rejects*, and what it accepts and then stops distributing — a four-minute Reel uploads fine and is shown only to existing followers, which is the worse of the two because nothing tells you. Errors, warnings and tips, each phrased with the real numbers ("0:20 over", "cutting 1:00 would keep it in the feed"). Aspect is compared as a ratio, so 720x1280 reads as the right shape and merely soft. Limits verified 2026-08-25 and advisory — Kerf says what it thinks and exports what you ask for. --- crates/kerf-core/src/engine/cli.rs | 270 ++++++++++++++++-- crates/kerf-core/src/engine/mod.rs | 8 +- crates/kerf-core/src/lib.rs | 8 +- crates/kerf-core/src/platform.rs | 440 +++++++++++++++++++++++++++++ crates/kerf-core/src/project.rs | 109 +++++++ 5 files changed, 808 insertions(+), 27 deletions(-) create mode 100644 crates/kerf-core/src/platform.rs diff --git a/crates/kerf-core/src/engine/cli.rs b/crates/kerf-core/src/engine/cli.rs index 26721a1..97557c6 100644 --- a/crates/kerf-core/src/engine/cli.rs +++ b/crates/kerf-core/src/engine/cli.rs @@ -1960,6 +1960,14 @@ impl ExportFormat { /// Derive the output shape from the first clip (across all tracks) that carries /// a video stream and the first that carries audio, falling back to 1080p30 /// stereo defaults. When `opts` carries resolution or fps overrides those win. +/// The frame this timeline actually renders at: the project's delivery format +/// when one is set, otherwise the shape its footage gives it. What a readiness +/// check has to compare a platform's expectations against. +pub fn delivery_frame(timeline: &Timeline, assets: &[Asset]) -> (u32, u32) { + let f = export_format(timeline, assets, &ExportOptions::default()); + (f.width, f.height) +} + fn export_format(timeline: &Timeline, assets: &[Asset], opts: &ExportOptions) -> ExportFormat { let stream_of = |clip: &crate::model::Clip, kind: StreamKind| { assets @@ -3994,15 +4002,59 @@ pub fn timeline_frame( max_width: u32, quality: u8, ) -> Result> { + run_still(timeline, assets, opts, t, max_width, &StillOutput::JpegPipe { quality }) +} + +/// Write the composited still at timeline time `t` to `path` as a **cover +/// frame**: full delivery resolution (no preview downscale) in `format`. +/// +/// A cover is the picture a platform shows before anyone presses play, and this +/// renders it through the very graph the export uses — so the cover is literally +/// a frame of the video it fronts, at the project's delivery shape, rather than +/// a screenshot that has to be cropped back into agreement. +pub fn export_still( + timeline: &Timeline, + assets: &[Asset], + opts: &ExportOptions, + t: f64, + path: &Path, + format: ImageFormat, + quality: u8, +) -> Result { + if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) { + std::fs::create_dir_all(dir)?; + } + let out = StillOutput::File { + path: path.to_string_lossy().into_owned(), + format, + quality, + }; + // `u32::MAX` asks for no cap: `build_still_args` clamps to the delivery + // width, which is exactly the cover size. + run_still(timeline, assets, opts, t, u32::MAX, &out)?; + Ok(path.to_path_buf()) +} + +/// Run a composited still through ffmpeg, retrying in software if hardware +/// decode was asked for and failed. Returns stdout (empty for a file sink). +fn run_still( + timeline: &Timeline, + assets: &[Asset], + opts: &ExportOptions, + t: f64, + max_width: u32, + out: &StillOutput, +) -> Result> { + let piping = matches!(out, StillOutput::JpegPipe { .. }); let run = |o: &ExportOptions| -> Result> { - let args = build_timeline_frame_args(timeline, assets, o, t, max_width, quality)?; + let args = build_still_args(timeline, assets, o, t, max_width, out)?; let bin = ffmpeg_bin(); let output = command(&bin) .args(&args) .stderr(Stdio::piped()) .output() .map_err(|e| launch_err(&bin, e))?; - if !output.status.success() || output.stdout.is_empty() { + if !output.status.success() || (piping && output.stdout.is_empty()) { return Err(Error::Engine(format!( "could not render timeline frame at {t:.3}s: {}", String::from_utf8_lossy(&output.stderr).trim() @@ -4032,7 +4084,103 @@ pub fn timeline_frame( } } -/// Pure arg builder for [`timeline_frame`] (no I/O, unit-tested). +/// The JPEG-pipe still args — [`timeline_frame`]'s shape, and what every still +/// test asserts against. +#[cfg(test)] +fn build_timeline_frame_args( + timeline: &Timeline, + assets: &[Asset], + opts: &ExportOptions, + t: f64, + max_width: u32, + quality: u8, +) -> Result> { + build_still_args(timeline, assets, opts, t, max_width, &StillOutput::JpegPipe { quality }) +} + +/// Where a composited still is written, and in what image format. +/// +/// The still compositor serves two callers with the same graph: the preview / +/// agent path wants JPEG bytes on stdout, and a **cover frame** wants a real +/// file at full delivery resolution — so only the sink differs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StillOutput { + /// MJPEG on stdout (`quality` = `-q:v`) — the preview and agent path. + JpegPipe { quality: u8 }, + /// An image file. JPEG honors `quality`; PNG is lossless and ignores it. + File { path: String, format: ImageFormat, quality: u8 }, +} + +/// The image formats a cover frame can be written as. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ImageFormat { + Jpeg, + Png, +} + +impl ImageFormat { + /// The customary file extension (no dot). + pub fn ext(self) -> &'static str { + match self { + ImageFormat::Jpeg => "jpg", + ImageFormat::Png => "png", + } + } + + /// The format a path's extension asks for, defaulting to JPEG — the format + /// every platform accepts as a cover. + pub fn from_path(path: &Path) -> Self { + match path.extension().and_then(|e| e.to_str()) { + Some(e) if e.eq_ignore_ascii_case("png") => ImageFormat::Png, + _ => ImageFormat::Jpeg, + } + } + + fn encoder(self) -> &'static str { + match self { + ImageFormat::Jpeg => "mjpeg", + ImageFormat::Png => "png", + } + } +} + +impl StillOutput { + /// The trailing output arguments: one frame, encoded and sent to the sink. + fn args(&self) -> Vec { + let mut a: Vec = vec!["-frames:v".to_string(), "1".to_string()]; + match self { + StillOutput::JpegPipe { quality } => { + a.extend([ + "-q:v".to_string(), + quality.to_string(), + "-f".to_string(), + "image2pipe".to_string(), + "-vcodec".to_string(), + "mjpeg".to_string(), + "pipe:1".to_string(), + ]); + } + StillOutput::File { path, format, quality } => { + if *format == ImageFormat::Jpeg { + a.extend(["-q:v".to_string(), quality.to_string()]); + } + a.extend([ + "-f".to_string(), + "image2".to_string(), + "-vcodec".to_string(), + format.encoder().to_string(), + "-y".to_string(), + path.clone(), + ]); + } + } + a + } +} + +/// Pure arg builder for a composited still, parameterized by its sink (no I/O, +/// unit-tested). /// /// Every video clip whose timeline span contains `t` is decoded at its /// corresponding source time (`-ss` input seek), put through the same geometry / @@ -4043,13 +4191,13 @@ pub fn timeline_frame( /// keeps the export aspect ratio capped to `max_width`. Static blends /// (mid-crossfade dissolve, dip-to-black) are intentionally *not* reproduced; the /// still shows the frame each visible clip contributes at `t`. -fn build_timeline_frame_args( +fn build_still_args( timeline: &Timeline, assets: &[Asset], opts: &ExportOptions, t: f64, max_width: u32, - quality: u8, + out: &StillOutput, ) -> Result> { // Same gate as the export, so the still shows the cut that would render. let rendered = timeline.for_render(); @@ -4142,21 +4290,8 @@ fn build_timeline_frame_args( chains.push(format!("[{cur}]null[outv]")); let filter = chains.join(";"); - args.extend([ - "-filter_complex".to_string(), - filter, - "-map".to_string(), - "[outv]".to_string(), - "-frames:v".to_string(), - "1".to_string(), - "-q:v".to_string(), - quality.to_string(), - "-f".to_string(), - "image2pipe".to_string(), - "-vcodec".to_string(), - "mjpeg".to_string(), - "pipe:1".to_string(), - ]); + args.extend(["-filter_complex".to_string(), filter, "-map".to_string(), "[outv]".to_string()]); + args.extend(out.args()); Ok(args) } @@ -4514,6 +4649,101 @@ mod tests { assert!(at < args.iter().position(|a| a == "-i").unwrap()); } + #[test] + fn cover_frame_renders_to_a_file_at_full_delivery_size() { + let asset = test_asset(vec![video_stream(3840, 2160, 30.0)]); + let assets = vec![asset.clone()]; + let mut timeline = single(vec![make_clip(asset.id, 0.0, 5.0, 0.0)]); + timeline.format = Some(crate::model::Delivery { + width: 1080, + height: 1920, + fit: Fit::Cover, + }); + let out = StillOutput::File { + path: "/covers/cover.jpg".to_string(), + format: ImageFormat::Jpeg, + quality: 2, + }; + let args = build_still_args(&timeline, &assets, &ExportOptions::default(), 1.0, u32::MAX, &out).unwrap(); + // The uncapped width resolves to the project's delivery frame, not to a + // preview size — a cover is a delivered image. + let graph = args[args.iter().position(|a| a == "-filter_complex").unwrap() + 1].clone(); + assert!(graph.contains("s=1080x1920"), "canvas is the delivery frame: {graph}"); + // Written as a real file, overwriting, with the muxer stated. + assert!(args.windows(2).any(|w| w[0] == "-f" && w[1] == "image2")); + assert!(args.windows(2).any(|w| w[0] == "-vcodec" && w[1] == "mjpeg")); + assert!(args.windows(2).any(|w| w[0] == "-q:v" && w[1] == "2")); + assert!(args.contains(&"-y".to_string())); + assert!(!args.contains(&"pipe:1".to_string())); + assert_eq!(args.last().unwrap(), "/covers/cover.jpg"); + // Exactly one frame, whichever sink it goes to. + assert!(args.windows(2).any(|w| w[0] == "-frames:v" && w[1] == "1")); + } + + #[test] + fn a_png_cover_is_lossless_and_carries_no_jpeg_quality() { + let asset = test_asset(vec![video_stream(1920, 1080, 30.0)]); + let assets = vec![asset.clone()]; + let timeline = single(vec![make_clip(asset.id, 0.0, 5.0, 0.0)]); + let out = StillOutput::File { + path: "/covers/cover.png".to_string(), + format: ImageFormat::Png, + quality: 2, + }; + let args = build_still_args(&timeline, &assets, &ExportOptions::default(), 1.0, u32::MAX, &out).unwrap(); + assert!(args.windows(2).any(|w| w[0] == "-vcodec" && w[1] == "png")); + assert!(!args.contains(&"-q:v".to_string()), "-q:v is meaningless for png: {args:?}"); + } + + #[test] + #[ignore = "needs the ffmpeg binary"] + fn a_cover_frame_is_really_written_at_the_delivery_size() { + // The arg builder has unit tests, but nothing above ever ran the binary + // — and a still sink that never produced a file would look identical. + let dir = std::env::temp_dir().join(format!("kerf-cover-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let media = dir.join("src.mp4"); + let ok = command(&ffmpeg_bin()) + .args(["-hide_banner", "-loglevel", "error", "-y"]) + .args(["-f", "lavfi", "-i", "testsrc=size=1920x1080:rate=30:duration=2"]) + .args(["-c:v", "libx264", "-pix_fmt", "yuv420p"]) + .arg(&media) + .status() + .expect("run ffmpeg"); + assert!(ok.success()); + + let mut asset = av_asset(Uuid::new_v4(), 2.0); + asset.path = media.to_string_lossy().into_owned(); + asset.streams = vec![video_stream(1920, 1080, 30.0)]; + let mut timeline = single(vec![make_clip(asset.id, 0.0, 2.0, 0.0)]); + timeline.format = Some(crate::model::Delivery { + width: 1080, + height: 1350, + fit: Fit::Cover, + }); + + for (name, format) in [("cover.jpg", ImageFormat::Jpeg), ("cover.png", ImageFormat::Png)] { + let out = dir.join(name); + export_still(&timeline, &[asset.clone()], &ExportOptions::default(), 1.0, &out, format, 2).expect("cover"); + let size = std::fs::metadata(&out).expect("cover written").len(); + assert!(size > 1024, "{name} should be a real image, got {size} bytes"); + // And it is the delivery frame, not the source shape. + let probe = probe(&out).expect("probe the cover"); + let stream = probe.streams.first().expect("a video stream"); + assert_eq!((stream.width, stream.height), (Some(1080), Some(1350)), "{name}"); + } + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn cover_format_follows_the_file_extension() { + assert_eq!(ImageFormat::from_path(Path::new("/a/cover.png")), ImageFormat::Png); + assert_eq!(ImageFormat::from_path(Path::new("/a/cover.PNG")), ImageFormat::Png); + assert_eq!(ImageFormat::from_path(Path::new("/a/cover.jpg")), ImageFormat::Jpeg); + // Anything else falls back to the format every platform accepts. + assert_eq!(ImageFormat::from_path(Path::new("/a/cover")), ImageFormat::Jpeg); + } + #[test] fn insta360_lens_pairs_the_two_capture_files() { // Either lens resolves to the other, and to one shared display name. diff --git a/crates/kerf-core/src/engine/mod.rs b/crates/kerf-core/src/engine/mod.rs index b5d95c3..f9750e1 100644 --- a/crates/kerf-core/src/engine/mod.rs +++ b/crates/kerf-core/src/engine/mod.rs @@ -39,10 +39,10 @@ mod ffmpeg; // Analysis, frame and waveform extraction always go through the CLI backend — // they only need the FFmpeg binaries, never the dev libraries. pub use cli::{ - audio_effects_filter, audio_pcm, contact_sheet, decode_hwaccel, detect_scenes, detect_silence, frame_at, frame_jpeg, - generate_proxy, hw_encoders, insta360_pair, proxy_path, proxy_width, ready_proxy, stitch_insta360, stitched_path, - stream_preview, timeline_frame, validate_export, waveform, Container, ExportOptions, ExportProgress, Fit, PreviewFrame, - RateControl, RenderStatus, + audio_effects_filter, audio_pcm, contact_sheet, decode_hwaccel, delivery_frame, detect_scenes, detect_silence, export_still, frame_at, + frame_jpeg, generate_proxy, hw_encoders, insta360_pair, proxy_path, proxy_width, ready_proxy, stitch_insta360, + stitched_path, stream_preview, timeline_frame, validate_export, waveform, Container, ExportOptions, ExportProgress, Fit, + ImageFormat, PreviewFrame, RateControl, RenderStatus, }; pub(crate) use cli::insta360_pair_name; diff --git a/crates/kerf-core/src/lib.rs b/crates/kerf-core/src/lib.rs index ad47b62..dee5134 100644 --- a/crates/kerf-core/src/lib.rs +++ b/crates/kerf-core/src/lib.rs @@ -9,6 +9,7 @@ pub mod analysis; pub mod error; pub mod fonts; pub mod model; +pub mod platform; pub mod project; mod engine; @@ -21,12 +22,13 @@ pub use analysis::{ ProgressFn, RhythmAnalyzer, SceneDetector, SilenceDetector, Transcriber, TranscriptionStatus, WhisperFilterTranscriber, }; pub use engine::{ - download_speech_model, generate_proxy, hw_encoders, insta360_pair, proxy_path, proxy_width, render_with, + download_speech_model, export_still, generate_proxy, hw_encoders, insta360_pair, proxy_path, proxy_width, render_with, render_with_progress, set_speech_model, speech_model_names, stitch_insta360, stitched_path, stream_preview, validate_export, - Container, DownloadProgress, ExportOptions, ExportProgress, Fit, PreviewFrame, RateControl, RenderStatus, SpeechModelInfo, - DEFAULT_SPEECH_MODEL, + Container, DownloadProgress, ExportOptions, ExportProgress, Fit, ImageFormat, PreviewFrame, RateControl, RenderStatus, + SpeechModelInfo, DEFAULT_SPEECH_MODEL, }; pub use error::{Error, Result}; +pub use platform::{check_all as check_platforms, CutSummary, DeliveryCheck, DeliveryIssue, PlatformTarget, Severity, TARGETS as PLATFORM_TARGETS}; pub use fonts::list_system_fonts; pub use model::{ Asset, AssetAnalysis, AudioEffect, Clip, Color, Delivery, DiffEntry, DiffKind, EditSource, Keyframe, Marker, Projection, diff --git a/crates/kerf-core/src/platform.rs b/crates/kerf-core/src/platform.rs new file mode 100644 index 0000000..93b06b8 --- /dev/null +++ b/crates/kerf-core/src/platform.rs @@ -0,0 +1,440 @@ +//! Where a finished cut is going, and whether it is ready to go there. +//! +//! Everything else in Kerf ends at the rendered file. A creator's job does not: +//! the file still has to be accepted by a platform and then actually shown to +//! people. Those are two different bars, and the second one fails silently — +//! which is what this module exists to say out loud before the render, while the +//! cut can still be changed. + +use serde::{Deserialize, Serialize}; + +use crate::model::Delivery; + +/// A place a cut gets published, and what it asks of the file. +/// +/// Two kinds of limit matter and they are not the same. A **hard** limit is what +/// the platform refuses to accept. A **reach** limit is what it accepts and then +/// stops distributing: a four-minute Reel uploads fine and is shown only to +/// people who already follow you. That is the worse outcome of the two, because +/// nothing tells you it happened. +/// +/// The numbers were verified 2026-08-25. They are other companies' product +/// decisions and they move, so they are **advisory** — Kerf says what it thinks +/// and still exports whatever you ask for. +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +pub struct PlatformTarget { + pub id: &'static str, + pub label: &'static str, + /// The delivery frame this platform is authored for. + pub width: u32, + pub height: u32, + /// Aspect ratios that get full treatment, as `(w, h)`. Anything else is + /// letterboxed or pillarboxed by the platform rather than rejected. + pub accepts: &'static [(u32, u32)], + /// Longest the platform will accept at all. + pub max_secs: Option, + /// Longest that still gets distributed to non-followers. + pub reach_max_secs: Option, + /// Shortest the platform will accept. + pub min_secs: Option, + /// Where the limits come from / what else to know. + pub notes: &'static str, +} + +const VERTICAL: &[(u32, u32)] = &[(9, 16)]; +const VERTICAL_OR_SQUARE: &[(u32, u32)] = &[(9, 16), (4, 5), (1, 1)]; +const LANDSCAPE: &[(u32, u32)] = &[(16, 9)]; + +/// The publishing targets Kerf knows about, in the order a small brand tends to +/// reach for them. +pub const TARGETS: &[PlatformTarget] = &[ + PlatformTarget { + id: "reels", + label: "Instagram Reels", + width: 1080, + height: 1920, + accepts: VERTICAL, + max_secs: Some(20.0 * 60.0), + reach_max_secs: Some(3.0 * 60.0), + min_secs: Some(3.0), + notes: "Uploads accept up to 20 min, but past 3 min a Reel is only shown to existing followers.", + }, + PlatformTarget { + id: "shorts", + label: "YouTube Shorts", + width: 1080, + height: 1920, + accepts: VERTICAL_OR_SQUARE, + max_secs: Some(3.0 * 60.0), + reach_max_secs: None, + min_secs: None, + notes: "Hard 3 min cap since Oct 2024; anything longer is published as a regular video instead.", + }, + PlatformTarget { + id: "tiktok", + label: "TikTok", + width: 1080, + height: 1920, + accepts: VERTICAL, + max_secs: Some(60.0 * 60.0), + reach_max_secs: Some(10.0 * 60.0), + min_secs: Some(3.0), + notes: "Uploads accept up to 60 min. Under 3 min the file must stay below 500 MB, 3-10 min below 2 GB.", + }, + PlatformTarget { + id: "ig_feed", + label: "Instagram feed", + width: 1080, + height: 1350, + accepts: &[(4, 5), (1, 1)], + max_secs: Some(20.0 * 60.0), + reach_max_secs: Some(3.0 * 60.0), + min_secs: Some(3.0), + notes: "4:5 takes the most vertical space in the feed. Feed video is distributed as a Reel.", + }, + PlatformTarget { + id: "youtube", + label: "YouTube", + width: 1920, + height: 1080, + accepts: LANDSCAPE, + max_secs: None, + reach_max_secs: None, + min_secs: None, + notes: "No practical length limit; 16:9 fills the player without bars.", + }, +]; + +/// The target with this id. +pub fn target(id: &str) -> Option<&'static PlatformTarget> { + TARGETS.iter().find(|t| t.id == id) +} + +/// How much a finding matters. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Severity { + /// The platform will not accept this. + Error, + /// It will be accepted and then under-distributed, or shown letterboxed. + Warning, + /// Advice — nothing is wrong. + Tip, +} + +/// One thing to know before publishing this cut to a target. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DeliveryIssue { + pub severity: Severity, + /// Already phrased for a person, with the actual numbers in it. + pub message: String, +} + +/// A cut's readiness for one target. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DeliveryCheck { + pub target: String, + pub label: String, + /// True when nothing would be rejected — warnings and tips can still stand. + pub ok: bool, + pub issues: Vec, +} + +/// What a readiness check needs to know about a cut. Resolved by the caller +/// (`Project::platform_check`) so the check itself stays pure. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct CutSummary { + pub duration: f64, + /// The frame the cut renders at — the project's delivery format, or the + /// shape the footage gives it when none is set. + pub width: u32, + pub height: u32, + pub has_audio: bool, + /// Whether any text overlay is on screen — a proxy for "this reads muted". + pub has_text: bool, +} + +impl CutSummary { + /// The cut's frame as a [`Delivery`]-shaped pair, for comparing aspects. + fn aspect(&self) -> f64 { + if self.height == 0 { + return 0.0; + } + self.width as f64 / self.height as f64 + } +} + +/// `m:ss`, how a length is spoken about. +fn fmt_dur(secs: f64) -> String { + let s = secs.max(0.0).round() as i64; + format!("{}:{:02}", s / 60, s % 60) +} + +fn ratio_label(w: u32, h: u32) -> String { + let gcd = |mut a: u32, mut b: u32| { + while b != 0 { + let t = b; + b = a % b; + a = t; + } + a.max(1) + }; + let d = gcd(w, h); + format!("{}:{}", w / d, h / d) +} + +/// Check one cut against one target. +/// +/// Aspect is compared as a ratio within 1%, not as exact pixels: a 1080x1920 +/// cut and a 720x1280 one are the same shape and the platform treats them the +/// same way. +pub fn check(target: &PlatformTarget, cut: &CutSummary) -> DeliveryCheck { + let mut issues = Vec::new(); + + if cut.duration <= 0.0 { + issues.push(DeliveryIssue { + severity: Severity::Error, + message: "The timeline is empty — there is nothing to publish.".to_string(), + }); + } + if let Some(min) = target.min_secs { + if cut.duration > 0.0 && cut.duration < min { + issues.push(DeliveryIssue { + severity: Severity::Error, + message: format!( + "{} is shorter than {}'s {:.0}s minimum.", + fmt_dur(cut.duration), + target.label, + min + ), + }); + } + } + if let Some(max) = target.max_secs { + if cut.duration > max { + issues.push(DeliveryIssue { + severity: Severity::Error, + message: format!( + "{} is over {}'s {} limit — trim {} to fit.", + fmt_dur(cut.duration), + target.label, + fmt_dur(max), + fmt_dur(cut.duration - max) + ), + }); + } + } + // The quiet one: accepted, then not shown to anyone new. + if let Some(reach) = target.reach_max_secs { + let within_hard = target.max_secs.is_none_or(|m| cut.duration <= m); + if cut.duration > reach && within_hard { + issues.push(DeliveryIssue { + severity: Severity::Warning, + message: format!( + "Over {}, {} stops showing this to people who don't already follow you. Cutting {} would keep it in the feed.", + fmt_dur(reach), + target.label, + fmt_dur(cut.duration - reach) + ), + }); + } + } + + // Shape. + let want = target.width as f64 / target.height.max(1) as f64; + let have = cut.aspect(); + let matches = |(w, h): &(u32, u32)| { + let r = *w as f64 / *h as f64; + (have - r).abs() <= r * 0.01 + }; + if have > 0.0 && !target.accepts.iter().any(matches) { + let accepted = target + .accepts + .iter() + .map(|(w, h)| ratio_label(*w, *h)) + .collect::>() + .join(" or "); + issues.push(DeliveryIssue { + severity: Severity::Warning, + message: format!( + "This cut is {} ({}×{}); {} shows {}, so it will be letterboxed. Set the delivery frame to {}×{}.", + ratio_label(cut.width, cut.height), + cut.width, + cut.height, + target.label, + accepted, + target.width, + target.height + ), + }); + } else if have > 0.0 && cut.height < target.height && (have - want).abs() <= want * 0.01 { + // Right shape, not enough pixels — the platform will upscale it. + issues.push(DeliveryIssue { + severity: Severity::Warning, + message: format!( + "{}×{} is below {}'s {}×{}; the platform will upscale it and it will look soft.", + cut.width, cut.height, target.label, target.width, target.height + ), + }); + } + + if !cut.has_text && cut.has_audio { + issues.push(DeliveryIssue { + severity: Severity::Tip, + message: "The feed autoplays muted. Captions or a title would carry this for the people who never turn sound on." + .to_string(), + }); + } + + DeliveryCheck { + target: target.id.to_string(), + label: target.label.to_string(), + ok: !issues.iter().any(|i| i.severity == Severity::Error), + issues, + } +} + +/// Check a cut against every known target. +pub fn check_all(cut: &CutSummary) -> Vec { + TARGETS.iter().map(|t| check(t, cut)).collect() +} + +/// The target a delivery frame is evidently aimed at, if any — used to lead the +/// UI with the one the user already chose to cut for. +pub fn target_for(format: Option<&Delivery>) -> Option<&'static PlatformTarget> { + let f = format?; + TARGETS + .iter() + .find(|t| t.width == f.width && t.height == f.height) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn vertical_cut(duration: f64) -> CutSummary { + CutSummary { + duration, + width: 1080, + height: 1920, + has_audio: true, + has_text: true, + } + } + + #[test] + fn a_well_shaped_cut_passes_clean() { + let c = check(target("reels").unwrap(), &vertical_cut(45.0)); + assert!(c.ok); + assert!(c.issues.is_empty(), "{:?}", c.issues); + } + + #[test] + fn over_the_hard_limit_is_an_error_and_says_how_much_to_cut() { + let c = check(target("shorts").unwrap(), &vertical_cut(200.0)); + assert!(!c.ok); + let m = &c.issues[0].message; + assert_eq!(c.issues[0].severity, Severity::Error); + assert!(m.contains("3:00"), "names the limit: {m}"); + assert!(m.contains("0:20"), "names the overshoot: {m}"); + } + + #[test] + fn over_the_reach_limit_is_a_warning_not_a_rejection() { + // The whole point: this uploads fine, and then nobody new sees it. + let c = check(target("reels").unwrap(), &vertical_cut(4.0 * 60.0)); + assert!(c.ok, "a long Reel is still accepted"); + let issue = c.issues.iter().find(|i| i.severity == Severity::Warning).expect("warned"); + assert!(issue.message.contains("follow you"), "{}", issue.message); + assert!(issue.message.contains("1:00"), "says how much to cut: {}", issue.message); + } + + #[test] + fn a_reach_warning_is_not_repeated_once_it_is_already_rejected() { + // TikTok's reach limit is 10 min and its hard limit 60; past 60 the only + // useful thing to say is that it will not upload. + let c = check(target("tiktok").unwrap(), &vertical_cut(70.0 * 60.0)); + assert_eq!(c.issues.iter().filter(|i| i.severity == Severity::Warning).count(), 0); + assert_eq!(c.issues.iter().filter(|i| i.severity == Severity::Error).count(), 1); + } + + #[test] + fn a_landscape_cut_is_flagged_for_a_vertical_feed() { + let cut = CutSummary { + width: 1920, + height: 1080, + ..vertical_cut(30.0) + }; + let c = check(target("reels").unwrap(), &cut); + assert!(c.ok, "wrong shape is not a rejection"); + let m = &c.issues[0].message; + assert!(m.contains("16:9") && m.contains("9:16"), "{m}"); + assert!(m.contains("1080×1920"), "names the fix: {m}"); + } + + #[test] + fn aspect_is_compared_as_a_ratio_not_as_pixels() { + // 720x1280 is the same shape as 1080x1920; only the softness is worth a word. + let cut = CutSummary { + width: 720, + height: 1280, + ..vertical_cut(30.0) + }; + let c = check(target("reels").unwrap(), &cut); + assert_eq!(c.issues.len(), 1); + assert!(c.issues[0].message.contains("upscale"), "{:?}", c.issues[0]); + } + + #[test] + fn square_passes_shorts_but_not_reels() { + let cut = CutSummary { + width: 1080, + height: 1080, + ..vertical_cut(30.0) + }; + assert!(check(target("shorts").unwrap(), &cut) + .issues + .iter() + .all(|i| i.severity != Severity::Warning)); + assert!(check(target("reels").unwrap(), &cut) + .issues + .iter() + .any(|i| i.severity == Severity::Warning)); + } + + #[test] + fn a_silent_feed_gets_a_captions_tip_only_when_there_is_sound_to_miss() { + let mut cut = vertical_cut(30.0); + cut.has_text = false; + assert!(check(target("reels").unwrap(), &cut) + .issues + .iter() + .any(|i| i.severity == Severity::Tip)); + // No audio at all: nothing is lost by muting, so the tip would be noise. + cut.has_audio = false; + assert!(check(target("reels").unwrap(), &cut) + .issues + .iter() + .all(|i| i.severity != Severity::Tip)); + } + + #[test] + fn an_empty_timeline_is_rejected_everywhere() { + let cut = CutSummary { + duration: 0.0, + ..vertical_cut(0.0) + }; + assert!(check_all(&cut).iter().all(|c| !c.ok)); + } + + #[test] + fn a_delivery_frame_resolves_to_the_target_it_was_chosen_for() { + let d = Delivery { + width: 1080, + height: 1350, + fit: crate::model::Fit::Cover, + }; + assert_eq!(target_for(Some(&d)).map(|t| t.id), Some("ig_feed")); + assert_eq!(target_for(None), None); + } +} diff --git a/crates/kerf-core/src/project.rs b/crates/kerf-core/src/project.rs index 32031d5..581847e 100644 --- a/crates/kerf-core/src/project.rs +++ b/crates/kerf-core/src/project.rs @@ -615,6 +615,86 @@ impl Project { engine::timeline_frame(timeline, assets, &opts, time_secs, max_width, quality) } + /// How ready the current cut is for each publishing target — what a platform + /// would reject, what it would accept and then quietly under-distribute, and + /// what would simply be better. + /// + /// Reads the **working** timeline, so an agent assembling a cut sees the + /// verdict on its own proposal rather than on the user's live one. + pub fn platform_check(&self) -> Result> { + Ok(crate::platform::check_all(&self.cut_summary()?)) + } + + /// What the readiness check needs to know about the current cut. + pub fn cut_summary(&self) -> Result { + let timeline = self.working_timeline()?; + let assets = self.list_assets()?; + // The same gate the export applies, so a muted track is as absent here + // as it will be in the file. + let rendered = timeline.for_render(); + let (width, height) = engine::delivery_frame(&rendered, &assets); + // Audio-bearing means what the export means by it: any clip whose asset + // carries an audio stream, on a video track as much as an audio one. + let has_audio = rendered.tracks.iter().flat_map(|t| t.clips.iter()).any(|c| { + assets + .iter() + .find(|a| a.id == c.asset_id) + .is_some_and(|a| a.has_audio()) + }); + Ok(crate::platform::CutSummary { + duration: rendered.duration(), + width, + height, + has_audio, + has_text: !rendered.overlays.is_empty(), + }) + } + + /// The owned inputs a **cover frame** render needs: the working timeline and + /// the **original** assets. Deliberately not the proxy-swapped preview list + /// — a cover is a delivered image, so it comes off the same media the export + /// reads. Resolved together so the caller can drop the project lock before + /// [`Project::render_still`] runs ffmpeg. + pub fn export_still_inputs(&self) -> Result<(Timeline, Vec)> { + Ok((self.working_timeline()?, self.list_assets()?)) + } + + /// Write the composited frame at `time_secs` to `path` as a cover image, + /// **without** `&self` — the lock-free half of [`Project::export_still`]. + /// + /// `format` defaults to whatever the path's extension asks for. The frame is + /// rendered at the project's full delivery resolution through the export + /// graph, so the cover is a real frame of the finished video. + pub fn render_still( + timeline: &Timeline, + assets: &[Asset], + time_secs: f64, + path: impl AsRef, + format: Option, + ) -> Result { + let path = path.as_ref(); + let format = format.unwrap_or_else(|| engine::ImageFormat::from_path(path)); + let opts = engine::ExportOptions { + hwaccel: engine::decode_hwaccel(), + ..engine::ExportOptions::default() + }; + // `-q:v 2` is the highest useful JPEG quality; a cover is re-encoded by + // whatever platform receives it, so there is nothing to gain by saving + // bytes here and plenty to lose. + engine::export_still(timeline, assets, &opts, time_secs, path, format, 2) + } + + /// Render a cover frame for the current timeline at `time_secs`. + pub fn export_still( + &self, + time_secs: f64, + path: impl AsRef, + format: Option, + ) -> Result { + let (timeline, assets) = self.export_still_inputs()?; + Self::render_still(&timeline, &assets, time_secs, path, format) + } + /// [`Project::list_assets`], but with each eligible asset's `path` swapped to /// its ready proxy — the asset list the timeline-preview compositor decodes /// from. Stream metadata (resolution / fps) is kept from the original, so the @@ -2807,6 +2887,35 @@ mod tests { assert!(total_clips >= 3); } + #[test] + fn platform_check_reads_the_cut_the_export_would_render() { + let project = Project::sample().unwrap(); + // Cut for a vertical feed: the frame the checks compare against is the + // project's delivery format, not the footage's shape. + project + .set_delivery_format(Some(crate::model::Delivery { + width: 1080, + height: 1920, + fit: Fit::Cover, + })) + .unwrap(); + let summary = project.cut_summary().unwrap(); + assert_eq!((summary.width, summary.height), (1080, 1920)); + assert!(summary.duration > 0.0); + assert!(summary.has_audio, "the sample has audio-bearing clips"); + + let checks = project.platform_check().unwrap(); + assert_eq!(checks.len(), crate::platform::TARGETS.len()); + let reels = checks.iter().find(|c| c.target == "reels").unwrap(); + assert!(reels.ok, "a short vertical cut is publishable: {:?}", reels.issues); + // The same cut is the wrong shape for a landscape player, and says so. + let youtube = checks.iter().find(|c| c.target == "youtube").unwrap(); + assert!(youtube + .issues + .iter() + .any(|i| i.severity == crate::platform::Severity::Warning && i.message.contains("letterboxed"))); + } + #[test] fn importing_the_same_media_twice_reuses_the_asset() { // Both halves of an Insta360 pair stitch to one cached file and arrive From bcc1e50260ef46a8879da849766e8f4353e1f4b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Tue, 25 Aug 2026 21:18:49 +0200 Subject: [PATCH 2/7] expose cover frames and platform checks on both surfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GUI gets export_cover / platform_targets / platform_check, plus reveal_path so a finished render can be found — it opens the containing folder rather than the file, since "show me where it went" is not a request to launch a player. The heavy one follows the usual shape: resolve under the lock, decode with it released. The agent gets platform_check and export_cover, and the server instructions now tell it to check before reporting a cut finished — an agent that assembles a four-minute Reel has done the work and lost the audience, and nothing in the file would have told it. --- crates/kerf-app/src/lib.rs | 55 ++++++++++++++++++++++++++++++++++++++ crates/kerf-app/src/mcp.rs | 52 ++++++++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/crates/kerf-app/src/lib.rs b/crates/kerf-app/src/lib.rs index 1a02d9a..6cf8c97 100644 --- a/crates/kerf-app/src/lib.rs +++ b/crates/kerf-app/src/lib.rs @@ -1311,6 +1311,57 @@ async fn export_timeline( .await } +/// Write the composited frame at `time_secs` to `output_path` as a **cover +/// image** — full delivery resolution, decoded from the original media rather +/// than a preview proxy. `format` follows the file extension when omitted. +#[tauri::command] +async fn export_cover( + state: State<'_, AppState>, + time_secs: f64, + output_path: String, + format: Option, +) -> CmdResult { + let shared = state.project.clone(); + blocking(move || { + // Same shape as every heavy command: resolve under the lock, render + // without it. A 4K still is a real decode. + let (timeline, assets) = lock_user(&shared).export_still_inputs().map_err(|e| e.to_string())?; + let path = Project::render_still(&timeline, &assets, time_secs, &output_path, format).map_err(|e| e.to_string())?; + Ok(path.to_string_lossy().into_owned()) + }) + .await +} + +/// Every publishing target Kerf knows about, with its frame and length limits. +#[tauri::command(async)] +fn platform_targets() -> Vec { + kerf_core::PLATFORM_TARGETS.to_vec() +} + +/// How ready the current cut is for each target — what would be rejected, what +/// would be accepted and then under-distributed, and what would just be better. +#[tauri::command(async)] +fn platform_check(state: State<'_, AppState>) -> CmdResult> { + state.project().platform_check().map_err(|e| e.to_string()) +} + +/// Show a file in the OS file manager. The last step of an export: the render +/// finished somewhere, and "somewhere" is not much use on its own. +#[tauri::command(async)] +fn reveal_path(app: AppHandle, path: String) -> CmdResult<()> { + use tauri_plugin_opener::OpenerExt; + // Open the containing folder, not the file — opening the file would launch + // a player, which is not what "show me where it went" means. + let target = std::path::Path::new(&path) + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .map(|p| p.to_path_buf()) + .unwrap_or_else(|| std::path::PathBuf::from(&path)); + app.opener() + .open_path(target.to_string_lossy().into_owned(), None::<&str>) + .map_err(|e| e.to_string()) +} + /// Request cancellation of the in-flight export (if any). The running /// [`export_timeline`] observes the flag on its next progress poll, stops /// ffmpeg, and returns the `"export cancelled"` error. @@ -1553,6 +1604,10 @@ pub fn run() { hw_encoders, export_timeline, cancel_export, + export_cover, + platform_targets, + platform_check, + reveal_path, mcp_endpoint, log_dir, reveal_logs diff --git a/crates/kerf-app/src/mcp.rs b/crates/kerf-app/src/mcp.rs index c7fe201..284f4ec 100644 --- a/crates/kerf-app/src/mcp.rs +++ b/crates/kerf-app/src/mcp.rs @@ -648,6 +648,14 @@ struct TimelineFrameParams { max_width: Option, } +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +struct CoverParams { + #[schemars(description = "Timeline position to capture (seconds)")] + time_secs: f64, + #[schemars(description = "Output image path. A .png extension writes PNG, anything else JPEG.")] + output_path: String, +} + #[derive(Serialize)] struct AssetMetadata { asset: kerf_core::Asset, @@ -1429,6 +1437,43 @@ impl KerfMcp { json(&serde_json::json!({ "output": output_path })) } + #[tool( + description = "Check the assembled cut against each publishing target (Instagram Reels / YouTube Shorts / \ + TikTok / Instagram feed / YouTube): length limits, frame shape, and the reach limits a platform \ + enforces silently — e.g. a Reel over 3 minutes uploads fine and is then shown only to existing \ + followers. Returns errors (would be rejected), warnings (accepted but under-distributed or \ + letterboxed) and tips. Advisory: export is never blocked. Use before reporting a cut finished." + )] + fn platform_check(&self) -> Result { + let project = self.lock(); + let summary = project.cut_summary().map_err(core_err)?; + json(&serde_json::json!({ + "cut": { + "duration_secs": summary.duration, + "frame": format!("{}x{}", summary.width, summary.height), + "has_audio": summary.has_audio, + "has_text": summary.has_text, + }, + "targets": project.platform_check().map_err(core_err)?, + })) + } + + #[tool( + description = "Write the composited timeline at a given time to an image file as a cover / thumbnail — full \ + delivery resolution, rendered through the export graph, so it is a real frame of the finished \ + video at the shape the project is cut for." + )] + async fn export_cover(&self, Parameters(p): Parameters) -> Result { + let project = self.project.clone(); + let (time_secs, output_path) = (p.time_secs, p.output_path); + let written = blocking(move || { + let (timeline, assets) = lock_agent(&project).export_still_inputs().map_err(core_err)?; + Project::render_still(&timeline, &assets, time_secs, &output_path, None).map_err(core_err) + }) + .await?; + json(&serde_json::json!({ "output": written.to_string_lossy() })) + } + // ---- agent task queue -------------------------------------------------- #[tool(description = "List the agent task queue with each task's status (queued/working/ready/done/failed)")] @@ -1719,7 +1764,12 @@ impl ServerHandler for KerfMcp { Every applied edit is tracked: history lists the revisions, \ revision_diff explains one of them, and undo / redo / revert_to roll \ changes back (they work on the live cut, so apply or discard your \ - staged edits first). Call export to render." + staged edits first). Before you report a cut finished, run \ + platform_check: it says whether the length and frame shape suit \ + where it is going, including the reach limits a platform enforces \ + silently (a Reel over 3 minutes uploads fine and then reaches only \ + existing followers). Call export to render, and export_cover to \ + write the thumbnail the platform shows before anyone presses play." .to_string(), ); info From ab9847be6650e2a13d91ebadc8e2d1e5f57756ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Tue, 25 Aug 2026 21:28:54 +0200 Subject: [PATCH 3/7] show what the cut is ready for, and save its cover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The export dialog now leads with where this render can go: "Ready for Instagram Reels · YouTube Shorts · TikTok", then any length error or reach warning on its own line, then one collapsed line for shape. That last grouping is the point — a landscape cut earns a near-identical "will be letterboxed" from all four vertical feeds, and four lines say nothing four times, so issues carry an `IssueKind` and the panel folds them into one line naming the platforms. It judges the frame this render will actually produce, not the project's: a 9:16 project exported at 1920x1080 is a landscape file and is told so. The cover frame is saved from the preview's context menu at the playhead, and both a finished export and a saved cover offer "Show in folder" — a path in a toast is not much use on its own. kerf-core decides all of it. `platforms.ts` mirrors the check for the browser harness only, so the panel can be driven under `bun run dev`; verified there against both a 16:9 and a 9:16 render. --- CLAUDE.md | 51 +++++- crates/kerf-app/src/lib.rs | 8 +- crates/kerf-app/src/mcp.rs | 4 +- crates/kerf-core/src/lib.rs | 5 +- crates/kerf-core/src/platform.rs | 26 +++ crates/kerf-core/src/project.rs | 14 +- frontend/src/lib/api.ts | 48 +++++ .../lib/components/editor/ExportDialog.svelte | 93 +++++++++- .../src/lib/components/editor/Preview.svelte | 31 +++- frontend/src/lib/components/editor/icons.ts | 6 +- frontend/src/lib/platforms.test.ts | 75 ++++++++ frontend/src/lib/platforms.ts | 164 ++++++++++++++++++ frontend/src/lib/types.ts | 35 ++++ 13 files changed, 540 insertions(+), 20 deletions(-) create mode 100644 frontend/src/lib/platforms.test.ts create mode 100644 frontend/src/lib/platforms.ts diff --git a/CLAUDE.md b/CLAUDE.md index 4405d3e..de4d58e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,9 +24,14 @@ so the feature is **only** activated through these forwards — which is what ma detection, preview frames (`frame_at`; `frame_jpeg` for a low-res JPEG), the per-asset **contact sheet** (`contact_sheet` — a `tile`d grid of frames sampled across a range, for skimming footage) and the **composited timeline still** - (`timeline_frame` / `build_timeline_frame_args`, pure + unit-tested — overlays + (`timeline_frame` / `build_still_args`, pure + unit-tested — overlays every clip visible at a timeline time onto a black canvas, mirroring the export - geometry, so an agent can *see the cut*), waveforms, and export all live here, so + geometry, so an agent can *see the cut*), the **cover frame** (`export_still` — + the same graph and the same builder, but a `StillOutput::File` sink and no + preview width cap, so the thumbnail is a real frame of the finished video at + the delivery shape rather than a screenshot to crop back into agreement; the + preview keeps its MJPEG pipe, which is why every pre-existing still test is + byte-identical), waveforms, and export all live here, so they work in the `--no-default-features` build — only the binaries are needed, never the dev libraries. Preview decodes go through a **cached all-intra proxy** (`generate_proxy` in the background, `ready_proxy` never blocks, resolved by @@ -279,6 +284,24 @@ no editing logic in the adapter. every clip having been replaced, and a removed track is one entry instead of one per orphaned clip. `StagedEdit` is a pending proposal (base seq, the edit labels, `stale`, and its diff). +- `platform.rs` — **where the cut is going.** A static `TARGETS` table (Reels / + Shorts / TikTok / Instagram feed / YouTube: delivery frame, accepted aspects, + length limits) plus a pure, unit-tested `check` over a `CutSummary`. It keeps + two limits apart that are usually conflated: a **hard** limit is what a + platform rejects, a **reach** limit is what it accepts and then stops + distributing — a four-minute Reel uploads fine and is shown only to existing + followers, the worse outcome because nothing tells you. Findings carry a + `Severity` (error / warning / tip) *and* an `IssueKind` (empty / length / shape + / resolution / captions), because a landscape cut earns a near-identical shape + complaint from every vertical feed and the UI has to collapse those into one + line naming four platforms. Messages are phrased with the real numbers + ("0:20 over", "cutting 1:00 would keep it in the feed"); aspect is compared as + a **ratio**, so 720x1280 reads as the right shape and merely soft. The numbers + are other companies' product decisions, verified 2026-08-25 and **advisory** — + nothing here ever blocks an export. `Project::platform_check(frame)` resolves + the summary from `working_timeline` (so an agent is judged on its own + proposal) with an optional frame override, which the export dialog passes when + a render resizes away from the project frame. - `project.rs` — `Project` wraps a `rusqlite::Connection`. **Persistence shape:** `assets` and `analysis` are real tables (streams/analysis stored as JSON columns); the **entire timeline is a single JSON blob** in a one-row `timeline` table. All @@ -367,6 +390,10 @@ proposal appears for review, not that the cut changes: the read tools (`get_timeline_state`, `timeline_summary`, `preview_timeline`, `export`) go through `working_timeline`, so the agent sees the cut it is building, and `timeline_summary` carries `staged_changes` so it cannot mistake one for the other. +`platform_check` tells it whether the cut is publishable where it is going +(and the server `instructions` tell it to run that before reporting a cut +finished — an agent that assembles a four-minute Reel has done the work and lost +the audience), and `export_cover` writes the thumbnail. `stage_edits` / `staged_diff` (the entries plus a rendered text summary) / `apply_staged_edits` / `discard_staged_edits` drive it explicitly, and `revision_diff` explains a past revision. The server `instructions` spell the flow @@ -400,7 +427,12 @@ refreshed `Timeline`), media (`get_frame` → base64 PNG data URL, `get_waveform counter, because start and stop are separate async calls that can arrive out of order and a late stop must not kill the stream that replaced it — `get_audio` → a clip window as **raw mono s16le PCM via `tauri::ipc::Response`**, the -only non-JSON command — the preview's Web Audio playback decodes it), the +only non-JSON command — the preview's Web Audio playback decodes it), +delivery (`export_cover` → a cover image at the full delivery frame, +`platform_targets` / `platform_check` → the readiness verdict, `reveal_path` → +show a rendered file in the OS file manager, opening its *containing folder* +rather than the file, since "show me where it went" is not a request to launch a +player), the agent task queue (`list_tasks`, `add_task` → the new `Task`; `resolve_task` / `remove_task` → the refreshed `Task[]`), the agent's staged proposal (`get_staged_edit` → the `StagedEdit` *with its diff*, so the review card renders @@ -557,7 +589,18 @@ frame is height-bound in a wide pane, not squashed), and for a vertical or squar delivery it draws **safe-area guides** (the platform's top strip / caption rail / action column, plus a title-safe box; `ui.safeAreas`, toggled from the preview context menu). The export dialog's "Source" resolution relabels to -**Project frame (WxH)** so the two surfaces cannot silently disagree. +**Project frame (WxH)** so the two surfaces cannot silently disagree. It also +leads with a **readiness panel**: "Ready for Instagram Reels · YouTube Shorts · +TikTok", then any length errors / reach warnings one line each, then a *single* +collapsed line for shape ("A 16:9 cut is letterboxed on … Pick a delivery frame +in the toolbar") — grouped by `IssueKind`, because otherwise four vertical feeds +each say the same thing. It re-checks against `opts.resolution`, so a 9:16 +project exported at 1920×1080 is judged as the landscape file it will be. +`kerf_core::platform` decides all of it; `src/lib/platforms.ts` is a bun-tested +mirror used **only** by the browser harness, so the panel is drivable under +`bun run dev`. The **cover frame** is saved from the preview's context menu +(`Save cover frame…` → `export_cover` at the playhead), and both a finished +export and a saved cover offer **Show in folder** in their toast. `Preview` shows the composited frame under the playhead, and during **forward 1× playback it switches to the streamed frame source** (`start_playback`) — per-frame `get_timeline_frame` decodes stay for scrubbing, shuttle and the diff --git a/crates/kerf-app/src/lib.rs b/crates/kerf-app/src/lib.rs index 6cf8c97..207f2f6 100644 --- a/crates/kerf-app/src/lib.rs +++ b/crates/kerf-app/src/lib.rs @@ -1341,8 +1341,12 @@ fn platform_targets() -> Vec { /// How ready the current cut is for each target — what would be rejected, what /// would be accepted and then under-distributed, and what would just be better. #[tauri::command(async)] -fn platform_check(state: State<'_, AppState>) -> CmdResult> { - state.project().platform_check().map_err(|e| e.to_string()) +fn platform_check(state: State<'_, AppState>, width: Option, height: Option) -> CmdResult> { + // The export dialog can resize away from the project frame; when it does it + // passes the frame it is actually about to render, so the verdict is about + // the file that will exist rather than the one the project defaults to. + let frame = width.zip(height); + state.project().platform_check(frame).map_err(|e| e.to_string()) } /// Show a file in the OS file manager. The last step of an export: the render diff --git a/crates/kerf-app/src/mcp.rs b/crates/kerf-app/src/mcp.rs index 284f4ec..b4f94b4 100644 --- a/crates/kerf-app/src/mcp.rs +++ b/crates/kerf-app/src/mcp.rs @@ -1446,7 +1446,7 @@ impl KerfMcp { )] fn platform_check(&self) -> Result { let project = self.lock(); - let summary = project.cut_summary().map_err(core_err)?; + let summary = project.cut_summary(None).map_err(core_err)?; json(&serde_json::json!({ "cut": { "duration_secs": summary.duration, @@ -1454,7 +1454,7 @@ impl KerfMcp { "has_audio": summary.has_audio, "has_text": summary.has_text, }, - "targets": project.platform_check().map_err(core_err)?, + "targets": project.platform_check(None).map_err(core_err)?, })) } diff --git a/crates/kerf-core/src/lib.rs b/crates/kerf-core/src/lib.rs index dee5134..9aff907 100644 --- a/crates/kerf-core/src/lib.rs +++ b/crates/kerf-core/src/lib.rs @@ -28,7 +28,10 @@ pub use engine::{ SpeechModelInfo, DEFAULT_SPEECH_MODEL, }; pub use error::{Error, Result}; -pub use platform::{check_all as check_platforms, CutSummary, DeliveryCheck, DeliveryIssue, PlatformTarget, Severity, TARGETS as PLATFORM_TARGETS}; +pub use platform::{ + check_all as check_platforms, CutSummary, DeliveryCheck, DeliveryIssue, IssueKind, PlatformTarget, Severity, + TARGETS as PLATFORM_TARGETS, +}; pub use fonts::list_system_fonts; pub use model::{ Asset, AssetAnalysis, AudioEffect, Clip, Color, Delivery, DiffEntry, DiffKind, EditSource, Keyframe, Marker, Projection, diff --git a/crates/kerf-core/src/platform.rs b/crates/kerf-core/src/platform.rs index 93b06b8..d5d8b35 100644 --- a/crates/kerf-core/src/platform.rs +++ b/crates/kerf-core/src/platform.rs @@ -122,10 +122,29 @@ pub enum Severity { Tip, } +/// What an issue is *about*, so a UI can group it. Four targets that all want a +/// vertical frame produce four near-identical shape complaints; grouped, that is +/// one line naming four platforms. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum IssueKind { + /// Nothing to publish. + Empty, + /// Too long or too short — includes the reach limit. + Length, + /// The wrong aspect for this target. + Shape, + /// The right aspect, too few pixels. + Resolution, + /// Advice about reading muted. + Captions, +} + /// One thing to know before publishing this cut to a target. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct DeliveryIssue { pub severity: Severity, + pub kind: IssueKind, /// Already phrased for a person, with the actual numbers in it. pub message: String, } @@ -194,6 +213,7 @@ pub fn check(target: &PlatformTarget, cut: &CutSummary) -> DeliveryCheck { if cut.duration <= 0.0 { issues.push(DeliveryIssue { severity: Severity::Error, + kind: IssueKind::Empty, message: "The timeline is empty — there is nothing to publish.".to_string(), }); } @@ -201,6 +221,7 @@ pub fn check(target: &PlatformTarget, cut: &CutSummary) -> DeliveryCheck { if cut.duration > 0.0 && cut.duration < min { issues.push(DeliveryIssue { severity: Severity::Error, + kind: IssueKind::Length, message: format!( "{} is shorter than {}'s {:.0}s minimum.", fmt_dur(cut.duration), @@ -214,6 +235,7 @@ pub fn check(target: &PlatformTarget, cut: &CutSummary) -> DeliveryCheck { if cut.duration > max { issues.push(DeliveryIssue { severity: Severity::Error, + kind: IssueKind::Length, message: format!( "{} is over {}'s {} limit — trim {} to fit.", fmt_dur(cut.duration), @@ -230,6 +252,7 @@ pub fn check(target: &PlatformTarget, cut: &CutSummary) -> DeliveryCheck { if cut.duration > reach && within_hard { issues.push(DeliveryIssue { severity: Severity::Warning, + kind: IssueKind::Length, message: format!( "Over {}, {} stops showing this to people who don't already follow you. Cutting {} would keep it in the feed.", fmt_dur(reach), @@ -256,6 +279,7 @@ pub fn check(target: &PlatformTarget, cut: &CutSummary) -> DeliveryCheck { .join(" or "); issues.push(DeliveryIssue { severity: Severity::Warning, + kind: IssueKind::Shape, message: format!( "This cut is {} ({}×{}); {} shows {}, so it will be letterboxed. Set the delivery frame to {}×{}.", ratio_label(cut.width, cut.height), @@ -271,6 +295,7 @@ pub fn check(target: &PlatformTarget, cut: &CutSummary) -> DeliveryCheck { // Right shape, not enough pixels — the platform will upscale it. issues.push(DeliveryIssue { severity: Severity::Warning, + kind: IssueKind::Resolution, message: format!( "{}×{} is below {}'s {}×{}; the platform will upscale it and it will look soft.", cut.width, cut.height, target.label, target.width, target.height @@ -281,6 +306,7 @@ pub fn check(target: &PlatformTarget, cut: &CutSummary) -> DeliveryCheck { if !cut.has_text && cut.has_audio { issues.push(DeliveryIssue { severity: Severity::Tip, + kind: IssueKind::Captions, message: "The feed autoplays muted. Captions or a title would carry this for the people who never turn sound on." .to_string(), }); diff --git a/crates/kerf-core/src/project.rs b/crates/kerf-core/src/project.rs index 581847e..c29b3e8 100644 --- a/crates/kerf-core/src/project.rs +++ b/crates/kerf-core/src/project.rs @@ -621,18 +621,20 @@ impl Project { /// /// Reads the **working** timeline, so an agent assembling a cut sees the /// verdict on its own proposal rather than on the user's live one. - pub fn platform_check(&self) -> Result> { - Ok(crate::platform::check_all(&self.cut_summary()?)) + /// `frame` overrides the shape the cut is judged at — what the export dialog + /// passes when a render is about to resize away from the project frame. + pub fn platform_check(&self, frame: Option<(u32, u32)>) -> Result> { + Ok(crate::platform::check_all(&self.cut_summary(frame)?)) } /// What the readiness check needs to know about the current cut. - pub fn cut_summary(&self) -> Result { + pub fn cut_summary(&self, frame: Option<(u32, u32)>) -> Result { let timeline = self.working_timeline()?; let assets = self.list_assets()?; // The same gate the export applies, so a muted track is as absent here // as it will be in the file. let rendered = timeline.for_render(); - let (width, height) = engine::delivery_frame(&rendered, &assets); + let (width, height) = frame.unwrap_or_else(|| engine::delivery_frame(&rendered, &assets)); // Audio-bearing means what the export means by it: any clip whose asset // carries an audio stream, on a video track as much as an audio one. let has_audio = rendered.tracks.iter().flat_map(|t| t.clips.iter()).any(|c| { @@ -2899,12 +2901,12 @@ mod tests { fit: Fit::Cover, })) .unwrap(); - let summary = project.cut_summary().unwrap(); + let summary = project.cut_summary(None).unwrap(); assert_eq!((summary.width, summary.height), (1080, 1920)); assert!(summary.duration > 0.0); assert!(summary.has_audio, "the sample has audio-bearing clips"); - let checks = project.platform_check().unwrap(); + let checks = project.platform_check(None).unwrap(); assert_eq!(checks.len(), crate::platform::TARGETS.len()); let reels = checks.iter().find(|c| c.target == "reels").unwrap(); assert!(reels.ok, "a short vertical cut is publishable: {:?}", reels.issues); diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index b69c632..b0dad9a 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -13,6 +13,7 @@ import type { Clip, Color, Delivery, + DeliveryCheck, EditSource, ExportOptions, ExportProgress, @@ -39,6 +40,7 @@ import type { import { clipDuration, DEFAULT_COLOR, DEFAULT_REFRAME, DEFAULT_TRANSFORM } from './types'; import { alignCutsToBeats, beatGrid, defaultBeatTolerance } from './beats'; import { formatTime as fmtTime } from './diff'; +import { checkAll } from './platforms'; export function inTauri(): boolean { return typeof window !== 'undefined' && '__TAURI_INTERNALS__' in window; @@ -1620,6 +1622,52 @@ export async function pickExportPath(ext: string): Promise { return typeof path === 'string' ? path : null; } +/** Write the composited frame at `timeSecs` to `outputPath` as a cover image — + * full delivery resolution, through the export graph. */ +export async function exportCover(timeSecs: number, outputPath: string): Promise { + if (!inTauri()) throw new Error('saving a cover frame is only available in the desktop app'); + return invoke('export_cover', { timeSecs, outputPath, format: null }); +} + +/** Open a save dialog for a cover image. */ +export async function pickCoverPath(): Promise { + if (!inTauri()) return null; + const { save } = await import('@tauri-apps/plugin-dialog'); + const path = await save({ + filters: [{ name: 'Image', extensions: ['jpg', 'png'] }], + defaultPath: 'kerf-cover.jpg' + }); + return typeof path === 'string' ? path : null; +} + +/** Show a rendered file in the OS file manager (opens its containing folder). */ +export async function revealPath(path: string): Promise { + if (!inTauri()) return; + await invoke('reveal_path', { path }); +} + +/** How ready the current cut is for each publishing target. `frame` overrides + * the shape it is judged at — the export dialog passes the resolution it is + * about to render when that differs from the project frame. + * + * `kerf_core::platform` decides this in the app. The browser harness runs the + * mirror in `platforms.ts` over the dev timeline so the panel is explorable + * under `bun run dev`. */ +export async function platformCheck(frame?: [number, number] | null): Promise { + if (!inTauri()) { + const fmt = devTimeline.format; + const first = sampleAssets[0]; + return checkAll({ + duration: timelineDuration(devTimeline), + width: frame?.[0] ?? fmt?.width ?? first.streams[0]?.width ?? 1920, + height: frame?.[1] ?? fmt?.height ?? first.streams[0]?.height ?? 1080, + has_audio: true, + has_text: (devTimeline.overlays ?? []).length > 0 + }); + } + return invoke('platform_check', { width: frame?.[0] ?? null, height: frame?.[1] ?? null }); +} + // ---- agent connection (MCP endpoint) --------------------------------------- /** The local MCP endpoint a connected agent points at (e.g. http://127.0.0.1:7777/mcp). */ diff --git a/frontend/src/lib/components/editor/ExportDialog.svelte b/frontend/src/lib/components/editor/ExportDialog.svelte index fd5f58f..7b10cc3 100644 --- a/frontend/src/lib/components/editor/ExportDialog.svelte +++ b/frontend/src/lib/components/editor/ExportDialog.svelte @@ -3,9 +3,10 @@ import Btn from './Btn.svelte'; import { editor } from '$lib/state.svelte'; import { ui } from '$lib/editor-ui.svelte'; - import { inTauri, pickExportPath, cancelExport, onExportProgress, hwEncoders } from '$lib/api'; + import { inTauri, pickExportPath, cancelExport, onExportProgress, hwEncoders, platformCheck, revealPath } from '$lib/api'; import { toast } from 'svelte-sonner'; - import type { Container, ExportOptions, ExportProgress, Fit, RateControl } from '$lib/types'; + import { ratioLabel } from '$lib/delivery-formats'; + import type { Container, DeliveryCheck, ExportOptions, ExportProgress, Fit, RateControl } from '$lib/types'; import { PRESETS, CONTAINERS, @@ -43,6 +44,45 @@ let showCommand = $state(false); let useRange = $state(false); + // Where this cut is going: the platform limits it meets or misses. Judged at + // the resolution *this render* will produce, which is not always the project + // frame — a 9:16 project exported at 1920x1080 is a landscape file, and the + // panel has to say so. + let checks = $state([]); + $effect(() => { + const frame = opts.resolution ?? null; + platformCheck(frame) + .then((c) => (checks = c)) + .catch(() => (checks = [])); + }); + + /** Targets with nothing but tips against them. */ + const readyFor = $derived(checks.filter((c) => !c.issues.some((i) => i.severity !== 'tip'))); + /** Everything specific enough to be worth its own line — which is everything + * except the shape complaint, since a landscape cut earns one of those from + * every vertical feed and four near-identical lines say nothing four times. */ + const notes = $derived( + checks.flatMap((c) => + c.issues.filter((i) => i.severity !== 'tip' && i.kind !== 'shape').map((i) => ({ label: c.label, ...i })) + ) + ); + /** The targets this frame would be letterboxed on, collapsed to one line. */ + const wrongShape = $derived(checks.filter((c) => c.issues.some((i) => i.kind === 'shape')).map((c) => c.label)); + /** The frame this render will actually produce, as a ratio. */ + const cutRatio = $derived.by(() => { + const r = opts.resolution ?? (editor.timeline.format ? [editor.timeline.format.width, editor.timeline.format.height] : null); + return r ? ratioLabel(r[0], r[1]) : null; + }); + /** The tips, deduplicated — the same advice lands on every target. */ + const tips = $derived([ + ...new Set(checks.flatMap((c) => c.issues.filter((i) => i.severity === 'tip').map((i) => i.message))) + ]); + + function listLabels(labels: string[]): string { + if (labels.length < 2) return labels.join(''); + return `${labels.slice(0, -1).join(', ')} and ${labels[labels.length - 1]}`; + } + // GPU encoders the backend verified usable on this machine; merged into the // codec choices once known (empty in the browser harness). let hwCodecs = $state([]); @@ -180,7 +220,10 @@ try { const finalOpts = useRange && marks ? { ...opts, range: marks } : opts; const out = await editor.export(outputPath, finalOpts); - toast.success(`Exported → ${out}`); + // A path in a toast is not much use on its own — offer the folder. + toast.success(`Exported → ${out}`, { + action: { label: 'Show in folder', onClick: () => void revealPath(out).catch(() => {}) } + }); onClose(); } catch (e) { const m = msg(e); @@ -315,6 +358,50 @@ {summary} + + {#if checks.length} +
+ {#if readyFor.length} +
+ + Ready for {readyFor.map((c) => c.label).join(' · ')} +
+ {/if} + {#each notes as note, i (i)} +
+ + + + + {note.label} + — {note.message} + +
+ {/each} + {#if wrongShape.length} +
+ + + {#if cutRatio}A {cutRatio} cut is letterboxed on{:else}This frame is letterboxed on{/if} + {listLabels(wrongShape)}. Pick a delivery frame in the toolbar to cut for one of them. + +
+ {/if} + {#each tips as tip (tip)} +
+ + {tip} +
+ {/each} +
+ {/if} + {@render secHead('Destination')} {@render selectRow( diff --git a/frontend/src/lib/components/editor/Preview.svelte b/frontend/src/lib/components/editor/Preview.svelte index 94bb16e..4e72d47 100644 --- a/frontend/src/lib/components/editor/Preview.svelte +++ b/frontend/src/lib/components/editor/Preview.svelte @@ -5,7 +5,8 @@ import { ui } from '$lib/editor-ui.svelte'; import { editor } from '$lib/state.svelte'; import { contextMenu } from '$lib/context-menu.svelte'; - import { getTimelineFrame, startPlayback } from '$lib/api'; + import { exportCover, getTimelineFrame, inTauri, pickCoverPath, revealPath, startPlayback } from '$lib/api'; + import { toast } from 'svelte-sonner'; import { createFrameGate, PLAYBACK_FPS } from '$lib/playback-sync'; import { clipDuration } from '$lib/types'; @@ -196,6 +197,27 @@ const CHROME = { top: 0.08, bottom: 0.2, right: 0.14 }; const showGuides = $derived(ui.safeAreas && !!delivery && aspect < 1.2); + /** Write the frame under the playhead as a cover image — the thumbnail a + * platform shows before anyone presses play. Rendered at the full delivery + * frame from the original media, so it is the picture people actually see, + * not the downscaled preview on screen. */ + async function saveCover() { + if (!inTauri()) { + toast.info('Cover frames are rendered with FFmpeg in the desktop app.'); + return; + } + const path = await pickCoverPath(); + if (!path) return; + try { + const out = await exportCover(ui.time, path); + toast.success(`Cover saved → ${out}`, { + action: { label: 'Show in folder', onClick: () => void revealPath(out).catch(() => {}) } + }); + } catch (e) { + toast.error(e instanceof Error ? e.message : String(e)); + } + } + function onPreviewContextMenu(e: MouseEvent) { contextMenu.show(e, [ { @@ -220,6 +242,13 @@ icon: 'crop', disabled: !delivery, action: () => (ui.safeAreas = !ui.safeAreas) + }, + { type: 'separator' }, + { + label: 'Save cover frame…', + icon: 'image', + disabled: empty, + action: () => void saveCover() } ]); } diff --git a/frontend/src/lib/components/editor/icons.ts b/frontend/src/lib/components/editor/icons.ts index 86cf089..6e61f88 100644 --- a/frontend/src/lib/components/editor/icons.ts +++ b/frontend/src/lib/components/editor/icons.ts @@ -50,6 +50,8 @@ import { X, Download, RefreshCw, + TriangleAlert, + Lightbulb, ExternalLink } from '@lucide/svelte'; @@ -105,7 +107,9 @@ export const icons: Record = { 'x': X, 'download': Download, 'refresh-cw': RefreshCw, - 'external-link': ExternalLink + 'external-link': ExternalLink, + 'alert-triangle': TriangleAlert, + 'lightbulb': Lightbulb }; export type IconName = keyof typeof icons; diff --git a/frontend/src/lib/platforms.test.ts b/frontend/src/lib/platforms.test.ts new file mode 100644 index 0000000..a8490c7 --- /dev/null +++ b/frontend/src/lib/platforms.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, test } from 'bun:test'; +import { check, checkAll, fmtDur, TARGETS } from './platforms'; +import type { CutSummary } from './platforms'; + +const target = (id: string) => TARGETS.find((t) => t.id === id)!; + +const vertical = (duration: number): CutSummary => ({ + duration, + width: 1080, + height: 1920, + has_audio: true, + has_text: true +}); + +describe('fmtDur', () => { + test('reads as a length, not a timecode', () => { + expect(fmtDur(90)).toBe('1:30'); + expect(fmtDur(180)).toBe('3:00'); + expect(fmtDur(9)).toBe('0:09'); + }); +}); + +describe('check', () => { + test('a well-shaped cut passes clean', () => { + const c = check(target('reels'), vertical(45)); + expect(c.ok).toBe(true); + expect(c.issues).toEqual([]); + }); + + test('over the hard limit is an error naming the overshoot', () => { + const c = check(target('shorts'), vertical(200)); + expect(c.ok).toBe(false); + expect(c.issues[0].severity).toBe('error'); + expect(c.issues[0].message).toContain('3:00'); + expect(c.issues[0].message).toContain('0:20'); + }); + + test('over the reach limit is a warning, not a rejection', () => { + // The one that matters: this uploads fine and then nobody new sees it. + const c = check(target('reels'), vertical(4 * 60)); + expect(c.ok).toBe(true); + const warn = c.issues.find((i) => i.severity === 'warning')!; + expect(warn.message).toContain('follow you'); + expect(warn.message).toContain('1:00'); + }); + + test('a reach warning is dropped once the cut is already rejected', () => { + const c = check(target('tiktok'), vertical(70 * 60)); + expect(c.issues.filter((i) => i.severity === 'warning')).toHaveLength(0); + expect(c.issues.filter((i) => i.severity === 'error')).toHaveLength(1); + }); + + test('a landscape cut is flagged for a vertical feed', () => { + const c = check(target('reels'), { ...vertical(30), width: 1920, height: 1080 }); + expect(c.ok).toBe(true); + expect(c.issues[0].message).toContain('16:9'); + expect(c.issues[0].message).toContain('1080×1920'); + }); + + test('aspect is compared as a ratio, not as pixels', () => { + const c = check(target('reels'), { ...vertical(30), width: 720, height: 1280 }); + expect(c.issues).toHaveLength(1); + expect(c.issues[0].message).toContain('upscale'); + }); + + test('the muted-feed tip only fires when there is sound to miss', () => { + const noText = { ...vertical(30), has_text: false }; + expect(check(target('reels'), noText).issues.some((i) => i.severity === 'tip')).toBe(true); + expect(check(target('reels'), { ...noText, has_audio: false }).issues.some((i) => i.severity === 'tip')).toBe(false); + }); + + test('an empty timeline is rejected everywhere', () => { + expect(checkAll({ ...vertical(0), duration: 0 }).every((c) => !c.ok)).toBe(true); + }); +}); diff --git a/frontend/src/lib/platforms.ts b/frontend/src/lib/platforms.ts new file mode 100644 index 0000000..6fd0961 --- /dev/null +++ b/frontend/src/lib/platforms.ts @@ -0,0 +1,164 @@ +/* Publishing-target readiness, for the **browser dev harness only**. + * + * `kerf_core::platform` is authoritative: the desktop app always asks the + * backend, and the UI renders whatever it gets back. This mirror exists so the + * readiness panel can be built and driven under `bun run dev`, the same way the + * harness mirrors the rest of the project's ops — not as a second engine. + * + * Keep the numbers in step with `crates/kerf-core/src/platform.rs`. */ + +import type { DeliveryCheck, DeliveryIssue, PlatformTarget } from './types'; + +const VERTICAL: [number, number][] = [[9, 16]]; + +export const TARGETS: PlatformTarget[] = [ + { + id: 'reels', + label: 'Instagram Reels', + width: 1080, + height: 1920, + accepts: VERTICAL, + max_secs: 20 * 60, + reach_max_secs: 3 * 60, + min_secs: 3, + notes: 'Uploads accept up to 20 min, but past 3 min a Reel is only shown to existing followers.' + }, + { + id: 'shorts', + label: 'YouTube Shorts', + width: 1080, + height: 1920, + accepts: [ + [9, 16], + [4, 5], + [1, 1] + ], + max_secs: 3 * 60, + reach_max_secs: null, + min_secs: null, + notes: 'Hard 3 min cap since Oct 2024; anything longer is published as a regular video instead.' + }, + { + id: 'tiktok', + label: 'TikTok', + width: 1080, + height: 1920, + accepts: VERTICAL, + max_secs: 60 * 60, + reach_max_secs: 10 * 60, + min_secs: 3, + notes: 'Uploads accept up to 60 min. Under 3 min the file must stay below 500 MB, 3-10 min below 2 GB.' + }, + { + id: 'ig_feed', + label: 'Instagram feed', + width: 1080, + height: 1350, + accepts: [ + [4, 5], + [1, 1] + ], + max_secs: 20 * 60, + reach_max_secs: 3 * 60, + min_secs: 3, + notes: '4:5 takes the most vertical space in the feed. Feed video is distributed as a Reel.' + }, + { + id: 'youtube', + label: 'YouTube', + width: 1920, + height: 1080, + accepts: [[16, 9]], + max_secs: null, + reach_max_secs: null, + min_secs: null, + notes: 'No practical length limit; 16:9 fills the player without bars.' + } +]; + +export interface CutSummary { + duration: number; + width: number; + height: number; + has_audio: boolean; + has_text: boolean; +} + +/** `m:ss`, how a length is spoken about. */ +export function fmtDur(secs: number): string { + const s = Math.max(0, Math.round(secs)); + return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`; +} + +function ratioLabel(w: number, h: number): string { + const gcd = (a: number, b: number): number => (b ? gcd(b, a % b) : a); + const d = gcd(w, h) || 1; + return `${w / d}:${h / d}`; +} + +export function check(target: PlatformTarget, cut: CutSummary): DeliveryCheck { + const issues: DeliveryIssue[] = []; + + if (cut.duration <= 0) { + issues.push({ severity: 'error', kind: 'empty', message: 'The timeline is empty — there is nothing to publish.' }); + } + if (target.min_secs != null && cut.duration > 0 && cut.duration < target.min_secs) { + issues.push({ + severity: 'error', + kind: 'length', + message: `${fmtDur(cut.duration)} is shorter than ${target.label}'s ${target.min_secs}s minimum.` + }); + } + if (target.max_secs != null && cut.duration > target.max_secs) { + issues.push({ + severity: 'error', + kind: 'length', + message: `${fmtDur(cut.duration)} is over ${target.label}'s ${fmtDur(target.max_secs)} limit — trim ${fmtDur(cut.duration - target.max_secs)} to fit.` + }); + } + const withinHard = target.max_secs == null || cut.duration <= target.max_secs; + if (target.reach_max_secs != null && cut.duration > target.reach_max_secs && withinHard) { + issues.push({ + severity: 'warning', + kind: 'length', + message: `Over ${fmtDur(target.reach_max_secs)}, ${target.label} stops showing this to people who don't already follow you. Cutting ${fmtDur(cut.duration - target.reach_max_secs)} would keep it in the feed.` + }); + } + + const have = cut.height > 0 ? cut.width / cut.height : 0; + const want = target.width / target.height; + const fits = target.accepts.some(([w, h]) => Math.abs(have - w / h) <= (w / h) * 0.01); + if (have > 0 && !fits) { + const accepted = target.accepts.map(([w, h]) => ratioLabel(w, h)).join(' or '); + issues.push({ + severity: 'warning', + kind: 'shape', + message: `This cut is ${ratioLabel(cut.width, cut.height)} (${cut.width}×${cut.height}); ${target.label} shows ${accepted}, so it will be letterboxed. Set the delivery frame to ${target.width}×${target.height}.` + }); + } else if (have > 0 && cut.height < target.height && Math.abs(have - want) <= want * 0.01) { + issues.push({ + severity: 'warning', + kind: 'resolution', + message: `${cut.width}×${cut.height} is below ${target.label}'s ${target.width}×${target.height}; the platform will upscale it and it will look soft.` + }); + } + + if (!cut.has_text && cut.has_audio) { + issues.push({ + severity: 'tip', + kind: 'captions', + message: 'The feed autoplays muted. Captions or a title would carry this for the people who never turn sound on.' + }); + } + + return { + target: target.id, + label: target.label, + ok: !issues.some((i) => i.severity === 'error'), + issues + }; +} + +export function checkAll(cut: CutSummary): DeliveryCheck[] { + return TARGETS.map((t) => check(t, cut)); +} diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index bcb230c..4787eb2 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -399,6 +399,41 @@ export interface ExportOptions { /** How a clip's picture is fitted to an output frame of a different shape. */ export type Fit = 'contain' | 'cover'; +/** A place a finished cut gets published, mirroring `kerf_core::platform`. */ +export interface PlatformTarget { + id: string; + label: string; + width: number; + height: number; + accepts: [number, number][]; + max_secs: number | null; + reach_max_secs: number | null; + min_secs: number | null; + notes: string; +} + +/** `error` = the platform rejects it, `warning` = accepted then under-distributed + * or letterboxed, `tip` = advice. */ +export type Severity = 'error' | 'warning' | 'tip'; + +/** What an issue is about, so a UI can group four identical shape complaints + * into one line naming four platforms. */ +export type IssueKind = 'empty' | 'length' | 'shape' | 'resolution' | 'captions'; + +export interface DeliveryIssue { + severity: Severity; + kind: IssueKind; + message: string; +} + +/** A cut's readiness for one publishing target. */ +export interface DeliveryCheck { + target: string; + label: string; + ok: boolean; + issues: DeliveryIssue[]; +} + /** Payload of the `export-progress` event streamed during a render. */ export interface ExportProgress { fraction: number; From 99581fa402e23a7da50d2a5b5fbba84a9d79800f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Thu, 27 Aug 2026 01:55:33 +0200 Subject: [PATCH 4/7] find where a shot's content is, and crop to the delivery frame around it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reshaping a cut throws away most of one axis, and both fits pick that axis without looking: Cover takes the middle, Contain keeps everything and shrinks it into a letterboxed strip. Neither is right when the subject stands in the left third, which is where a subject usually stands. engine::salience_map decodes ~48 tiny gray frames of a source window in one ffmpeg pass and scores each cell by edge energy plus frame-to-frame motion, so a locked-off talking head scores on detail and a follow shot on both. Not face detection: no model to ship, and the answer only has to beat a centre crop. SalienceMap::crop_for then slides a window of the delivery aspect across that map and returns the crop that keeps the content, with a centre bias so a flat map resolves to the plain centre crop rather than to whichever edge won by rounding. Project::smart_crop applies it per clip as one revision, split for the lock-free pattern so the decodes don't hold the project lock. The result is an ordinary Transform crop, which the graph already applies before the fit scale — so the preview, the still and the export all follow, and the inspector's sliders still have the last word. --- crates/kerf-core/src/engine/cli.rs | 275 +++++++++++++++++++++++++- crates/kerf-core/src/engine/mod.rs | 2 +- crates/kerf-core/src/lib.rs | 6 +- crates/kerf-core/src/model.rs | 278 ++++++++++++++++++++++++++ crates/kerf-core/src/project.rs | 307 ++++++++++++++++++++++++++++- 5 files changed, 860 insertions(+), 8 deletions(-) diff --git a/crates/kerf-core/src/engine/cli.rs b/crates/kerf-core/src/engine/cli.rs index 97557c6..0e3dc9d 100644 --- a/crates/kerf-core/src/engine/cli.rs +++ b/crates/kerf-core/src/engine/cli.rs @@ -15,8 +15,8 @@ use std::sync::{Mutex, OnceLock}; use super::ProbeResult; use crate::error::{Error, Result}; use crate::model::{ - Asset, AudioEffect, Clip, Color, Projection, Reframe, ReframeKeyframe, ResolvedReframe, StreamInfo, StreamKind, TextOverlay, - TimeRange, Timeline, Transform, TransitionKind, VideoEffect, + Asset, AudioEffect, Clip, Color, Projection, Reframe, ReframeKeyframe, ResolvedReframe, SalienceMap, StreamInfo, StreamKind, + TextOverlay, TimeRange, Timeline, Transform, TransitionKind, VideoEffect, }; /// A small process-global LRU of decoded single frames. Decoded frames are a @@ -556,6 +556,142 @@ fn field_after(line: &str, key: &str) -> Option { rest[..end].parse().ok() } +// ---- salience sampling (smart crop) ---------------------------------------- + +/// Grid the salience sampler decodes at. Coarse on purpose: the answer is a +/// crop window, not a mask, and 64x36 gray pixels is 2.3 KB a frame — small +/// enough that hundreds of samples cost less than one preview still. +pub const SALIENCE_COLS: usize = 64; +pub const SALIENCE_ROWS: usize = 36; + +/// Frames sampled across the window. Enough to average out a blink or a +/// handheld wobble without decoding the whole clip's worth of pictures. +const SALIENCE_SAMPLES: usize = 48; + +/// Weight of *motion* against *detail* in a cell's score. Motion is the stronger +/// signal when there is any — a subject who moves is the subject — but a +/// locked-off talking head has none at all, which is why detail carries the +/// floor rather than being a tie-break. +const SALIENCE_MOTION_WEIGHT: f64 = 3.0; + +/// Sample where the content of `path`'s `[start, end)` window sits, as a coarse +/// [`SalienceMap`]. +/// +/// One ffmpeg pass decodes a few dozen tiny grayscale frames; each cell scores +/// the picture's edge energy plus how much it changed since the last sample. +/// Hardware-accelerated like [`detect_scenes`], with the same software retry — +/// the decode is the expensive half on 4K footage, and the arithmetic here runs +/// on 2 KB frames. +pub fn salience_map(path: &Path, start: f64, end: f64) -> Result { + use std::sync::atomic::Ordering; + + let bin = ffmpeg_bin(); + let args = build_salience_args(path, start, end); + let run = |hw: Option<&str>| { + let mut cmd = command(&bin); + if let Some(hw) = hw { + cmd.args(["-hwaccel", hw]); + } + cmd.args(&args).stderr(Stdio::piped()).output().map_err(|e| launch_err(&bin, e)) + }; + let hw = decode_hwaccel(); + let output = match hw.as_deref() { + Some(h) => { + let out = run(Some(h))?; + if out.status.success() && !out.stdout.is_empty() { + out + } else { + let sw = run(None)?; + if sw.status.success() { + HWACCEL_OK.store(false, Ordering::Relaxed); + tracing::warn!("hardware decode failed while sampling salience; using software decode"); + } + sw + } + } + None => run(None)?, + }; + if !output.status.success() { + return Err(Error::Engine(format!( + "could not sample the shot: {}", + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + Ok(score_salience(&output.stdout)) +} + +/// Pure arg builder for [`salience_map`] (no I/O, unit-tested). Fast-seeks to +/// the window, decodes at most [`SALIENCE_SAMPLES`] frames spread across it, and +/// writes them as raw gray at the analysis grid. `-an` because nothing here +/// looks at sound, and `fps` before `scale` so the scaler runs on the frames +/// that survive rather than on every one. +fn build_salience_args(path: &Path, start: f64, end: f64) -> Vec { + let start = start.max(0.0); + let window = (end - start).max(0.04); + // Spread the samples across the window, but never ask for more frames per + // second than a sane source has — a 0.2s clip wants every frame it has, not + // 240 duplicated ones. + let fps = (SALIENCE_SAMPLES as f64 / window).clamp(0.2, 30.0); + let mut args: Vec = ["-hide_banner", "-loglevel", "error", "-nostats"] + .iter() + .map(|s| s.to_string()) + .collect(); + args.push("-ss".into()); + args.push(format!("{start:.3}")); + args.push("-t".into()); + args.push(format!("{window:.3}")); + args.push("-i".into()); + args.push(path.to_string_lossy().into_owned()); + args.push("-an".into()); + args.push("-map".into()); + args.push("0:v:0?".into()); + args.push("-vf".into()); + args.push(format!( + "fps={fps:.4},scale={SALIENCE_COLS}:{SALIENCE_ROWS}:flags=bilinear,format=gray" + )); + args.push("-frames:v".into()); + args.push(SALIENCE_SAMPLES.to_string()); + args.push("-f".into()); + args.push("rawvideo".into()); + args.push("-pix_fmt".into()); + args.push("gray".into()); + args.push("pipe:1".into()); + args +} + +/// Score a run of raw gray [`SALIENCE_COLS`]x[`SALIENCE_ROWS`] frames into a +/// [`SalienceMap`]: per cell, the local edge energy of every frame plus the +/// frame-to-frame change, averaged over the frames actually decoded. Pure over +/// the decoded bytes, so the scoring is unit-testable without ffmpeg. +fn score_salience(raw: &[u8]) -> SalienceMap { + let (w, h) = (SALIENCE_COLS, SALIENCE_ROWS); + let stride = w * h; + let frames = raw.len() / stride; + if frames == 0 { + return SalienceMap::default(); + } + let mut cells = vec![0.0f64; stride]; + let mut prev: Option<&[u8]> = None; + for f in 0..frames { + let frame = &raw[f * stride..(f + 1) * stride]; + for y in 0..h { + for x in 0..w { + let i = y * w + x; + let p = frame[i] as f64; + // Edge energy: how much this pixel differs from its right and + // lower neighbours. Texture and outlines score, flat sky doesn't. + let dx = if x + 1 < w { (frame[i + 1] as f64 - p).abs() } else { 0.0 }; + let dy = if y + 1 < h { (frame[i + w] as f64 - p).abs() } else { 0.0 }; + let motion = prev.map_or(0.0, |q| (q[i] as f64 - p).abs()); + cells[i] += dx + dy + SALIENCE_MOTION_WEIGHT * motion; + } + } + prev = Some(frame); + } + let scale = 1.0 / (frames as f64 * 255.0); + SalienceMap::new(w, h, cells.into_iter().map(|c| (c * scale) as f32).collect()) +} + // ---- frame / waveform extraction ------------------------------------------ /// Decode a single frame at `time_secs` and return it as PNG bytes, scaled to @@ -4468,6 +4604,81 @@ mod tests { use chrono::Utc; use uuid::Uuid; + + // ---- salience sampling -------------------------------------------------- + + #[test] + fn salience_args_sample_the_window_at_the_analysis_grid() { + let args = build_salience_args(Path::new("/m/a.mp4"), 12.0, 22.0); + let joined = args.join(" "); + assert!(joined.contains("-ss 12.000 -t 10.000 -i /m/a.mp4"), "{joined}"); + // 48 samples over 10s. + assert!(joined.contains("fps=4.8000,scale=64:36:flags=bilinear,format=gray"), "{joined}"); + assert!(joined.contains("-frames:v 48"), "{joined}"); + assert!(joined.contains("-f rawvideo -pix_fmt gray pipe:1"), "{joined}"); + assert!(joined.contains("-an"), "{joined}"); + } + + #[test] + fn salience_args_clamp_the_sample_rate_for_absurd_windows() { + // A quarter-second clip must not ask for 192 fps... + let fast = build_salience_args(Path::new("/m/a.mp4"), 0.0, 0.25).join(" "); + assert!(fast.contains("fps=30.0000"), "{fast}"); + // ...nor an hour-long one for a frame every 75 seconds. + let slow = build_salience_args(Path::new("/m/a.mp4"), 0.0, 3600.0).join(" "); + assert!(slow.contains("fps=0.2000"), "{slow}"); + // A zero-length window still produces a runnable command. + let empty = build_salience_args(Path::new("/m/a.mp4"), 5.0, 5.0).join(" "); + assert!(empty.contains("-t 0.040"), "{empty}"); + } + + #[test] + fn scoring_no_frames_yields_an_empty_map() { + assert_eq!(score_salience(&[]), SalienceMap::default()); + // A partial frame is not a frame. + assert_eq!(score_salience(&[7u8; 128]), SalienceMap::default()); + } + + #[test] + fn scoring_finds_the_detailed_half_of_a_flat_frame() { + let (w, h) = (SALIENCE_COLS, SALIENCE_ROWS); + // Left half flat gray, right half a hard checkerboard. + let mut frame = vec![40u8; w * h]; + for y in 0..h { + for x in w / 2..w { + frame[y * w + x] = if (x + y) % 2 == 0 { 0 } else { 255 }; + } + } + let map = score_salience(&frame); + assert_eq!((map.cols, map.rows), (w, h)); + let crop = map.crop_for(1920, 1080, 1080.0 / 1920.0).expect("crops"); + assert!(crop.offset > 0.0, "the textured half pulls the window right: {crop:?}"); + } + + #[test] + fn scoring_weighs_a_moving_subject_over_a_static_one() { + let (w, h) = (SALIENCE_COLS, SALIENCE_ROWS); + let block = |frame: &mut Vec, from: usize, to: usize, v: u8| { + for y in h / 3..2 * h / 3 { + for x in from..to { + frame[y * w + x] = v; + } + } + }; + // Two frames: a static block on the left, a block on the right that moves. + let mut a = vec![0u8; w * h]; + block(&mut a, 4, 12, 200); + block(&mut a, w - 16, w - 8, 200); + let mut b = a.clone(); + block(&mut b, w - 16, w - 8, 0); + block(&mut b, w - 20, w - 12, 200); + let mut raw = a.clone(); + raw.extend_from_slice(&b); + let map = score_salience(&raw); + let crop = map.crop_for(1920, 1080, 1080.0 / 1920.0).expect("crops"); + assert!(crop.offset > 0.0, "motion wins over equal detail: {crop:?}"); + } + #[test] fn parses_silence_pairs() { let log = "\ @@ -7154,6 +7365,66 @@ mod tests { assert_eq!(&noisy[s..e], &a[..]); } + #[test] + fn a_smart_cropped_clip_crops_before_the_fit_so_cover_has_nothing_left_to_take() { + let asset = av_asset(Uuid::new_v4(), 30.0); // 1920x1080 + let mut clip = make_clip(asset.id, 0.0, 5.0, 0.0); + // What `smart_crop` writes for a 9:16 delivery, pulled left of centre. + let map = SalienceMap::new(4, 2, vec![1.0, 1.0, 0.01, 0.01, 1.0, 1.0, 0.01, 0.01]); + let crop = map.crop_for(1920, 1080, 1080.0 / 1920.0).expect("crops"); + clip.transform.crop_left = crop.left; + clip.transform.crop_right = crop.right; + + let fmt = ExportFormat { + width: 1080, + height: 1920, + fit: Fit::Cover, + ..ExportFormat::default() + }; + let chain = video_clip_chain(&clip, &fmt, &ClipFx::default(), false, "c0"); + let cropped = chain.find("crop=w=iw*").expect("the smart crop is in the graph"); + let scaled = chain.find("scale=1080:1920").expect("the fit is in the graph"); + assert!(cropped < scaled, "the crop must pick the window before the fit scales it: {chain}"); + // The window is off-centre — a plain Cover would have taken the middle. + assert!(chain.contains(&format!("x=iw*{}", crop.left)), "{chain}"); + assert!(crop.left < 0.3, "the subject is left of centre: {crop:?}"); + } + + /// End to end against the real `ffmpeg` binary: synthesize a 16:9 shot whose + /// only content sits in the left third, and check the sampler finds it and + /// the 9:16 crop keeps it — the case a centre crop gets wrong, and the whole + /// reason smart crop exists. Not part of the normal (binary-free) run: + /// `cargo test -p kerf-core --no-default-features -- --ignored samples_a_real` + #[test] + #[ignore = "needs the ffmpeg binary"] + fn samples_a_real_shot_and_frames_its_subject() { + let dir = std::env::temp_dir().join(format!("kerf-salience-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let media = dir.join("left.mp4"); + let ok = command(&ffmpeg_bin()) + .args(["-hide_banner", "-loglevel", "error", "-y"]) + .args(["-f", "lavfi", "-i", "testsrc=size=360x240:rate=30:duration=3"]) + .args(["-f", "lavfi", "-i", "color=c=black:s=1280x720:rate=30:duration=3"]) + .args(["-filter_complex", "[1][0]overlay=x=80:y=240"]) + .args(["-c:v", "libx264", "-pix_fmt", "yuv420p"]) + .arg(&media) + .status() + .expect("run ffmpeg"); + assert!(ok.success(), "could not synthesize test media"); + + let map = salience_map(&media, 0.0, 3.0).expect("sample"); + let _ = std::fs::remove_dir_all(&dir); + + assert_eq!((map.cols, map.rows), (SALIENCE_COLS, SALIENCE_ROWS)); + assert!(map.cells.iter().any(|c| *c > 0.0), "the map is empty"); + + let crop = map.crop_for(1280, 720, 1080.0 / 1920.0).expect("crops"); + assert!(crop.offset < 0.0, "the subject is on the left: {crop:?}"); + // The subject spans x = 80..440 of 1280, i.e. 0.0625..0.344 — a centre + // crop (0.342..0.658) would miss it entirely. + assert!(crop.left < 0.0625 && 1.0 - crop.right > 0.344, "{crop:?}"); + } + /// End to end against the real `ffmpeg` binary: synthesize a clip, play two /// seconds of a two-track timeline out of it, and check real JPEGs arrive at /// roughly the requested rate. Not part of the normal (binary-free) run: diff --git a/crates/kerf-core/src/engine/mod.rs b/crates/kerf-core/src/engine/mod.rs index f9750e1..7cddd4a 100644 --- a/crates/kerf-core/src/engine/mod.rs +++ b/crates/kerf-core/src/engine/mod.rs @@ -40,7 +40,7 @@ mod ffmpeg; // they only need the FFmpeg binaries, never the dev libraries. pub use cli::{ audio_effects_filter, audio_pcm, contact_sheet, decode_hwaccel, delivery_frame, detect_scenes, detect_silence, export_still, frame_at, - frame_jpeg, generate_proxy, hw_encoders, insta360_pair, proxy_path, proxy_width, ready_proxy, stitch_insta360, + frame_jpeg, generate_proxy, hw_encoders, insta360_pair, proxy_path, proxy_width, ready_proxy, salience_map, stitch_insta360, stitched_path, stream_preview, timeline_frame, validate_export, waveform, Container, ExportOptions, ExportProgress, Fit, ImageFormat, PreviewFrame, RateControl, RenderStatus, }; diff --git a/crates/kerf-core/src/lib.rs b/crates/kerf-core/src/lib.rs index 9aff907..26152e2 100644 --- a/crates/kerf-core/src/lib.rs +++ b/crates/kerf-core/src/lib.rs @@ -34,9 +34,9 @@ pub use platform::{ }; pub use fonts::list_system_fonts; pub use model::{ - Asset, AssetAnalysis, AudioEffect, Clip, Color, Delivery, DiffEntry, DiffKind, EditSource, Keyframe, Marker, Projection, - Reframe, ReframeKeyframe, ResolvedReframe, Revision, Rhythm, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, + Asset, AssetAnalysis, AudioEffect, Clip, Color, CropFrame, Delivery, DiffEntry, DiffKind, EditSource, Keyframe, Marker, + Projection, Reframe, ReframeKeyframe, ResolvedReframe, Revision, Rhythm, SalienceMap, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, TextKeyframe, TextOverlay, TimeRange, Timeline, TimelineDiff, Track, TranscriptSegment, Transform, Transition, TransitionKind, VideoEffect, }; -pub use project::Project; +pub use project::{Project, SmartCropJob, SmartCropPlan}; diff --git a/crates/kerf-core/src/model.rs b/crates/kerf-core/src/model.rs index 98e9c45..37bb1db 100644 --- a/crates/kerf-core/src/model.rs +++ b/crates/kerf-core/src/model.rs @@ -1365,6 +1365,190 @@ impl Delivery { } } +/// A coarse map of where a shot's *content* is: `rows`×`cols` non-negative +/// weights sampled across a source window, row-major. +/// +/// Built by [`crate::engine::salience_map`] from a handful of tiny grayscale +/// frames — per cell, the edge energy of the picture plus how much it moved. +/// That combination is what makes it usable on both kinds of shot a social cut +/// is made of: a locked-off talking head has no motion but plenty of facial +/// detail against a soft background, and a handheld follow has both. It is +/// deliberately *not* face detection — no model to ship, no licence to carry, +/// and the answer only has to be good enough to beat a centre crop, which is +/// what the alternative actually is. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct SalienceMap { + pub cols: usize, + pub rows: usize, + /// `rows * cols` weights, row-major. + pub cells: Vec, +} + +/// How far the salient window is allowed to pull away from centre before the +/// pull has to be *earned*. Scored against the window's share of total +/// salience, so a flat map (an evenly-lit wide shot, a gradient, black) resolves +/// to the centre crop rather than to whichever edge won by rounding. +const CENTER_BIAS: f64 = 0.25; + +/// Candidate window positions evaluated across the cropped axis. The window +/// edges are interpolated within a bucket, so this is finer than `cols`. +const CROP_SEARCH_STEPS: usize = 240; + +/// Aspect ratios within this relative tolerance are the same shape — 1920x1080 +/// into a 1280x720 frame needs no crop, and neither does 1080x1350 into 4:5. +const ASPECT_TOLERANCE: f64 = 0.01; + +/// A crop window as the per-edge source fractions [`Transform`] takes. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct CropFrame { + pub left: f64, + pub right: f64, + pub top: f64, + pub bottom: f64, + /// How far the window sits from a plain centre crop, as a fraction of the + /// travel available to it (0.0 = dead centre, 1.0 = hard against an edge). + /// Reported so a caller can say *why* the shot moved. + pub offset: f64, +} + +impl CropFrame { + /// The centred crop keeping `keep` of `axis` — what a `Cover` fit does on + /// its own, and the answer whenever the content gives no reason to move. + fn centered(keep: f64, horizontal: bool) -> Self { + Self::at(0.5 * (1.0 - keep), keep, horizontal, 0.0) + } + + fn at(start: f64, keep: f64, horizontal: bool, offset: f64) -> Self { + let (near, far) = (start, (1.0 - start - keep).max(0.0)); + if horizontal { + Self { + left: near, + right: far, + top: 0.0, + bottom: 0.0, + offset, + } + } else { + Self { + left: 0.0, + right: 0.0, + top: near, + bottom: far, + offset, + } + } + } + + /// Whether this window is (near enough) the plain centre crop. + pub fn is_centered(&self) -> bool { + self.offset.abs() < 1e-6 + } +} + +/// Whether footage of this shape has to lose part of itself to fill a frame of +/// `target_aspect`. False when the two are the same shape within +/// [`ASPECT_TOLERANCE`] — 1920x1080 into a 1280x720 frame keeps all of itself — +/// and when either shape is nonsense. +pub fn needs_crop(source_w: u32, source_h: u32, target_aspect: f64) -> bool { + let source_aspect = source_w as f64 / source_h.max(1) as f64; + if !source_aspect.is_finite() || source_aspect <= 0.0 || !target_aspect.is_finite() || target_aspect <= 0.0 { + return false; + } + ((source_aspect - target_aspect) / target_aspect).abs() > ASPECT_TOLERANCE +} + +impl SalienceMap { + pub fn new(cols: usize, rows: usize, cells: Vec) -> Self { + Self { cols, rows, cells } + } + + fn is_valid(&self) -> bool { + self.cols > 0 && self.rows > 0 && self.cells.len() == self.cols * self.rows + } + + /// Salience collapsed onto one axis: per column when `horizontal`, else per + /// row. Negative weights are clamped away so a bad sample can't subtract. + fn axis(&self, horizontal: bool) -> Vec { + let n = if horizontal { self.cols } else { self.rows }; + let mut out = vec![0.0; n]; + for (i, cell) in self.cells.iter().enumerate() { + let bucket = if horizontal { i % self.cols } else { i / self.cols }; + out[bucket] += (*cell as f64).max(0.0); + } + out + } + + /// The crop that frames this shot's content for `target_aspect`. + /// + /// `None` when the source is already that shape — there is nothing to + /// choose, and writing a no-op crop into every clip would only be noise in + /// the inspector. Otherwise the long axis is cropped to the target ratio and + /// the window is placed where the content is, which is the whole point: a + /// 16:9 interview with the subject on the left third loses their head to a + /// centre crop, and that is the default every other path here would take. + pub fn crop_for(&self, source_w: u32, source_h: u32, target_aspect: f64) -> Option { + if !needs_crop(source_w, source_h, target_aspect) { + return None; + } + let source_aspect = source_w as f64 / source_h.max(1) as f64; + // Wider than the frame → crop the width; taller → crop the height. + let horizontal = source_aspect > target_aspect; + let keep = if horizontal { + target_aspect / source_aspect + } else { + source_aspect / target_aspect + } + .clamp(0.01, 1.0); + + if !self.is_valid() { + return Some(CropFrame::centered(keep, horizontal)); + } + let weights = self.axis(horizontal); + let total: f64 = weights.iter().sum(); + if total <= 0.0 { + return Some(CropFrame::centered(keep, horizontal)); + } + + let travel = 1.0 - keep; + let mut best = (f64::NEG_INFINITY, 0.5 * travel); + for step in 0..=CROP_SEARCH_STEPS { + let start = travel * step as f64 / CROP_SEARCH_STEPS as f64; + let share = window_sum(&weights, start, start + keep) / total; + let drift = ((start + 0.5 * keep) - 0.5).abs() * 2.0; + let score = share - CENTER_BIAS * drift; + if score > best.0 { + best = (score, start); + } + } + + let start = best.1; + // Report — and store — the exact centre when the search landed on it, so + // an unmoved shot reads as unmoved instead of as a 0.4% pan. + let offset = if travel > 1e-9 { (start / travel - 0.5) * 2.0 } else { 0.0 }; + if offset.abs() < 0.02 { + return Some(CropFrame::centered(keep, horizontal)); + } + Some(CropFrame::at(start, keep, horizontal, offset)) + } +} + +/// Salience between two positions on a 0..1 axis, with the end buckets counted +/// by the fraction of them the window actually covers — so sliding the window +/// by less than a bucket changes the score smoothly instead of in steps. +fn window_sum(weights: &[f64], from: f64, to: f64) -> f64 { + let n = weights.len() as f64; + let (from, to) = (from.clamp(0.0, 1.0) * n, to.clamp(0.0, 1.0) * n); + let mut sum = 0.0; + for (i, w) in weights.iter().enumerate() { + let (lo, hi) = (i as f64, i as f64 + 1.0); + let overlap = to.min(hi) - from.max(lo); + if overlap > 0.0 { + sum += w * overlap; + } + } + sum +} + /// The non-destructive timeline (EDL): a set of multi-kind tracks, the text /// overlays (titles / lower-thirds / captions) drawn over the composited /// picture, and the user's markers. @@ -2746,4 +2930,98 @@ mod tests { // Every entry lands in the rendered summary the agent reads back. assert_eq!(diff.summary().lines().count(), 4); } + + // ---- smart crop --------------------------------------------------------- + + /// A map whose salience sits in one horizontal band of `cols`, so a test can + /// say "the subject is on the left third" and nothing else. + fn map_with_column_band(cols: usize, from: usize, to: usize) -> SalienceMap { + let rows = 4; + let mut cells = vec![0.01f32; cols * rows]; + for r in 0..rows { + for c in from..to { + cells[r * cols + c] = 1.0; + } + } + SalienceMap::new(cols, rows, cells) + } + + #[test] + fn a_matching_aspect_needs_no_crop() { + let map = map_with_column_band(32, 0, 32); + // 1080x1920 delivered at 9:16, and 1920x1080 at a 1280x720 frame. + assert!(map.crop_for(1080, 1920, 1080.0 / 1920.0).is_none()); + assert!(map.crop_for(1920, 1080, 1280.0 / 720.0).is_none()); + } + + #[test] + fn a_vertical_delivery_crops_width_toward_the_subject() { + // 16:9 footage into a 9:16 frame with the subject in the left third. + let map = map_with_column_band(48, 4, 16); + let crop = map.crop_for(1920, 1080, 1080.0 / 1920.0).expect("crops"); + assert_eq!((crop.top, crop.bottom), (0.0, 0.0)); + // 9:16 of 16:9 keeps 0.3164 of the width; the rest is cut. + assert!((crop.left + crop.right - (1.0 - 1080.0 * 1080.0 / (1920.0 * 1920.0))).abs() < 1e-6); + // The kept window contains the band, which a centre crop would miss. + assert!(crop.left < 4.0 / 48.0 && 1.0 - crop.right > 16.0 / 48.0, "{crop:?}"); + assert!(!crop.is_centered()); + assert!(crop.offset < 0.0, "a left-hand subject pulls the window left"); + } + + #[test] + fn flat_salience_falls_back_to_the_centre_crop() { + let map = map_with_column_band(48, 0, 48); + let crop = map.crop_for(1920, 1080, 1080.0 / 1920.0).expect("crops"); + assert!((crop.left - crop.right).abs() < 1e-9, "{crop:?}"); + assert!(crop.is_centered()); + } + + #[test] + fn an_off_centre_subject_still_loses_to_a_hard_pull_it_cannot_earn() { + // Salience a hair off centre: the centre bias should hold the window put + // rather than pan for a rounding difference. + let map = map_with_column_band(48, 23, 27); + let crop = map.crop_for(1920, 1080, 1080.0 / 1920.0).expect("crops"); + assert!(crop.is_centered(), "{crop:?}"); + } + + #[test] + fn a_landscape_delivery_crops_height_of_vertical_footage() { + // 9:16 footage into 16:9, subject in the top rows. + let cols = 4; + let rows = 32; + let mut cells = vec![0.01f32; cols * rows]; + for r in 2..8 { + for c in 0..cols { + cells[r * cols + c] = 1.0; + } + } + let map = SalienceMap::new(cols, rows, cells); + let crop = map.crop_for(1080, 1920, 1920.0 / 1080.0).expect("crops"); + assert_eq!((crop.left, crop.right), (0.0, 0.0)); + // The search grid is finer than a bucket but not exact, so allow the top + // edge to land a hair inside the band it is framing. + assert!(crop.top <= 2.0 / 32.0 + 0.01 && 1.0 - crop.bottom > 8.0 / 32.0, "{crop:?}"); + } + + #[test] + fn a_map_with_nothing_in_it_still_yields_the_centre_crop() { + for map in [ + SalienceMap::default(), + SalienceMap::new(8, 2, vec![0.0; 16]), + SalienceMap::new(8, 2, vec![0.0; 3]), + ] { + let crop = map.crop_for(1920, 1080, 1080.0 / 1920.0).expect("crops"); + assert!(crop.is_centered(), "{crop:?}"); + assert!((crop.left - crop.right).abs() < 1e-9); + } + } + + #[test] + fn a_degenerate_aspect_is_refused_rather_than_guessed() { + let map = map_with_column_band(8, 0, 4); + assert!(map.crop_for(1920, 1080, 0.0).is_none()); + assert!(map.crop_for(1920, 1080, f64::NAN).is_none()); + assert!(map.crop_for(0, 1080, 1.0).is_none()); + } } diff --git a/crates/kerf-core/src/project.rs b/crates/kerf-core/src/project.rs index c29b3e8..c5b5774 100644 --- a/crates/kerf-core/src/project.rs +++ b/crates/kerf-core/src/project.rs @@ -13,11 +13,32 @@ use crate::engine::{self, ExportProgress}; use crate::error::{Error, Result}; use crate::model::default_beat_tolerance; use crate::model::{ - Asset, AssetAnalysis, AudioEffect, Clip, Delivery, EditSource, Keyframe, Marker, Projection, Reframe, ReframeKeyframe, - Revision, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, Tempo, TextKeyframe, TextOverlay, TimeRange, Timeline, + Asset, AssetAnalysis, AudioEffect, Clip, CropFrame, Delivery, EditSource, Keyframe, Marker, Projection, Reframe, + ReframeKeyframe, Revision, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, Tempo, TextKeyframe, TextOverlay, TimeRange, Timeline, TimelineDiff, Track, Transition, VideoEffect, MAX_FOV, MIN_FOV, }; +/// One clip queued for smart-crop sampling: which media to look at, over which +/// source window, and the shape it was shot in. +#[derive(Debug, Clone)] +pub struct SmartCropJob { + pub clip_id: Uuid, + pub path: PathBuf, + pub start: f64, + pub end: f64, + pub width: u32, + pub height: u32, +} + +/// Everything the smart-crop sampler needs, resolved in one pass under the +/// project lock so the decodes it drives can run with the guard dropped. +#[derive(Debug, Clone)] +pub struct SmartCropPlan { + /// The aspect of the delivery frame — what every job is framed for. + pub target_aspect: f64, + pub jobs: Vec, +} + const SCHEMA: &str = r#" PRAGMA foreign_keys = ON; @@ -1675,6 +1696,167 @@ impl Project { }) } + // ---- smart crop --------------------------------------------------------- + + /// Frame each shot for the delivery frame instead of centring it blindly. + /// + /// Reshaping a cut — 16:9 footage into a 9:16 Reel — throws away most of the + /// width, and both fits pick that width without looking: `Cover` takes the + /// middle, `Contain` keeps everything and shrinks it into a letterboxed + /// strip. Neither is right when the subject stands in the left third, which + /// is where a subject usually stands. This samples where each shot's content + /// actually is and writes the crop that keeps it, per clip — so a cut of six + /// shots gets six framings rather than one compromise. + /// + /// The result is an ordinary `Transform` crop: visible in the inspector, + /// adjustable by hand, undoable in one step, and rendered by the graph that + /// was already there. Kerf proposes the framing; the crop sliders remain the + /// truth. `clip_id` narrows it to one clip; `None` reframes every clip on an + /// unlocked video track. Returns how many clips moved. + pub fn smart_crop(&self, clip_id: Option) -> Result { + let plan = self.smart_crop_inputs(clip_id)?; + let crops = Self::sample_smart_crops(&plan)?; + self.apply_smart_crops(&crops) + } + + /// Resolve what [`Project::smart_crop`] has to look at, **without** decoding + /// anything — so a caller can pull this out under the shared project lock and + /// drop the guard before [`Project::sample_smart_crops`] runs ffmpeg over + /// every clip. Mirrors [`Project::timeline_frame_inputs`]' shape. + /// + /// Clips already the right shape are left out rather than sampled: there is + /// no window to choose, and a no-op crop written into every clip would only + /// be noise. A 360 clip is left out too — its `reframe` already aims a camera + /// at the sphere, and that is the framing decision. + pub fn smart_crop_inputs(&self, clip_id: Option) -> Result { + let timeline = self.working_timeline()?; + let assets = self.list_assets()?; + let (fw, fh) = engine::delivery_frame(&timeline, &assets); + let target_aspect = fw as f64 / fh.max(1) as f64; + + let mut jobs = Vec::new(); + let mut skipped_shape = 0usize; + for track in timeline.tracks.iter().filter(|t| t.kind == StreamKind::Video) { + // A locked track is locked against a bulk pass, but naming one of its + // clips is still an explicit instruction. + if track.locked && clip_id.is_none() { + continue; + } + for clip in &track.clips { + if clip_id.is_some_and(|id| id != clip.id) { + continue; + } + if clip.reframe.is_some() { + continue; + } + let Some(asset) = assets.iter().find(|a| a.id == clip.asset_id) else { + continue; + }; + let Some((w, h)) = asset + .streams + .iter() + .find(|s| s.kind == StreamKind::Video) + .and_then(|s| s.width.zip(s.height)) + else { + continue; + }; + // Nothing to choose when the shot is already the delivery shape. + if !crate::model::needs_crop(w, h, target_aspect) { + skipped_shape += 1; + continue; + } + // A still has one frame at t=0 and no source timeline to seek + // into — sampling its clip window would decode nothing. + let (start, end) = if asset.is_image() { + (0.0, 0.04) + } else { + (clip.source_in.min(clip.source_out), clip.source_in.max(clip.source_out)) + }; + jobs.push(SmartCropJob { + clip_id: clip.id, + path: PathBuf::from(&asset.path), + start, + end, + width: w, + height: h, + }); + } + } + + if jobs.is_empty() { + let why = if skipped_shape > 0 { + format!("every shot is already {fw}x{fh}-shaped — nothing to reframe") + } else if clip_id.is_some() { + "that clip cannot be reframed — it is 360 footage, or its asset has no video".to_string() + } else { + "no video clips to reframe".to_string() + }; + return Err(Error::InvalidArgument(why)); + } + Ok(SmartCropPlan { target_aspect, jobs }) + } + + /// Sample every job in `plan` and pick its crop. Static and lock-free — this + /// is the slow half (one short ffmpeg decode per clip), so it must not run + /// with the project locked. A clip whose media cannot be read is skipped + /// rather than failing the batch; if *nothing* could be read, the first error + /// is returned so the caller has something to show. + pub fn sample_smart_crops(plan: &SmartCropPlan) -> Result> { + let mut crops = Vec::new(); + let mut first_error = None; + for job in &plan.jobs { + match engine::salience_map(&job.path, job.start, job.end) { + Ok(map) => { + if let Some(crop) = map.crop_for(job.width, job.height, plan.target_aspect) { + crops.push((job.clip_id, crop)); + } + } + Err(e) => { + tracing::warn!(clip = %job.clip_id, path = %job.path.display(), error = %e, "could not sample a shot for smart crop"); + first_error.get_or_insert(e); + } + } + } + match (crops.is_empty(), first_error) { + (true, Some(e)) => Err(e), + _ => Ok(crops), + } + } + + /// Write sampled crops onto their clips as one undoable edit. Returns how + /// many clips moved. + /// + /// Crops matching what a clip already had are dropped first, so re-running + /// the pass over an unchanged cut reports 0 *and* leaves the history alone — + /// a revision that changed nothing is only noise in the edit log. + pub fn apply_smart_crops(&self, crops: &[(Uuid, CropFrame)]) -> Result { + let timeline = self.working_timeline()?; + let pending: Vec<_> = crops + .iter() + .filter(|(clip_id, crop)| { + timeline.locate(*clip_id).is_some_and(|(ti, ci)| { + let t = &timeline.tracks[ti].clips[ci].transform; + (t.crop_left, t.crop_right, t.crop_top, t.crop_bottom) != (crop.left, crop.right, crop.top, crop.bottom) + }) + }) + .collect(); + if pending.is_empty() { + return Ok(0); + } + self.edit_timeline("Smart crop", |timeline| { + let mut changed = 0; + for (clip_id, crop) in &pending { + let Some((ti, ci)) = timeline.locate(*clip_id) else { + continue; + }; + let t = &mut timeline.tracks[ti].clips[ci].transform; + (t.crop_left, t.crop_right, t.crop_top, t.crop_bottom) = (crop.left, crop.right, crop.top, crop.bottom); + changed += 1; + } + Ok(changed) + }) + } + /// Update a clip's color correction. Each `None` leaves that field unchanged. pub fn set_color( &self, @@ -4043,4 +4225,125 @@ mod tests { // The baseline revision changed nothing by definition. assert!(project.revision_diff(0).unwrap().is_empty()); } + + // ---- smart crop --------------------------------------------------------- + + #[test] + fn smart_crop_has_nothing_to_do_when_the_footage_is_already_the_frame() { + let project = Project::sample().unwrap(); + // The sample is 1920x1080 with no explicit delivery frame, so the frame + // is derived from the footage and every shot already fills it. + let err = project.smart_crop_inputs(None).unwrap_err().to_string(); + assert!(err.contains("already"), "{err}"); + } + + #[test] + fn smart_crop_plans_every_video_clip_for_a_vertical_delivery() { + let project = Project::sample().unwrap(); + project.set_delivery_format(Some(Delivery::new(1080, 1920, Fit::Cover))).unwrap(); + let plan = project.smart_crop_inputs(None).unwrap(); + assert!((plan.target_aspect - 1080.0 / 1920.0).abs() < 1e-9); + let video_clips: usize = project + .timeline() + .unwrap() + .tracks + .iter() + .filter(|t| t.kind == StreamKind::Video) + .map(|t| t.clips.len()) + .sum(); + assert_eq!(plan.jobs.len(), video_clips); + // Each job points at real media over the clip's own source window. + for job in &plan.jobs { + assert!(job.path.to_string_lossy().ends_with(".mp4")); + assert!(job.end > job.start); + // Both sample sources are 16:9 — 1080p and 4K. + assert!((job.width as f64 / job.height as f64 - 16.0 / 9.0).abs() < 1e-6); + } + } + + #[test] + fn smart_crop_can_be_narrowed_to_one_clip() { + let project = Project::sample().unwrap(); + project.set_delivery_format(Some(Delivery::new(1080, 1920, Fit::Cover))).unwrap(); + let clip = first_video_clip(&project); + let plan = project.smart_crop_inputs(Some(clip)).unwrap(); + assert_eq!(plan.jobs.len(), 1); + assert_eq!(plan.jobs[0].clip_id, clip); + } + + #[test] + fn smart_crop_skips_a_locked_track_but_not_a_clip_named_on_one() { + let project = Project::sample().unwrap(); + project.set_delivery_format(Some(Delivery::new(1080, 1920, Fit::Cover))).unwrap(); + let clip = first_video_clip(&project); + let track = project + .timeline() + .unwrap() + .tracks + .iter() + .find(|t| t.clips.iter().any(|c| c.id == clip)) + .unwrap() + .id; + project.set_track_locked(track, true).unwrap(); + // The bulk pass leaves the locked track alone... + let bulk = project.smart_crop_inputs(None); + assert!(!bulk.map(|p| p.jobs.iter().any(|j| j.clip_id == clip)).unwrap_or(false)); + // ...but naming one of its clips is an explicit instruction. + assert_eq!(project.smart_crop_inputs(Some(clip)).unwrap().jobs.len(), 1); + } + + #[test] + fn applying_crops_is_one_undoable_edit_that_only_counts_real_changes() { + let project = Project::sample().unwrap(); + project.set_delivery_format(Some(Delivery::new(1080, 1920, Fit::Cover))).unwrap(); + let clip = first_video_clip(&project); + let crop = CropFrame { + left: 0.1, + right: 0.5836, + top: 0.0, + bottom: 0.0, + offset: -0.6, + }; + let before = project.history().unwrap().len(); + assert_eq!(project.apply_smart_crops(&[(clip, crop)]).unwrap(), 1); + let t = clip_transform(&project, clip); + assert!((t.crop_left - 0.1).abs() < 1e-9 && (t.crop_right - 0.5836).abs() < 1e-9); + assert_eq!(project.history().unwrap().len(), before + 1); + assert_eq!(project.history().unwrap().last().unwrap().label, "Smart crop"); + + // Re-running the same pass changes nothing, and leaves no revision behind. + assert_eq!(project.apply_smart_crops(&[(clip, crop)]).unwrap(), 0); + assert_eq!(project.history().unwrap().len(), before + 1); + + // The whole pass undoes in one step. + project.undo().unwrap(); + assert_eq!(clip_transform(&project, clip).crop_left, 0.0); + } + + #[test] + fn applying_no_crops_writes_no_revision() { + let project = Project::sample().unwrap(); + let before = project.history().unwrap().len(); + assert_eq!(project.apply_smart_crops(&[]).unwrap(), 0); + assert_eq!(project.history().unwrap().len(), before); + } + + fn first_video_clip(project: &Project) -> Uuid { + project + .timeline() + .unwrap() + .tracks + .iter() + .filter(|t| t.kind == StreamKind::Video) + .flat_map(|t| t.clips.iter()) + .next() + .unwrap() + .id + } + + fn clip_transform(project: &Project, clip_id: Uuid) -> crate::model::Transform { + let timeline = project.timeline().unwrap(); + let (ti, ci) = timeline.locate(clip_id).unwrap(); + timeline.tracks[ti].clips[ci].transform + } } From 0eb22fd987d171e89cd29b76c8761bcba2ba8f75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Thu, 27 Aug 2026 01:55:45 +0200 Subject: [PATCH 5/7] expose smart crop on both surfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both follow the shape every heavy op here uses: plan under the project lock, decode with it released, apply under it again — one short ffmpeg pass per clip must not freeze the window or stall the other MCP tools. The server instructions pair smart_crop with set_delivery_format, because an agent that reshapes a cut to 9:16 without it keeps whatever happened to be in the middle of every shot and has no way to know. --- crates/kerf-app/src/lib.rs | 22 ++++++++++++++++++++++ crates/kerf-app/src/mcp.rs | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/crates/kerf-app/src/lib.rs b/crates/kerf-app/src/lib.rs index 207f2f6..3d96763 100644 --- a/crates/kerf-app/src/lib.rs +++ b/crates/kerf-app/src/lib.rs @@ -1332,6 +1332,27 @@ async fn export_cover( .await } +/// Frame every shot for the delivery frame instead of centring it blindly — +/// samples where each clip's content sits and writes the crop that keeps it. +/// One clip when `clip_id` is given, otherwise every clip on an unlocked video +/// track. The result is an ordinary transform crop, so the inspector's sliders +/// still have the last word. +#[tauri::command] +async fn smart_crop(state: State<'_, AppState>, clip_id: Option) -> CmdResult { + let clip = clip_id.as_deref().map(id).transpose()?; + let shared = state.project.clone(); + blocking(move || { + // The usual shape for a heavy command: plan under the lock, decode + // without it (one short ffmpeg pass per clip), apply under it again. + let plan = lock_user(&shared).smart_crop_inputs(clip).map_err(|e| e.to_string())?; + let crops = Project::sample_smart_crops(&plan).map_err(|e| e.to_string())?; + let project = lock_user(&shared); + project.apply_smart_crops(&crops).map_err(|e| e.to_string())?; + project.timeline().map_err(|e| e.to_string()) + }) + .await +} + /// Every publishing target Kerf knows about, with its frame and length limits. #[tauri::command(async)] fn platform_targets() -> Vec { @@ -1583,6 +1604,7 @@ pub fn run() { export_srt, remove_silence, snap_to_beats, + smart_crop, extract_audio, concatenate, get_history, diff --git a/crates/kerf-app/src/mcp.rs b/crates/kerf-app/src/mcp.rs index b4f94b4..04545b5 100644 --- a/crates/kerf-app/src/mcp.rs +++ b/crates/kerf-app/src/mcp.rs @@ -162,6 +162,14 @@ struct TrackIdParams { track_id: String, } +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +struct SmartCropParams { + #[schemars( + description = "UUID of the clip to reframe; every clip on an unlocked video track when omitted" + )] + clip_id: Option, +} + #[derive(Debug, serde::Deserialize, schemars::JsonSchema)] struct SnapToBeatsParams { #[schemars(description = "UUID of the track to align; every unlocked video track when omitted")] @@ -1281,6 +1289,28 @@ impl KerfMcp { json(&out) } + #[tool( + description = "Smart crop: frame each shot for the delivery frame instead of centring it. Samples where a clip's content actually sits and writes the crop that keeps it, per clip — so 16:9 footage delivered at 9:16 keeps the subject rather than whatever happened to be in the middle. Set the delivery frame first (set_delivery_format); clips already that shape are left alone. The result is an ordinary transform crop the user can adjust or undo" + )] + async fn smart_crop(&self, Parameters(p): Parameters) -> Result { + let clip = p.clip_id.as_deref().map(parse_id).transpose()?; + let project = self.project.clone(); + let (moved, timeline) = blocking(move || { + // Plan under the lock, sample with it released (one short ffmpeg + // decode per clip), apply under it again — the same shape as + // analyze_asset, for the same reason. + let plan = lock_agent(&project).smart_crop_inputs(clip).map_err(core_err)?; + let crops = Project::sample_smart_crops(&plan).map_err(core_err)?; + let guard = lock_agent(&project); + let moved = guard.apply_smart_crops(&crops).map_err(core_err)?; + let timeline = guard.working_timeline().map_err(core_err)?; + Ok((moved, timeline)) + }) + .await?; + self.changed(); + json(&serde_json::json!({ "clips_reframed": moved, "timeline": timeline })) + } + #[tool( description = "Cut to the beat: ripple a track's cuts onto the beat grid of the analyzed music on the audio tracks, retrimming each clip so its outgoing cut lands on a beat. Analyze the music asset first" )] @@ -1751,6 +1781,12 @@ impl ServerHandler for KerfMcp { (drawn over the cut; list_fonts lists installed system fonts to pass \ as update_overlay's font), or captions_from_transcript to caption an \ analyzed asset in one call; export_srt writes a subtitle file. \ + When the cut is going somewhere vertical, set_delivery_format sets \ + the frame it is being made for and smart_crop then frames each shot \ + for it — reshaping 16:9 footage to 9:16 throws away most of the \ + width, and without this the middle is what survives, subject or \ + not. Look at the result with preview_timeline; the crop is an \ + ordinary transform the user can adjust. \ Your task edits are STAGED, not applied: claiming a task opens a \ proposal, and every edit you make goes into it instead of changing \ the cut the user is looking at. Your own reads follow the proposal, \ From 240e04c733f77ffbe5e3bf64efeeafb12b56ef62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Thu, 27 Aug 2026 01:55:45 +0200 Subject: [PATCH 6/7] frame every shot for the delivery frame from the editor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inspector grows a Framing section above the crop sliders it writes: Smart crop for the selected shot, Reset crop, and — when the shot already matches the frame, or is 360 and framed by its virtual camera — a line saying why the button is off rather than a control that would refuse. The agent panel gains a "Frame for the delivery" chip for the whole cut, next to the other presets that run a local op. smart-crop.ts mirrors only the *shape* arithmetic for the browser harness, the way platforms.ts does: with no decoder to sample with it lands on the centre window, and which part of the shot survives is the half that only exists with media behind it. --- CLAUDE.md | 44 ++++++++++++-- frontend/src/lib/api.ts | 39 +++++++++++++ .../lib/components/editor/AgentPanel.svelte | 9 +++ .../lib/components/editor/Inspector.svelte | 57 +++++++++++++++++++ frontend/src/lib/components/editor/data.ts | 9 ++- frontend/src/lib/smart-crop.test.ts | 43 ++++++++++++++ frontend/src/lib/smart-crop.ts | 46 +++++++++++++++ frontend/src/lib/state.svelte.ts | 9 +++ 8 files changed, 250 insertions(+), 6 deletions(-) create mode 100644 frontend/src/lib/smart-crop.test.ts create mode 100644 frontend/src/lib/smart-crop.ts diff --git a/CLAUDE.md b/CLAUDE.md index de4d58e..df1fe96 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,7 +23,13 @@ so the feature is **only** activated through these forwards — which is what ma (override with `KERF_FFMPEG` / `KERF_FFPROBE`). Probe, `silencedetect`, scene detection, preview frames (`frame_at`; `frame_jpeg` for a low-res JPEG), the per-asset **contact sheet** (`contact_sheet` — a `tile`d grid of frames sampled - across a range, for skimming footage) and the **composited timeline still** + across a range, for skimming footage), the **salience map** behind smart crop + (`salience_map` / `build_salience_args` / `score_salience`, the last two pure + + unit-tested — one pass decodes ~48 tiny gray frames of a source window and scores + each cell by edge energy plus frame-to-frame motion, so a locked-off talking head + scores on detail and a follow shot on both; deliberately *not* face detection — + no model to ship, and the answer only has to beat a centre crop) and the + **composited timeline still** (`timeline_frame` / `build_still_args`, pure + unit-tested — overlays every clip visible at a timeline time onto a black canvas, mirroring the export geometry, so an agent can *see the cut*), the **cover frame** (`export_still` — @@ -265,6 +271,12 @@ no editing logic in the adapter. carries a `duck` flag (sidechain-ducked under the rest of the mix on export). `Fit` and `Delivery` live here (the domain owns the delivery shape; `engine::cli` re-exports `Fit`), and `Timeline.format` is the frame the project is cut for. + **Smart crop** is here too and pure + unit-tested: `SalienceMap::crop_for` slides a + window of the delivery aspect across the sampled map and returns the `CropFrame` + (per-edge fractions, plus how far off centre it landed) that keeps the content — + with a `CENTER_BIAS` so a flat map resolves to the plain centre crop rather than to + whichever edge won by rounding, and `needs_crop` short-circuiting footage that is + already the delivery shape. Inherent helpers (`Timeline::locate`, `Track::end`/`reflow`, `Clip::duration`, `Timeline::slice` — the shifted sub-timeline copy behind range export) back the operations. **Beat alignment** lives here too and is pure + unit-tested: @@ -315,7 +327,18 @@ no editing logic in the adapter. every asset's cached `Tempo`, builds the grid and aligns one track (or every unlocked video track) to it, defaulting the tolerance to half a beat so each cut moves to the beat it is already nearest; it errors when nothing rhythmic has been - analyzed rather than silently doing nothing. The **agent task queue** is a real `tasks` table (one row per `Task`, + analyzed rather than silently doing nothing. + `smart_crop(clip_id)` is "frame it for where it's going": reshaping a cut throws + away most of one axis and both fits pick that axis blindly — `Cover` takes the + middle, `Contain` letterboxes — so it samples where each shot's content actually + sits and writes the crop that keeps it, **per clip**, as one `Smart crop` revision. + Split three ways for the lock-free pattern (`smart_crop_inputs` under the lock → + the static `sample_smart_crops` with it released → `apply_smart_crops` under it + again); the result is an ordinary `Transform` crop, which the graph already applies + *before* the fit scale, so the preview, the still and the export all follow and the + inspector's sliders still have the last word. Clips already the delivery shape and + 360-reframed clips are left out (that camera *is* the framing decision), and a pass + that changes nothing writes no revision. The **agent task queue** is a real `tasks` table (one row per `Task`, columns not JSON): `add_task` / `list_tasks` / `claim_next_task` / `complete_task` / `fail_task` / `resolve_task` / `remove_task` drive the `queued → working → ready → done` (or `failed`) lifecycle in `model.rs`. @@ -390,6 +413,9 @@ proposal appears for review, not that the cut changes: the read tools (`get_timeline_state`, `timeline_summary`, `preview_timeline`, `export`) go through `working_timeline`, so the agent sees the cut it is building, and `timeline_summary` carries `staged_changes` so it cannot mistake one for the other. +`smart_crop` frames each shot for the delivery frame (the server `instructions` +pair it with `set_delivery_format`, since reshaping to 9:16 otherwise keeps +whatever was in the middle). `platform_check` tells it whether the cut is publishable where it is going (and the server `instructions` tell it to run that before reporting a cut finished — an agent that assembles a four-minute Reel has done the work and lost @@ -420,6 +446,7 @@ width/height to clear it), `remove_clip`, `set_volume`, `set_fade`, `set_asset_projection` (asset-level 360 mark; returns the `Asset`), `add_overlay` / `update_overlay` / `remove_overlay` / `set_overlay_keyframes`, `captions_from_transcript`, `export_srt`, `remove_silence`, `snap_to_beats`, +`smart_crop` (frame each shot for the delivery frame), `extract_audio`, `concatenate` — each returns the refreshed `Timeline`), media (`get_frame` → base64 PNG data URL, `get_waveform`, `start_playback` / `stop_playback` — streamed composited frames over a @@ -532,7 +559,10 @@ editor-grade workspace under `src/lib/components/editor/` — bespoke atoms (`Bt `routes/+page.svelte`. The `Inspector` (right panel) edits the selected clip — trim, volume, fades, speed, transform, color, transition, plus **video / audio effect chains** (add / tune / remove), **keyframe animation** (the Transform panel -auto-keyframes at the playhead and shows the sampled pose), a **360 reframe** +auto-keyframes at the playhead and shows the sampled pose), a **Framing** section +(a `Smart crop` button that frames *this* shot for the delivery frame, plus +`Reset crop`, above the crop sliders it writes — greyed out with a reason when the +shot already matches the frame or is 360), a **360 reframe** section (yaw / pitch / roll / FOV, auto-keyframing at the playhead like Transform — note its `lerpAngle` takes the shortest arc, which plain `lerp` would read as a 340° swing across the seam; for a source Kerf did not @@ -598,7 +628,10 @@ each say the same thing. It re-checks against `opts.resolution`, so a 9:16 project exported at 1920×1080 is judged as the landscape file it will be. `kerf_core::platform` decides all of it; `src/lib/platforms.ts` is a bun-tested mirror used **only** by the browser harness, so the panel is drivable under -`bun run dev`. The **cover frame** is saved from the preview's context menu +`bun run dev`. `src/lib/smart-crop.ts` is the same arrangement for smart crop: only +the *shape* arithmetic is mirrored (bun-tested), because the harness has no decoder +to sample with and so lands on the centre window — which part of the shot survives +is the half that only exists with media behind it. The **cover frame** is saved from the preview's context menu (`Save cover frame…` → `export_cover` at the playhead), and both a finished export and a saved cover offer **Show in folder** in their toast. `Preview` shows the composited frame under the playhead, and during @@ -632,7 +665,8 @@ queue** (status · queue · history · add-task) — Kerf has no in-app chat; a LLM claims tasks over MCP. The queue is `agent` state (`src/lib/agent.svelte.ts`, a third runes singleton) backed by the `tasks` table over Tauri/MCP: the add-task box and preset chips `agent.add(...)` real tasks, and `ready` tasks show Apply/Dismiss (`resolve_task`/`remove_task`). -Three preset chips (`Remove silences` / `Assemble rough cut` / `Cut to the beat` — which +Four preset chips (`Remove silences` / `Assemble rough cut` / `Frame for the delivery` +/ `Cut to the beat` — which analyzes whatever is on the audio tracks first, then calls `snap_to_beats`, and says "No cuts were near a beat" instead of claiming an alignment when the grid never reached them) also run the matching local op and diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index b0dad9a..ee39060 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -41,6 +41,7 @@ import { clipDuration, DEFAULT_COLOR, DEFAULT_REFRAME, DEFAULT_TRANSFORM } from import { alignCutsToBeats, beatGrid, defaultBeatTolerance } from './beats'; import { formatTime as fmtTime } from './diff'; import { checkAll } from './platforms'; +import { centeredCrop } from './smart-crop'; export function inTauri(): boolean { return typeof window !== 'undefined' && '__TAURI_INTERNALS__' in window; @@ -1264,6 +1265,44 @@ export async function snapToBeats(trackId?: string, tolerance?: number): Promise return invoke('snap_to_beats', { trackId, tolerance }); } +/** Frame each shot for the delivery frame instead of centring it blindly. + * + * The app samples where each clip's content actually sits (one short ffmpeg + * pass per clip) and writes the crop that keeps it. The browser harness has no + * decoder, so it applies the centre window from `smart-crop.ts` — the shape is + * right, the choice of *which* part of the shot survives is the half that only + * exists with media behind it. Either way the result is an ordinary transform + * crop the inspector can adjust. */ +export async function smartCrop(clipId?: string): Promise { + if (!inTauri()) { + const fmt = devTimeline.format; + const first = sampleAssets[0]?.streams.find((s) => s.kind === 'video'); + const aspect = (fmt?.width ?? first?.width ?? 1920) / (fmt?.height ?? first?.height ?? 1080); + let moved = 0; + for (const track of devTimeline.tracks) { + if (track.kind !== 'video' || (track.locked && !clipId)) continue; + for (const clip of track.clips) { + if (clipId && clip.id !== clipId) continue; + const stream = assetById(clip.asset_id)?.streams.find((s) => s.kind === 'video'); + const crop = stream?.width && stream?.height ? centeredCrop(stream.width, stream.height, aspect) : null; + if (!crop) continue; + clip.transform = { + ...(clip.transform ?? DEFAULT_TRANSFORM), + crop_left: crop.left, + crop_right: crop.right, + crop_top: crop.top, + crop_bottom: crop.bottom + }; + moved += 1; + } + } + if (!moved) throw new Error('every shot is already that shape — nothing to reframe'); + recordDev('Smart crop'); + return snapshot(); + } + return invoke('smart_crop', { clipId }); +} + export async function extractAudio(assetId: string): Promise { if (!inTauri()) { const asset = assetById(assetId); diff --git a/frontend/src/lib/components/editor/AgentPanel.svelte b/frontend/src/lib/components/editor/AgentPanel.svelte index 80fa4d9..0ee08f2 100644 --- a/frontend/src/lib/components/editor/AgentPanel.svelte +++ b/frontend/src/lib/components/editor/AgentPanel.svelte @@ -205,6 +205,15 @@ // rather than claiming an alignment that never happened. if (cutSignature() === before) toast.info('No cuts were near a beat'); else toast.success('Aligned the cuts to the beat'); + } else if (task && p === 'Frame for the delivery') { + // Smart crop only matters once the project has a frame to be cut + // for; without one the frame follows the footage and every shot + // already fills it. + await editor.smartCrop(); + await agent.resolve(task.id); + toast.success('Framed every shot for the delivery frame', { + action: { label: 'Undo', onClick: () => void editor.undo() } + }); } else { toast.info(`Queued “${p}” — your connected agent claims tasks over MCP`); } diff --git a/frontend/src/lib/components/editor/Inspector.svelte b/frontend/src/lib/components/editor/Inspector.svelte index 7065bd2..a6e0cf8 100644 --- a/frontend/src/lib/components/editor/Inspector.svelte +++ b/frontend/src/lib/components/editor/Inspector.svelte @@ -8,6 +8,7 @@ import type { MenuItem } from '$lib/context-menu.svelte'; import { clipDuration, DEFAULT_COLOR, DEFAULT_REFRAME, DEFAULT_TRANSFORM } from '$lib/types'; import { COLOR_LOOKS, TEXT_STYLES, activeLook } from '$lib/style-presets'; + import { needsCrop } from '$lib/smart-crop'; import type { TextStyle } from '$lib/style-presets'; import type { AudioEffect, @@ -244,6 +245,30 @@ .padStart(2, '0')}`; } + // Smart crop only has a decision to make when the shot and the delivery frame + // are different shapes — a 16:9 interview headed for a 9:16 Reel loses most + // of its width, and which half survives is the whole question. When they + // already match there is nothing to choose, so say so instead of offering a + // button that would refuse. + const deliveryAspect = $derived.by(() => { + const fmt = editor.timeline.format; + if (fmt?.width && fmt?.height) return fmt.width / fmt.height; + const first = editor.assets + .flatMap((a) => a.streams) + .find((st) => st.kind === 'video' && st.width && st.height); + return first?.width && first?.height ? first.width / first.height : 16 / 9; + }); + const shotAspect = $derived.by(() => { + const v = asset?.streams.find((st) => st.kind === 'video'); + return v?.width && v?.height ? { w: v.width, h: v.height } : null; + }); + const reframable = $derived( + !!shotAspect && !clip?.reframe && needsCrop(shotAspect.w, shotAspect.h, deliveryAspect) + ); + const hasCrop = $derived( + tf.crop_left > 0 || tf.crop_right > 0 || tf.crop_top > 0 || tf.crop_bottom > 0 + ); + async function run(op: () => Promise) { try { await op(); @@ -747,6 +772,38 @@ {@render rangeRow('Opacity', tf.opacity, 0, 1, 0.05, (v) => `${Math.round(v * 100)}%`, (v) => setTf({ opacity: v }) )} + {@render secHead('Framing')} +
+ + +
+ {#if !reframable} +
+ {clip.reframe + ? 'A 360 clip is framed by its virtual camera, above.' + : 'This shot already matches the delivery frame.'} +
+ {/if} {@render rangeRow('Crop L', tf.crop_left, 0, 0.9, 0.01, (v) => v.toFixed(2), (v) => run(() => editor.setTransform(clip.id, { crop_left: v })) )} diff --git a/frontend/src/lib/components/editor/data.ts b/frontend/src/lib/components/editor/data.ts index 04d53df..17e4842 100644 --- a/frontend/src/lib/components/editor/data.ts +++ b/frontend/src/lib/components/editor/data.ts @@ -13,4 +13,11 @@ export const STATUS_MAP: Record { + test('same shape at a different size needs nothing', () => { + expect(needsCrop(1920, 1080, 1280 / 720)).toBe(false); + expect(needsCrop(1080, 1920, 1080 / 1920)).toBe(false); + }); + + test('a different shape does', () => { + expect(needsCrop(1920, 1080, 1080 / 1920)).toBe(true); + expect(needsCrop(1080, 1920, 16 / 9)).toBe(true); + expect(needsCrop(1920, 1080, 4 / 5)).toBe(true); + }); + + test('nonsense is refused rather than guessed', () => { + expect(needsCrop(1920, 1080, 0)).toBe(false); + expect(needsCrop(1920, 1080, NaN)).toBe(false); + expect(needsCrop(0, 1080, 1)).toBe(false); + }); +}); + +describe('centeredCrop', () => { + test('16:9 into 9:16 crops the width symmetrically', () => { + const crop = centeredCrop(1920, 1080, 1080 / 1920)!; + expect(crop.top).toBe(0); + expect(crop.bottom).toBe(0); + expect(crop.left).toBeCloseTo(crop.right, 12); + // 9:16 of 16:9 keeps 0.3164 of the width. + expect(1 - crop.left - crop.right).toBeCloseTo((1080 * 1080) / (1920 * 1920), 12); + }); + + test('9:16 into 16:9 crops the height instead', () => { + const crop = centeredCrop(1080, 1920, 16 / 9)!; + expect(crop.left).toBe(0); + expect(crop.right).toBe(0); + expect(crop.top).toBeCloseTo(crop.bottom, 12); + }); + + test('matching footage is left alone', () => { + expect(centeredCrop(1920, 1080, 16 / 9)).toBeNull(); + }); +}); diff --git a/frontend/src/lib/smart-crop.ts b/frontend/src/lib/smart-crop.ts new file mode 100644 index 0000000..372e11e --- /dev/null +++ b/frontend/src/lib/smart-crop.ts @@ -0,0 +1,46 @@ +/** + * Smart crop, the shape half. + * + * `kerf_core::model` decides the real thing in the app: it samples where a + * shot's content sits and picks the crop window that keeps it. Only the + * *shape* arithmetic — does this footage have to be cropped at all, and how + * much of which axis survives — is mirrored here, for the browser harness, + * which has no decoder to sample with and so always lands on the centre + * window. Same split as `platforms.ts`: one decision, made in Rust, with just + * enough of it in TS to keep `bun run dev` drivable. + */ + +/** Aspects within this relative tolerance are the same shape. Mirrors `model.rs`. */ +export const ASPECT_TOLERANCE = 0.01; + +export interface CropFrame { + left: number; + right: number; + top: number; + bottom: number; + /** How far the window sits from a plain centre crop (0 = dead centre). */ + offset: number; +} + +/** Whether footage of this shape loses part of itself filling `targetAspect`. */ +export function needsCrop(sourceWidth: number, sourceHeight: number, targetAspect: number): boolean { + const source = sourceWidth / Math.max(sourceHeight, 1); + if (!isFinite(source) || source <= 0 || !isFinite(targetAspect) || targetAspect <= 0) return false; + return Math.abs((source - targetAspect) / targetAspect) > ASPECT_TOLERANCE; +} + +/** + * The centred crop for `targetAspect` — what a `Cover` fit does on its own, + * and what the harness proposes in place of a sampled window. `null` when the + * footage is already that shape. + */ +export function centeredCrop(sourceWidth: number, sourceHeight: number, targetAspect: number): CropFrame | null { + if (!needsCrop(sourceWidth, sourceHeight, targetAspect)) return null; + const source = sourceWidth / Math.max(sourceHeight, 1); + const horizontal = source > targetAspect; + const keep = Math.min(1, Math.max(0.01, horizontal ? targetAspect / source : source / targetAspect)); + const edge = (1 - keep) / 2; + return horizontal + ? { left: edge, right: edge, top: 0, bottom: 0, offset: 0 } + : { left: 0, right: 0, top: edge, bottom: edge, offset: 0 }; +} diff --git a/frontend/src/lib/state.svelte.ts b/frontend/src/lib/state.svelte.ts index 71a02e7..f776a21 100644 --- a/frontend/src/lib/state.svelte.ts +++ b/frontend/src/lib/state.svelte.ts @@ -40,6 +40,7 @@ import { removeClip, removeSilence, snapToBeats, + smartCrop, removeTrack, setTrackDuck, setDeliveryFormat, @@ -665,6 +666,14 @@ class EditorState { snapToBeats(trackId?: string, tolerance?: number) { return this.#apply(snapToBeats(trackId, tolerance)); } + /** + * Frame each shot for the delivery frame instead of centring it blindly. + * One clip when `clipId` is given, otherwise every clip on an unlocked video + * track. Lands as one undoable `Smart crop` revision. + */ + smartCrop(clipId?: string) { + return this.#apply(smartCrop(clipId)); + } extractAudio(assetId: string) { return this.#apply(extractAudio(assetId)); } From 527d3b3fcb50cd06bd473e1403d65e7c1ad4aef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Thu, 27 Aug 2026 01:59:19 +0200 Subject: [PATCH 7/7] run cargo fmt The delivery last mile went in unformatted too, so this covers platform.rs and platform_check alongside the smart crop additions. --- crates/kerf-app/src/lib.rs | 6 ++++- crates/kerf-app/src/mcp.rs | 4 +--- crates/kerf-core/src/engine/cli.rs | 23 ++++++++++++++---- crates/kerf-core/src/engine/mod.rs | 8 +++---- crates/kerf-core/src/lib.rs | 14 +++++------ crates/kerf-core/src/platform.rs | 4 +--- crates/kerf-core/src/project.rs | 38 ++++++++++++++++-------------- 7 files changed, 56 insertions(+), 41 deletions(-) diff --git a/crates/kerf-app/src/lib.rs b/crates/kerf-app/src/lib.rs index 3d96763..c990235 100644 --- a/crates/kerf-app/src/lib.rs +++ b/crates/kerf-app/src/lib.rs @@ -1362,7 +1362,11 @@ fn platform_targets() -> Vec { /// How ready the current cut is for each target — what would be rejected, what /// would be accepted and then under-distributed, and what would just be better. #[tauri::command(async)] -fn platform_check(state: State<'_, AppState>, width: Option, height: Option) -> CmdResult> { +fn platform_check( + state: State<'_, AppState>, + width: Option, + height: Option, +) -> CmdResult> { // The export dialog can resize away from the project frame; when it does it // passes the frame it is actually about to render, so the verdict is about // the file that will exist rather than the one the project defaults to. diff --git a/crates/kerf-app/src/mcp.rs b/crates/kerf-app/src/mcp.rs index 04545b5..9af2553 100644 --- a/crates/kerf-app/src/mcp.rs +++ b/crates/kerf-app/src/mcp.rs @@ -164,9 +164,7 @@ struct TrackIdParams { #[derive(Debug, serde::Deserialize, schemars::JsonSchema)] struct SmartCropParams { - #[schemars( - description = "UUID of the clip to reframe; every clip on an unlocked video track when omitted" - )] + #[schemars(description = "UUID of the clip to reframe; every clip on an unlocked video track when omitted")] clip_id: Option, } diff --git a/crates/kerf-core/src/engine/cli.rs b/crates/kerf-core/src/engine/cli.rs index 0e3dc9d..96d08df 100644 --- a/crates/kerf-core/src/engine/cli.rs +++ b/crates/kerf-core/src/engine/cli.rs @@ -592,7 +592,10 @@ pub fn salience_map(path: &Path, start: f64, end: f64) -> Result { if let Some(hw) = hw { cmd.args(["-hwaccel", hw]); } - cmd.args(&args).stderr(Stdio::piped()).output().map_err(|e| launch_err(&bin, e)) + cmd.args(&args) + .stderr(Stdio::piped()) + .output() + .map_err(|e| launch_err(&bin, e)) }; let hw = decode_hwaccel(); let output = match hw.as_deref() { @@ -4426,7 +4429,12 @@ fn build_still_args( chains.push(format!("[{cur}]null[outv]")); let filter = chains.join(";"); - args.extend(["-filter_complex".to_string(), filter, "-map".to_string(), "[outv]".to_string()]); + args.extend([ + "-filter_complex".to_string(), + filter, + "-map".to_string(), + "[outv]".to_string(), + ]); args.extend(out.args()); Ok(args) } @@ -4604,7 +4612,6 @@ mod tests { use chrono::Utc; use uuid::Uuid; - // ---- salience sampling -------------------------------------------------- #[test] @@ -4613,7 +4620,10 @@ mod tests { let joined = args.join(" "); assert!(joined.contains("-ss 12.000 -t 10.000 -i /m/a.mp4"), "{joined}"); // 48 samples over 10s. - assert!(joined.contains("fps=4.8000,scale=64:36:flags=bilinear,format=gray"), "{joined}"); + assert!( + joined.contains("fps=4.8000,scale=64:36:flags=bilinear,format=gray"), + "{joined}" + ); assert!(joined.contains("-frames:v 48"), "{joined}"); assert!(joined.contains("-f rawvideo -pix_fmt gray pipe:1"), "{joined}"); assert!(joined.contains("-an"), "{joined}"); @@ -7384,7 +7394,10 @@ mod tests { let chain = video_clip_chain(&clip, &fmt, &ClipFx::default(), false, "c0"); let cropped = chain.find("crop=w=iw*").expect("the smart crop is in the graph"); let scaled = chain.find("scale=1080:1920").expect("the fit is in the graph"); - assert!(cropped < scaled, "the crop must pick the window before the fit scales it: {chain}"); + assert!( + cropped < scaled, + "the crop must pick the window before the fit scales it: {chain}" + ); // The window is off-centre — a plain Cover would have taken the middle. assert!(chain.contains(&format!("x=iw*{}", crop.left)), "{chain}"); assert!(crop.left < 0.3, "the subject is left of centre: {crop:?}"); diff --git a/crates/kerf-core/src/engine/mod.rs b/crates/kerf-core/src/engine/mod.rs index 7cddd4a..09888c8 100644 --- a/crates/kerf-core/src/engine/mod.rs +++ b/crates/kerf-core/src/engine/mod.rs @@ -39,10 +39,10 @@ mod ffmpeg; // Analysis, frame and waveform extraction always go through the CLI backend — // they only need the FFmpeg binaries, never the dev libraries. pub use cli::{ - audio_effects_filter, audio_pcm, contact_sheet, decode_hwaccel, delivery_frame, detect_scenes, detect_silence, export_still, frame_at, - frame_jpeg, generate_proxy, hw_encoders, insta360_pair, proxy_path, proxy_width, ready_proxy, salience_map, stitch_insta360, - stitched_path, stream_preview, timeline_frame, validate_export, waveform, Container, ExportOptions, ExportProgress, Fit, - ImageFormat, PreviewFrame, RateControl, RenderStatus, + audio_effects_filter, audio_pcm, contact_sheet, decode_hwaccel, delivery_frame, detect_scenes, detect_silence, export_still, + frame_at, frame_jpeg, generate_proxy, hw_encoders, insta360_pair, proxy_path, proxy_width, ready_proxy, salience_map, + stitch_insta360, stitched_path, stream_preview, timeline_frame, validate_export, waveform, Container, ExportOptions, + ExportProgress, Fit, ImageFormat, PreviewFrame, RateControl, RenderStatus, }; pub(crate) use cli::insta360_pair_name; diff --git a/crates/kerf-core/src/lib.rs b/crates/kerf-core/src/lib.rs index 26152e2..ea74dfb 100644 --- a/crates/kerf-core/src/lib.rs +++ b/crates/kerf-core/src/lib.rs @@ -28,15 +28,15 @@ pub use engine::{ SpeechModelInfo, DEFAULT_SPEECH_MODEL, }; pub use error::{Error, Result}; -pub use platform::{ - check_all as check_platforms, CutSummary, DeliveryCheck, DeliveryIssue, IssueKind, PlatformTarget, Severity, - TARGETS as PLATFORM_TARGETS, -}; pub use fonts::list_system_fonts; pub use model::{ Asset, AssetAnalysis, AudioEffect, Clip, Color, CropFrame, Delivery, DiffEntry, DiffKind, EditSource, Keyframe, Marker, - Projection, Reframe, ReframeKeyframe, ResolvedReframe, Revision, Rhythm, SalienceMap, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, - TextKeyframe, TextOverlay, TimeRange, Timeline, TimelineDiff, Track, TranscriptSegment, Transform, Transition, - TransitionKind, VideoEffect, + Projection, Reframe, ReframeKeyframe, ResolvedReframe, Revision, Rhythm, SalienceMap, StagedEdit, StreamInfo, StreamKind, + Task, TaskStatus, TextKeyframe, TextOverlay, TimeRange, Timeline, TimelineDiff, Track, TranscriptSegment, Transform, + Transition, TransitionKind, VideoEffect, +}; +pub use platform::{ + check_all as check_platforms, CutSummary, DeliveryCheck, DeliveryIssue, IssueKind, PlatformTarget, Severity, + TARGETS as PLATFORM_TARGETS, }; pub use project::{Project, SmartCropJob, SmartCropPlan}; diff --git a/crates/kerf-core/src/platform.rs b/crates/kerf-core/src/platform.rs index d5d8b35..42ce008 100644 --- a/crates/kerf-core/src/platform.rs +++ b/crates/kerf-core/src/platform.rs @@ -329,9 +329,7 @@ pub fn check_all(cut: &CutSummary) -> Vec { /// UI with the one the user already chose to cut for. pub fn target_for(format: Option<&Delivery>) -> Option<&'static PlatformTarget> { let f = format?; - TARGETS - .iter() - .find(|t| t.width == f.width && t.height == f.height) + TARGETS.iter().find(|t| t.width == f.width && t.height == f.height) } #[cfg(test)] diff --git a/crates/kerf-core/src/project.rs b/crates/kerf-core/src/project.rs index c5b5774..562f51f 100644 --- a/crates/kerf-core/src/project.rs +++ b/crates/kerf-core/src/project.rs @@ -14,8 +14,8 @@ use crate::error::{Error, Result}; use crate::model::default_beat_tolerance; use crate::model::{ Asset, AssetAnalysis, AudioEffect, Clip, CropFrame, Delivery, EditSource, Keyframe, Marker, Projection, Reframe, - ReframeKeyframe, Revision, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, Tempo, TextKeyframe, TextOverlay, TimeRange, Timeline, - TimelineDiff, Track, Transition, VideoEffect, MAX_FOV, MIN_FOV, + ReframeKeyframe, Revision, StagedEdit, StreamInfo, StreamKind, Task, TaskStatus, Tempo, TextKeyframe, TextOverlay, TimeRange, + Timeline, TimelineDiff, Track, Transition, VideoEffect, MAX_FOV, MIN_FOV, }; /// One clip queued for smart-crop sampling: which media to look at, over which @@ -658,12 +658,11 @@ impl Project { let (width, height) = frame.unwrap_or_else(|| engine::delivery_frame(&rendered, &assets)); // Audio-bearing means what the export means by it: any clip whose asset // carries an audio stream, on a video track as much as an audio one. - let has_audio = rendered.tracks.iter().flat_map(|t| t.clips.iter()).any(|c| { - assets - .iter() - .find(|a| a.id == c.asset_id) - .is_some_and(|a| a.has_audio()) - }); + let has_audio = rendered + .tracks + .iter() + .flat_map(|t| t.clips.iter()) + .any(|c| assets.iter().find(|a| a.id == c.asset_id).is_some_and(|a| a.has_audio())); Ok(crate::platform::CutSummary { duration: rendered.duration(), width, @@ -708,12 +707,7 @@ impl Project { } /// Render a cover frame for the current timeline at `time_secs`. - pub fn export_still( - &self, - time_secs: f64, - path: impl AsRef, - format: Option, - ) -> Result { + pub fn export_still(&self, time_secs: f64, path: impl AsRef, format: Option) -> Result { let (timeline, assets) = self.export_still_inputs()?; Self::render_still(&timeline, &assets, time_secs, path, format) } @@ -4240,7 +4234,9 @@ mod tests { #[test] fn smart_crop_plans_every_video_clip_for_a_vertical_delivery() { let project = Project::sample().unwrap(); - project.set_delivery_format(Some(Delivery::new(1080, 1920, Fit::Cover))).unwrap(); + project + .set_delivery_format(Some(Delivery::new(1080, 1920, Fit::Cover))) + .unwrap(); let plan = project.smart_crop_inputs(None).unwrap(); assert!((plan.target_aspect - 1080.0 / 1920.0).abs() < 1e-9); let video_clips: usize = project @@ -4264,7 +4260,9 @@ mod tests { #[test] fn smart_crop_can_be_narrowed_to_one_clip() { let project = Project::sample().unwrap(); - project.set_delivery_format(Some(Delivery::new(1080, 1920, Fit::Cover))).unwrap(); + project + .set_delivery_format(Some(Delivery::new(1080, 1920, Fit::Cover))) + .unwrap(); let clip = first_video_clip(&project); let plan = project.smart_crop_inputs(Some(clip)).unwrap(); assert_eq!(plan.jobs.len(), 1); @@ -4274,7 +4272,9 @@ mod tests { #[test] fn smart_crop_skips_a_locked_track_but_not_a_clip_named_on_one() { let project = Project::sample().unwrap(); - project.set_delivery_format(Some(Delivery::new(1080, 1920, Fit::Cover))).unwrap(); + project + .set_delivery_format(Some(Delivery::new(1080, 1920, Fit::Cover))) + .unwrap(); let clip = first_video_clip(&project); let track = project .timeline() @@ -4295,7 +4295,9 @@ mod tests { #[test] fn applying_crops_is_one_undoable_edit_that_only_counts_real_changes() { let project = Project::sample().unwrap(); - project.set_delivery_format(Some(Delivery::new(1080, 1920, Fit::Cover))).unwrap(); + project + .set_delivery_format(Some(Delivery::new(1080, 1920, Fit::Cover))) + .unwrap(); let clip = first_video_clip(&project); let crop = CropFrame { left: 0.1,