From 7cb6384d1e4457fe8da9fd768de9bac16b0ea5e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Wed, 2 Sep 2026 00:00:17 +0200 Subject: [PATCH 1/2] let an agent zoom into a frame get_frame and preview_timeline take a region (fractions of the frame) that is cropped before the scale to max_width, and skim_asset takes a cell that opens one sheet cell at full detail. A vision model spends the same image tokens on whatever it is handed, so a quarter of the frame at 640 px shows four times the detail of the whole frame for the same cost. A zoom reads the original rather than the proxy, at JPEG quality 2, and never upscales; a full region is the byte-identical plain decode. --- CLAUDE.md | 19 ++- crates/kerf-app/src/mcp.rs | 150 +++++++++++++++-- crates/kerf-core/src/engine/cli.rs | 251 +++++++++++++++++++++++++++-- crates/kerf-core/src/engine/mod.rs | 9 +- crates/kerf-core/src/lib.rs | 8 +- crates/kerf-core/src/project.rs | 32 ++++ 6 files changed, 436 insertions(+), 33 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 38513ca..e263fde 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -518,7 +518,24 @@ contact-sheet montage of an asset + a text index of cell→timestamp, for findin parts) and `preview_timeline` (the composited cut at a timeline time) — return `Result` built by the `image_result` helper: a caption `Content::text` plus a `Content::image(bare_base64, "image/jpeg")` block the LLM can -actually *see* (rmcp wants bare base64 + MIME, **not** a `data:` URL). The `lock()` +actually *see* (rmcp wants bare base64 + MIME, **not** a `data:` URL). +**Look, then look closer**: `get_frame` and `preview_timeline` take an optional +`region` (a `Region` — fractions of the frame, normalized into it) that is +cropped out *before* the scale to `max_width`, and `skim_asset` takes a `cell` +that opens one sheet cell as a full frame (`contact_sheet_times` recomputes the +cell's moment, so the sheet is never rebuilt). A vision model spends the same +image tokens on whatever it is handed, so a quarter of the frame at 640 px +shows four times the detail of the whole frame at 640 px — and beats a larger +`max_width`, which costs more and still loses small text. A zoom reads the +**original** source rather than the 1280 proxy (`decode_preview_region` — the +proxy threw away the pixels being asked for), at `ZOOM_QUALITY` 2 instead of +the preview's 4, and never upscales: the composite (`timeline_frame_region`) +renders a canvas wide enough for the region alone to be `max_width`, capped at +the delivery frame, then crops. A full region is the byte-identical plain +decode. The caption echoes the region back after normalization so the model's +next crop is in the coordinates that were actually used. There is deliberately +no general image-ops tool — a crop for inspection is how a frame is presented, +not an edit. The `lock()` helper sets `EditSource::Agent` per-op under the shared lock (the GUI's `project()` helper sets `User` the same way); every **mutating** tool goes through the `edit()` helper, which runs the op under the lock, **releases it**, and only then emits a diff --git a/crates/kerf-app/src/mcp.rs b/crates/kerf-app/src/mcp.rs index c740c64..1b9a6f9 100644 --- a/crates/kerf-app/src/mcp.rs +++ b/crates/kerf-app/src/mcp.rs @@ -17,7 +17,7 @@ use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; use base64::Engine as _; use kerf_core::{ AudioEffect, CaptionOptions, CaptionStyle, Delivery, EditSource, ExportOptions, Fit, Keyframe, Mask, MaskShape, Project, - Projection, ReframeKeyframe, StreamKind, TextKeyframe, Transition, TransitionKind, VideoEffect, + Projection, ReframeKeyframe, Region, StreamKind, TextKeyframe, Transition, TransitionKind, VideoEffect, }; use rmcp::handler::server::wrapper::Parameters; use rmcp::model::{CallToolResult, ContentBlock, Implementation, ProgressNotificationParam, ServerCapabilities, ServerInfo}; @@ -40,6 +40,10 @@ const DEFAULT_ADDR: &str = "127.0.0.1:7777"; /// base64 payload nobody benefits from, and a waveform of a million buckets is /// megabytes of JSON that would bury the answer it was fetched to support. const MAX_PREVIEW_WIDTH: u32 = 1920; +/// JPEG `-q:v` for a zoomed frame: a zoom exists to read fine detail, and the +/// preview's `4` smears text and edges that `2` keeps. The image is small +/// anyway, so the bytes hardly move. +const ZOOM_QUALITY: u8 = 2; const MAX_WAVEFORM_BUCKETS: usize = 4096; #[derive(Clone)] @@ -703,6 +707,49 @@ struct FrameParams { time_secs: f64, #[schemars(description = "Maximum output width in pixels (default 640, capped at 1920)")] max_width: Option, + #[schemars( + description = "Zoom into part of the frame instead of seeing all of it: a rectangle in fractions of the full frame (0..1), `left`/`top` its top-left corner, `width`/`height` its size. The region is cropped out first and then scaled to max_width, so a quarter of the frame shows four times the detail for the same image cost — use it to check a face, on-screen text, a caption, a mask edge. Omit for the whole frame." + )] + region: Option, +} + +/// A region of a frame to zoom into, as the schema hands it to a model: the +/// same fractions [`Region`] takes, kept as a separate type so the tool schema +/// documents each edge. +#[derive(Debug, Clone, Copy, serde::Deserialize, schemars::JsonSchema)] +struct RegionParams { + #[schemars(description = "Left edge of the region as a fraction of the frame width (0 = left edge)")] + left: f64, + #[schemars(description = "Top edge of the region as a fraction of the frame height (0 = top edge)")] + top: f64, + #[schemars(description = "Width of the region as a fraction of the frame width")] + width: f64, + #[schemars(description = "Height of the region as a fraction of the frame height")] + height: f64, +} + +impl RegionParams { + /// The engine region, pulled into the frame — a model's fractions are a + /// request, not a proof. + fn region(self) -> Region { + Region { + left: self.left, + top: self.top, + width: self.width, + height: self.height, + } + .normalized() + } + + /// The region as the caption echoes it back, after normalization, so the + /// model's next crop is in the coordinates that were actually used. + fn describe(self) -> String { + let r = self.region(); + format!( + "region left={:.3} top={:.3} width={:.3} height={:.3} of the full frame", + r.left, r.top, r.width, r.height + ) + } } #[derive(Debug, serde::Deserialize, schemars::JsonSchema)] @@ -719,6 +766,12 @@ struct SkimParams { rows: Option, #[schemars(description = "Width of each grid cell in pixels (default 240)")] cell_width: Option, + #[schemars( + description = "Zoom into one cell of the sheet instead of building it: the 1-based, row-major cell number from a previous skim with the same range and grid. Returns that cell's moment as a single full-detail frame (max_width wide, from the original source) — the shortcut from 'cell 7 looks promising' to seeing it properly." + )] + cell: Option, + #[schemars(description = "Width of the zoomed cell frame in pixels when `cell` is given (default 640, capped at 1920)")] + max_width: Option, } #[derive(Debug, serde::Deserialize, schemars::JsonSchema)] @@ -727,6 +780,10 @@ struct TimelineFrameParams { time_secs: f64, #[schemars(description = "Maximum output width in pixels (default 640, capped at 1920)")] max_width: Option, + #[schemars( + description = "Zoom into part of the frame instead of seeing all of it: a rectangle in fractions of the full frame (0..1), `left`/`top` its top-left corner, `width`/`height` its size. The region is cropped out first and then scaled to max_width, so a quarter of the frame shows four times the detail for the same image cost — use it to check a face, on-screen text, a caption, a mask edge. Omit for the whole frame." + )] + region: Option, } #[derive(Debug, serde::Deserialize, schemars::JsonSchema)] @@ -1901,21 +1958,29 @@ impl KerfMcp { let id = parse_id(&p.asset_id)?; let project = self.project.clone(); let (time_secs, max_width) = (p.time_secs, p.max_width.unwrap_or(640).clamp(64, MAX_PREVIEW_WIDTH)); + let region = p.region; let jpeg = blocking(move || { // Resolve under the lock, decode with it released — mirrors the GUI's // `get_frame` so an agent drill-in can't stall the user's edits. let asset = lock_agent(&project).require_asset(id).map_err(core_err)?; - Project::decode_preview_frame(&asset, time_secs, max_width, 4, true).map_err(core_err) + match region { + // A zoom is a request for detail, so it decodes the original at + // the best JPEG quality rather than the proxy at the preview one. + Some(r) => Project::decode_preview_region(&asset, time_secs, r.region(), max_width, ZOOM_QUALITY), + None => Project::decode_preview_frame(&asset, time_secs, max_width, 4, true), + } + .map_err(core_err) }) .await?; - Ok(image_result( - format!("asset {} @ {}", p.asset_id, fmt_ts(p.time_secs.max(0.0))), - jpeg, - )) + let mut caption = format!("asset {} @ {}", p.asset_id, fmt_ts(p.time_secs.max(0.0))); + if let Some(r) = region { + caption.push_str(&format!(", {}", r.describe())); + } + Ok(image_result(caption, jpeg)) } #[tool( - description = "Skim an asset: sample frames evenly across a time range (default the whole asset) into one contact-sheet image, plus a text index of which source timestamp each grid cell shows. The cheap way to survey footage and find the good parts; then call get_frame to inspect a promising moment, and add_clip_to_timeline / cut_clip to use it." + description = "Skim an asset: sample frames evenly across a time range (default the whole asset) into one contact-sheet image, plus a text index of which source timestamp each grid cell shows. The cheap way to survey footage and find the good parts; then pass `cell` (same range and grid) to see one cell at full detail, or call get_frame — with a `region` to zoom — to inspect a promising moment, and add_clip_to_timeline / cut_clip to use it." )] async fn skim_asset(&self, Parameters(p): Parameters) -> Result { let id = parse_id(&p.asset_id)?; @@ -1923,6 +1988,9 @@ impl KerfMcp { let rows = p.rows.unwrap_or(4).clamp(1, 8); let cell_width = p.cell_width.unwrap_or(240).clamp(80, 640); let project = self.project.clone(); + if let Some(cell) = p.cell { + return self.skim_cell(id, &p, columns, rows, cell).await; + } let (jpeg, times) = blocking(move || { // Resolve under the lock, sample the (columns × rows) frames with it // released — a contact sheet is many seeks and would otherwise freeze @@ -1941,23 +2009,71 @@ impl KerfMcp { Ok(image_result(caption, jpeg)) } + /// The `cell` form of [`Self::skim_asset`]: the moment one cell of a sheet + /// showed, decoded on its own at full detail. The sheet is never rebuilt — + /// the cell's timestamp is pure arithmetic over the same range and grid. + async fn skim_cell( + &self, + id: uuid::Uuid, + p: &SkimParams, + columns: u32, + rows: u32, + cell: u32, + ) -> Result { + let cells = columns * rows; + if cell < 1 || cell > cells { + return Err(McpError::invalid_params( + format!("cell must be 1..={cells} for a {columns}x{rows} sheet, got {cell}"), + None, + )); + } + let max_width = p.max_width.unwrap_or(640).clamp(64, MAX_PREVIEW_WIDTH); + let (start, end) = (p.start, p.end); + let project = self.project.clone(); + let (jpeg, time) = blocking(move || { + let asset = lock_agent(&project).require_asset(id).map_err(core_err)?; + // The same range defaults `decode_contact_sheet` applies, so the cell + // lands on the frame the sheet showed. + let start = start.unwrap_or(0.0).max(0.0); + let end = end.unwrap_or(asset.duration).min(asset.duration).max(start); + let time = kerf_core::contact_sheet_times(start, end, columns, rows)[(cell - 1) as usize]; + let jpeg = Project::decode_preview_region(&asset, time, Region::FULL, max_width, ZOOM_QUALITY).map_err(core_err)?; + Ok::<_, McpError>((jpeg, time)) + }) + .await?; + Ok(image_result( + format!( + "cell {cell} of {columns}x{rows} sheet, asset {} @ {}", + p.asset_id, + fmt_ts(time) + ), + jpeg, + )) + } + #[tool( - description = "Render the assembled timeline at a timeline time into one composite image the model can see — the actual cut on screen at that moment (footage layered in track order, picture-in-picture placement, crop, color; gaps render black). Use to verify an edit you just made. A moment inside a transition is not: dissolves, dips and slides render as the plain cut." + description = "Render the assembled timeline at a timeline time into one composite image the model can see — the actual cut on screen at that moment (footage layered in track order, picture-in-picture placement, crop, color; gaps render black). Use to verify an edit you just made; pass `region` to zoom into a detail of it. A moment inside a transition is not: dissolves, dips and slides render as the plain cut." )] async fn preview_timeline(&self, Parameters(p): Parameters) -> Result { let project = self.project.clone(); let (time_secs, max_width) = (p.time_secs, p.max_width.unwrap_or(640).clamp(64, MAX_PREVIEW_WIDTH)); + let region = p.region; let jpeg = blocking(move || { // Snapshot the inputs under the lock, composite with it released — // mirrors the GUI's `get_timeline_frame`. let (timeline, assets) = lock_agent(&project).timeline_frame_inputs().map_err(core_err)?; - Project::composite_timeline_frame(&timeline, &assets, time_secs, max_width, 4).map_err(core_err) + match region { + Some(r) => Project::composite_timeline_region(&timeline, &assets, time_secs, r.region(), max_width, ZOOM_QUALITY), + None => Project::composite_timeline_frame(&timeline, &assets, time_secs, max_width, 4), + } + .map_err(core_err) }) .await?; - Ok(image_result( - format!("timeline composite @ {}", fmt_ts(p.time_secs.max(0.0))), - jpeg, - )) + let mut caption = format!("timeline composite @ {}", fmt_ts(p.time_secs.max(0.0))); + if let Some(r) = region { + caption.push_str(&format!(", {}", r.describe())); + } + Ok(image_result(caption, jpeg)) } #[tool(description = "Summarise the timeline: total duration, track count, clips per track, and any per-track gaps")] @@ -2121,7 +2237,13 @@ impl ServerHandler for KerfMcp { You can also SEE the footage: skim_asset returns a contact-sheet \ image of a clip (survey it to find the good parts), get_frame shows \ a single moment up close, and preview_timeline renders the cut you \ - have assembled at a given time so you can check it on screen. \ + have assembled at a given time so you can check it on screen. Look, \ + then look closer: skim_asset with `cell` opens one sheet cell at \ + full detail, and get_frame / preview_timeline take a `region` \ + (fractions of the frame) that is cropped out and enlarged — the way \ + to read a face, on-screen text, a caption against the safe area or \ + a mask edge, since a whole frame at the same width shows a fraction \ + of that detail. Prefer a region to a larger max_width. \ Then assemble a non-destructive edit with the \ cut/split/trim/add/reorder/move_clip/remove/ripple_delete tools \ (move_clip frees a clip to any position or same-kind track; \ diff --git a/crates/kerf-core/src/engine/cli.rs b/crates/kerf-core/src/engine/cli.rs index ffa1b34..bc9ec8b 100644 --- a/crates/kerf-core/src/engine/cli.rs +++ b/crates/kerf-core/src/engine/cli.rs @@ -725,6 +725,76 @@ pub fn frame_at(path: &Path, time_secs: f64, max_width: u32) -> Result> decode_frame(path, time_secs, &scale, "png", None, true) } +/// A rectangle of a frame to look at more closely, as fractions of the full +/// frame: `left`/`top` place its corner, `width`/`height` size it. +/// +/// This is the *zoom* half of "look, then look closer": a vision model spends +/// its image budget on whatever it is handed, so a quarter of the frame cropped +/// out and scaled to the same width shows four times the detail of the whole +/// frame for the same cost — a face, a caption, a mask edge. Nothing here is an +/// edit; it is only how a frame is presented. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Region { + pub left: f64, + pub top: f64, + pub width: f64, + pub height: f64, +} + +impl Region { + /// The smallest side a region may have, as a fraction of the frame: below + /// this a crop of a 640 px preview is a handful of pixels. + pub const MIN_SIDE: f64 = 0.02; + + /// The whole frame, which is what "no region" means. + pub const FULL: Region = Region { + left: 0.0, + top: 0.0, + width: 1.0, + height: 1.0, + }; + + /// Clamp into the frame: sides at least [`Region::MIN_SIDE`], the corner + /// inside, and the far edge pulled back to 1.0 by shrinking the size, not + /// by moving the corner — the corner is what the caller pointed at. + pub fn normalized(self) -> Region { + let n = |v: f64| if v.is_finite() { v } else { 0.0 }; + let left = n(self.left).clamp(0.0, 1.0 - Self::MIN_SIDE); + let top = n(self.top).clamp(0.0, 1.0 - Self::MIN_SIDE); + let width = n(self.width).clamp(Self::MIN_SIDE, 1.0 - left); + let height = n(self.height).clamp(Self::MIN_SIDE, 1.0 - top); + Region { + left, + top, + width, + height, + } + } + + /// Whether this is (within rounding) the whole frame — then no crop is + /// added, so a caller passing the full region gets the byte-identical + /// invocation it always got. + pub fn is_full(&self) -> bool { + let r = self.normalized(); + r.left < 1e-6 && r.top < 1e-6 && r.width > 1.0 - 1e-6 && r.height > 1.0 - 1e-6 + } + + /// The `crop` filter for this region against whatever frame it is applied + /// to, in `iw`/`ih` terms so it needs no knowledge of the frame size. The + /// width and height are kept even: MJPEG is 4:2:0 and an odd crop would be + /// realigned by the filter to a size the scale after it cannot predict. + fn crop_filter(&self) -> String { + let r = self.normalized(); + format!( + "crop=2*trunc(iw*{w:.4}/2):2*trunc(ih*{h:.4}/2):iw*{l:.4}:ih*{t:.4}", + w = r.width, + h = r.height, + l = r.left, + t = r.top + ) + } +} + /// Decode a single frame at `time_secs` as **JPEG** bytes, scaled to at most /// `max_width` pixels wide, at `quality` (ffmpeg `-q:v`, 2 = best … 31 = worst). /// JPEG is dramatically smaller than the PNG of [`frame_at`], which matters when @@ -736,6 +806,35 @@ pub fn frame_jpeg(path: &Path, time_secs: f64, max_width: u32, quality: u8, accu decode_frame(path, time_secs, &scale, "mjpeg", Some(quality), accurate) } +/// [`frame_jpeg`] of one `region` of the frame: the region is cropped out of +/// the decoded frame *first* and then scaled to at most `max_width`, so the +/// width budget is spent on the region rather than on the whole picture. Never +/// upscaled past the source's own pixels — a zoom shows real detail or none. +/// A full region is exactly [`frame_jpeg`]. +pub fn frame_jpeg_region( + path: &Path, + time_secs: f64, + region: Region, + max_width: u32, + quality: u8, + accurate: bool, +) -> Result> { + if region.is_full() { + return frame_jpeg(path, time_secs, max_width, quality, accurate); + } + let vf = region_frame_filter(region, max_width); + decode_frame(path, time_secs, &vf, "mjpeg", Some(quality), accurate) +} + +/// Pure filter chain for [`frame_jpeg_region`] (unit-tested): crop, then scale +/// to an even width no wider than `max_width` or the crop itself. +fn region_frame_filter(region: Region, max_width: u32) -> String { + format!( + "{crop},scale='2*trunc(min({max_width},iw)/2)':-2", + crop = region.crop_filter() + ) +} + /// Seek to `time_secs`, run the `-vf` chain on a single frame and pipe it out in /// the given image codec (`png` / `mjpeg`); `quality`, when set, becomes `-q:v`. /// Shared by [`frame_at`] and [`frame_jpeg`]. `-ss` is input-side (fast). With @@ -867,6 +966,18 @@ pub fn contact_sheet( Ok((output.stdout, times)) } +/// The source timestamp each cell of a `columns`×`rows` [`contact_sheet`] over +/// `[start, end)` shows, row-major: the start of each of the equal slices the +/// window is cut into. Public so a caller holding a sheet can turn "cell 7" +/// back into a moment to look at more closely without re-sampling the sheet. +pub fn contact_sheet_times(start: f64, end: f64, columns: u32, rows: u32) -> Vec { + let cells = (columns.max(1) * rows.max(1)) as usize; + let start = start.max(0.0); + let window = (end - start).max(0.0); + let step = if window > 0.0 { window / cells as f64 } else { 0.0 }; + (0..cells).map(|k| start + step * k as f64).collect() +} + /// Pure arg builder for [`contact_sheet`] (no I/O, unit-tested): the ffmpeg /// argument list and the row-major per-cell timestamps. Frames are sampled at /// the start of each of `columns*rows` equal slices of the window via the `fps` @@ -885,8 +996,7 @@ fn build_contact_sheet_args( let cells = (columns * rows) as usize; let start = start.max(0.0); let window = (end - start).max(0.0); - let step = if window > 0.0 { window / cells as f64 } else { 0.0 }; - let times: Vec = (0..cells).map(|k| start + step * k as f64).collect(); + let times = contact_sheet_times(start, end, columns, rows); // `fps` = one frame per slice over the seeked window; `tile` packs them and // `-frames:v 1` emits the single sheet. A degenerate window falls back to 1. let rate = if window > 0.0 { cells as f64 / window } else { 1.0 }; @@ -4422,7 +4532,32 @@ pub fn timeline_frame( max_width: u32, quality: u8, ) -> Result> { - run_still(timeline, assets, opts, t, max_width, &StillOutput::JpegPipe { quality }) + run_still(timeline, assets, opts, t, max_width, None, &StillOutput::JpegPipe { quality }) +} + +/// [`timeline_frame`] of one `region` of the composited canvas. The canvas is +/// rendered large enough that the region alone comes out `max_width` wide +/// (capped at the delivery frame, so nothing is invented), then cropped — a +/// zoom into the cut rather than a screenshot of it. +pub fn timeline_frame_region( + timeline: &Timeline, + assets: &[Asset], + opts: &ExportOptions, + t: f64, + region: Region, + max_width: u32, + quality: u8, +) -> Result> { + let region = (!region.is_full()).then_some(region); + run_still( + timeline, + assets, + opts, + t, + max_width, + region, + &StillOutput::JpegPipe { quality }, + ) } /// Write the composited still at timeline time `t` to `path` as a **cover @@ -4451,7 +4586,7 @@ pub fn export_still( }; // `u32::MAX` asks for no cap: `build_still_args` clamps to the delivery // width, which is exactly the cover size. - run_still(timeline, assets, opts, t, u32::MAX, &out)?; + run_still(timeline, assets, opts, t, u32::MAX, None, &out)?; Ok(path.to_path_buf()) } @@ -4463,11 +4598,12 @@ fn run_still( opts: &ExportOptions, t: f64, max_width: u32, + region: Option, out: &StillOutput, ) -> Result> { let piping = matches!(out, StillOutput::JpegPipe { .. }); let run = |o: &ExportOptions| -> Result> { - let mut args = build_still_args(timeline, assets, o, t, max_width, out)?; + let mut args = build_still_args(timeline, assets, o, t, max_width, region, out)?; cpu::limit_args(&mut args, cpu::budget_threads()); let bin = ffmpeg_bin(); let output = command(&bin) @@ -4516,7 +4652,7 @@ fn build_timeline_frame_args( max_width: u32, quality: u8, ) -> Result> { - build_still_args(timeline, assets, opts, t, max_width, &StillOutput::JpegPipe { quality }) + build_still_args(timeline, assets, opts, t, max_width, None, &StillOutput::JpegPipe { quality }) } /// Where a composited still is written, and in what image format. @@ -4618,6 +4754,7 @@ fn build_still_args( opts: &ExportOptions, t: f64, max_width: u32, + region: Option, out: &StillOutput, ) -> Result> { // Same gate as the export, so the still shows the cut that would render. @@ -4625,8 +4762,15 @@ fn build_still_args( let timeline = &rendered; let fmt = export_format(timeline, assets, opts); + // A region zoom composites a canvas wide enough that the region alone is + // `max_width` — the delivery frame caps it, so a zoom never upscales. + let region = region.map(Region::normalized); + let canvas_width = match region { + Some(r) => ((max_width as f64 / r.width).ceil() as u32).min(fmt.width), + None => max_width, + }; // Output canvas: export aspect ratio, capped to `max_width`, even dimensions. - let ow = (max_width.min(fmt.width).max(2)) & !1; + let ow = (canvas_width.min(fmt.width).max(2)) & !1; let oh = ((((ow as u64) * (fmt.height as u64)) / (fmt.width.max(1) as u64)) as u32).max(2) & !1; let t = t.max(0.0); let asset_of = |id| assets.iter().find(|a: &&Asset| a.id == id); @@ -4708,7 +4852,10 @@ fn build_still_args( chains.push(format!("[{cur}]{f}[{out}]", f = drawtext_still(ov, oh, t))); cur = out; } - chains.push(format!("[{cur}]null[outv]")); + match region { + Some(r) => chains.push(format!("[{cur}]{crop}[outv]", crop = r.crop_filter())), + None => chains.push(format!("[{cur}]null[outv]")), + } let filter = chains.join(";"); args.extend([ @@ -5197,7 +5344,7 @@ mod tests { format: ImageFormat::Jpeg, quality: 2, }; - let args = build_still_args(&timeline, &assets, &ExportOptions::default(), 1.0, u32::MAX, &out).unwrap(); + let args = build_still_args(&timeline, &assets, &ExportOptions::default(), 1.0, u32::MAX, None, &out).unwrap(); // The uncapped width resolves to the project's delivery frame, not to a // preview size — a cover is a delivered image. let graph = args[args.iter().position(|a| a == "-filter_complex").unwrap() + 1].clone(); @@ -5223,7 +5370,7 @@ mod tests { format: ImageFormat::Png, quality: 2, }; - let args = build_still_args(&timeline, &assets, &ExportOptions::default(), 1.0, u32::MAX, &out).unwrap(); + let args = build_still_args(&timeline, &assets, &ExportOptions::default(), 1.0, u32::MAX, None, &out).unwrap(); assert!(args.windows(2).any(|w| w[0] == "-vcodec" && w[1] == "png")); assert!(!args.contains(&"-q:v".to_string()), "-q:v is meaningless for png: {args:?}"); } @@ -5955,6 +6102,90 @@ mod tests { assert!(joined.ends_with("pipe:1")); } + /// A region zoom crops before it scales, so the width budget lands on the + /// region; the whole frame is left byte-identical to the plain decode. + #[test] + fn region_frame_crops_then_scales_and_a_full_region_is_no_crop() { + let r = Region { + left: 0.25, + top: 0.1, + width: 0.5, + height: 0.3, + }; + let vf = region_frame_filter(r, 640); + assert_eq!( + vf, + "crop=2*trunc(iw*0.5000/2):2*trunc(ih*0.3000/2):iw*0.2500:ih*0.1000,scale='2*trunc(min(640,iw)/2)':-2" + ); + assert!(Region::FULL.is_full()); + assert!(!r.is_full()); + } + + /// Whatever the model asks for is pulled into the frame: a corner past the + /// far edge comes back in, and a region hanging over the edge is shrunk + /// rather than moved — the corner is the thing that was pointed at. + #[test] + fn region_normalizes_into_the_frame() { + let r = Region { + left: 0.8, + top: -0.5, + width: 0.6, + height: f64::NAN, + } + .normalized(); + assert!((r.left - 0.8).abs() < 1e-9); + assert_eq!(r.top, 0.0); + assert!((r.width - 0.2).abs() < 1e-9, "shrunk to the edge: {r:?}"); + assert_eq!(r.height, Region::MIN_SIDE); + let far = Region { + left: 2.0, + top: 0.0, + width: 1.0, + height: 1.0, + } + .normalized(); + assert!((far.left - (1.0 - Region::MIN_SIDE)).abs() < 1e-9); + assert!((far.width - Region::MIN_SIDE).abs() < 1e-9); + } + + /// Zooming the composite renders a canvas large enough for the region to + /// fill the requested width, then crops it — never past the delivery frame. + #[test] + fn timeline_region_widens_the_canvas_and_crops_the_composite() { + let asset = test_asset(vec![video_stream(1920, 1080, 30.0)]); + let assets = vec![asset.clone()]; + let timeline = single(vec![make_clip(asset.id, 0.0, 5.0, 0.0)]); + let r = Region { + left: 0.5, + top: 0.5, + width: 0.25, + height: 0.25, + }; + let out = StillOutput::JpegPipe { quality: 2 }; + let args = build_still_args(&timeline, &assets, &ExportOptions::default(), 1.0, 320, Some(r), &out).unwrap(); + let graph = args[args.iter().position(|a| a == "-filter_complex").unwrap() + 1].clone(); + // 320 / 0.25 = 1280 canvas, so the crop comes out 320 wide. + assert!(graph.contains("s=1280x720"), "{graph}"); + assert!( + graph.ends_with("crop=2*trunc(iw*0.2500/2):2*trunc(ih*0.2500/2):iw*0.5000:ih*0.5000[outv]"), + "{graph}" + ); + // Capped at the delivery frame: a zoom invents no pixels. + let args = build_still_args(&timeline, &assets, &ExportOptions::default(), 1.0, 1920, Some(r), &out).unwrap(); + let graph = args[args.iter().position(|a| a == "-filter_complex").unwrap() + 1].clone(); + assert!(graph.contains("s=1920x1080"), "{graph}"); + // Without a region the graph is what it always was. + let plain = build_still_args(&timeline, &assets, &ExportOptions::default(), 1.0, 320, None, &out).unwrap(); + let graph = plain[plain.iter().position(|a| a == "-filter_complex").unwrap() + 1].clone(); + assert!(graph.contains("s=320x180") && graph.ends_with("null[outv]"), "{graph}"); + } + + #[test] + fn contact_sheet_times_match_the_sheet() { + let (_, times) = build_contact_sheet_args("/x.mp4", 10.0, 20.0, 2, 2, 160, 3); + assert_eq!(times, contact_sheet_times(10.0, 20.0, 2, 2)); + } + #[test] fn contact_sheet_respects_a_subrange() { let (args, times) = build_contact_sheet_args("/x.mp4", 10.0, 20.0, 2, 2, 160, 3); diff --git a/crates/kerf-core/src/engine/mod.rs b/crates/kerf-core/src/engine/mod.rs index cf77298..3664818 100644 --- a/crates/kerf-core/src/engine/mod.rs +++ b/crates/kerf-core/src/engine/mod.rs @@ -43,10 +43,11 @@ mod ffmpeg; // Analysis, frame and waveform extraction always go through the CLI backend — // they only need the FFmpeg binaries, never the dev libraries. pub use cli::{ - audio_effects_filter, audio_pcm, contact_sheet, decode_hwaccel, delivery_frame, detect_scenes, detect_silence, export_still, - frame_at, frame_jpeg, generate_proxy, hw_encoders, insta360_pair, proxy_path, proxy_width, ready_proxy, salience_map, - stitch_insta360, stitched_path, stream_preview, timeline_frame, validate_export, waveform, Container, ExportOptions, - ExportProgress, Fit, ImageFormat, PreviewFrame, RateControl, RenderStatus, + audio_effects_filter, audio_pcm, contact_sheet, contact_sheet_times, decode_hwaccel, delivery_frame, detect_scenes, + detect_silence, export_still, frame_at, frame_jpeg, frame_jpeg_region, generate_proxy, hw_encoders, insta360_pair, + proxy_path, proxy_width, ready_proxy, salience_map, stitch_insta360, stitched_path, stream_preview, timeline_frame, + timeline_frame_region, validate_export, waveform, Container, ExportOptions, ExportProgress, Fit, ImageFormat, PreviewFrame, + RateControl, Region, RenderStatus, }; pub(crate) use cli::insta360_pair_name; diff --git a/crates/kerf-core/src/lib.rs b/crates/kerf-core/src/lib.rs index d8bc708..7e2b6e8 100644 --- a/crates/kerf-core/src/lib.rs +++ b/crates/kerf-core/src/lib.rs @@ -26,10 +26,10 @@ pub use engine::cpu::{ budget_threads as cpu_threads, cores as cpu_cores, cpu_percent, set_cpu_percent, DEFAULT_CPU_PERCENT, MIN_CPU_PERCENT, }; pub use engine::{ - download_speech_model, export_still, generate_proxy, hw_encoders, insta360_pair, proxy_path, proxy_width, render_with, - render_with_progress, set_speech_model, speech_model_names, stitch_insta360, stitched_path, stream_preview, validate_export, - Container, DownloadProgress, ExportOptions, ExportProgress, Fit, ImageFormat, PreviewFrame, RateControl, RenderStatus, - SpeechModelInfo, DEFAULT_SPEECH_MODEL, + contact_sheet_times, download_speech_model, export_still, generate_proxy, hw_encoders, insta360_pair, proxy_path, + proxy_width, render_with, render_with_progress, set_speech_model, speech_model_names, stitch_insta360, stitched_path, + stream_preview, validate_export, Container, DownloadProgress, ExportOptions, ExportProgress, Fit, ImageFormat, PreviewFrame, + RateControl, Region, RenderStatus, SpeechModelInfo, DEFAULT_SPEECH_MODEL, }; pub use error::{Error, Result}; pub use fonts::list_system_fonts; diff --git a/crates/kerf-core/src/project.rs b/crates/kerf-core/src/project.rs index 640d5a4..ad520da 100644 --- a/crates/kerf-core/src/project.rs +++ b/crates/kerf-core/src/project.rs @@ -513,6 +513,20 @@ impl Project { /// before the (potentially slow) ffmpeg decode runs, instead of freezing /// every other project op for its duration. `accurate = false` snaps to the /// nearest keyframe for fast scrubbing; a still decodes its one frame at t=0. + /// [`Project::decode_preview_frame`] zoomed into `region` (fractions of + /// the frame). Reads the **original** source rather than the 1280-wide + /// proxy: a zoom is a request for the pixels the proxy threw away. + pub fn decode_preview_region( + asset: &Asset, + time_secs: f64, + region: engine::Region, + max_width: u32, + quality: u8, + ) -> Result> { + let time_secs = if asset.is_image() { 0.0 } else { time_secs }; + engine::frame_jpeg_region(Path::new(&asset.path), time_secs, region, max_width, quality, true) + } + pub fn decode_preview_frame(asset: &Asset, time_secs: f64, max_width: u32, quality: u8, accurate: bool) -> Result> { // A still image has one frame at t=0; seeking past it decodes nothing. let time_secs = if asset.is_image() { 0.0 } else { time_secs }; @@ -637,6 +651,24 @@ impl Project { engine::timeline_frame(timeline, assets, &opts, time_secs, max_width, quality) } + /// [`Project::composite_timeline_frame`] zoomed into `region` of the + /// composited canvas — the same lock-free shape, for an agent checking a + /// detail of the cut (a caption against the safe area, a mask edge). + pub fn composite_timeline_region( + timeline: &Timeline, + assets: &[Asset], + time_secs: f64, + region: engine::Region, + max_width: u32, + quality: u8, + ) -> Result> { + let opts = engine::ExportOptions { + hwaccel: engine::decode_hwaccel(), + ..engine::ExportOptions::default() + }; + engine::timeline_frame_region(timeline, assets, &opts, time_secs, region, max_width, quality) + } + /// How ready the current cut is for each publishing target — what a platform /// would reject, what it would accept and then quietly under-distribute, and /// what would simply be better. From b20b337b99f6bb14c4acac771e1abeef20632722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Wed, 2 Sep 2026 00:04:02 +0200 Subject: [PATCH 2/2] release 0.20.1 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- crates/kerf-app/tauri.conf.json | 2 +- frontend/package.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 61152d9..6f95a6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2193,7 +2193,7 @@ dependencies = [ [[package]] name = "kerf-app" -version = "0.20.0" +version = "0.20.1" dependencies = [ "anyhow", "axum", @@ -2218,7 +2218,7 @@ dependencies = [ [[package]] name = "kerf-core" -version = "0.20.0" +version = "0.20.1" dependencies = [ "chrono", "dirs", diff --git a/Cargo.toml b/Cargo.toml index 151cdde..811fa06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/kerf-core", "crates/kerf-app"] [workspace.package] -version = "0.20.0" +version = "0.20.1" edition = "2021" rust-version = "1.95" license = "PolyForm-Noncommercial-1.0.0" diff --git a/crates/kerf-app/tauri.conf.json b/crates/kerf-app/tauri.conf.json index d9ee475..9c2e428 100644 --- a/crates/kerf-app/tauri.conf.json +++ b/crates/kerf-app/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Kerf", - "version": "0.20.0", + "version": "0.20.1", "identifier": "ch.orellbuehler.kerf", "build": { "frontendDist": "../../frontend/build", diff --git a/frontend/package.json b/frontend/package.json index af97fd7..5a02648 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "0.20.0", + "version": "0.20.1", "type": "module", "scripts": { "dev": "vite dev",