diff --git a/app/Cargo.toml b/app/Cargo.toml index c10c899201c..09583998a11 100644 --- a/app/Cargo.toml +++ b/app/Cargo.toml @@ -461,6 +461,7 @@ default = [ "agent_mode_pre_plan_xml", "command_correction_key", "kitty_images", + "iterm_images", "grep_tool", "validate_autosuggestions", "clear_autosuggestion_on_escape", diff --git a/app/build.rs b/app/build.rs index e526f16e1ff..7cfa04f3857 100644 --- a/app/build.rs +++ b/app/build.rs @@ -30,7 +30,7 @@ fn main() -> Result<()> { let target_family = env::var("CARGO_CFG_TARGET_FAMILY")?; let target_env = env::var("CARGO_CFG_TARGET_ENV")?; - add_features(&target_family, &target_os); + add_features(&target_family); if target_os == "macos" && target_family != "wasm" { println!("cargo:rustc-link-lib=framework=MetalKit"); @@ -233,16 +233,12 @@ fn get_build_profile_name() -> String { .to_string() } -fn add_features(target_family: &str, target_os: &str) { +fn add_features(target_family: &str) { if target_family != "wasm" { println!("cargo:rustc-cfg=feature=\"local_fs\""); println!("cargo:rustc-cfg=feature=\"local_tty\""); } - if target_os != "windows" { - println!("cargo:rustc-cfg=feature=\"iterm_images\""); - } - if env::var("PROFILE").ok().is_some_and(|val| val == "debug") { println!("cargo:rustc-cfg=feature=\"agent_mode_debug\""); } diff --git a/app/src/lib.rs b/app/src/lib.rs index 7e97606daaf..ea9f0229cc5 100644 --- a/app/src/lib.rs +++ b/app/src/lib.rs @@ -2363,7 +2363,7 @@ pub fn enabled_features() -> HashSet { FeatureFlag::ImeMarkedText, #[cfg(feature = "partial_next_command_suggestions")] FeatureFlag::PartialNextCommandSuggestions, - #[cfg(feature = "iterm_images")] + #[cfg(all(not(windows), feature = "iterm_images"))] FeatureFlag::ITermImages, #[cfg(feature = "validate_autosuggestions")] FeatureFlag::ValidateAutosuggestions, diff --git a/app/src/terminal/event.rs b/app/src/terminal/event.rs index d39515a2f1f..bfe6a5f49c8 100644 --- a/app/src/terminal/event.rs +++ b/app/src/terminal/event.rs @@ -138,6 +138,13 @@ pub enum Event { image_data: Vec, image_protocol: ImageProtocol, }, + /// The animation frames of a kitty image changed (`a=f`/`a=a`). The frames + /// follow frame 1, which is the already-received image itself; an empty list + /// means the animation is stopped. + AnimatedImageReceived { + image_id: u32, + frames: Vec<(Vec, u32)>, + }, BootstrapPrecmdDone, /// A pluggable notification triggered via OSC 9 or OSC 777 escape sequences. /// External programs can use this to trigger notifications in Zap. @@ -480,6 +487,13 @@ impl Debug for Event { Event::ImageReceived { image_id, .. } => { write!(f, "ImageReceived(image_id: {image_id})") } + Event::AnimatedImageReceived { image_id, frames } => { + write!( + f, + "AnimatedImageReceived(image_id: {image_id}, frames: {})", + frames.len() + ) + } Event::BootstrapPrecmdDone => write!(f, "BootstrapPrecmdDone"), Event::PluggableNotification { title, body } => { write!(f, "PluggableNotification(title: {title:?}, body: {body})") diff --git a/app/src/terminal/grid_renderer.rs b/app/src/terminal/grid_renderer.rs index 9e236d05f02..a275c4d52a4 100644 --- a/app/src/terminal/grid_renderer.rs +++ b/app/src/terminal/grid_renderer.rs @@ -1,5 +1,6 @@ mod cell_glyph_cache; mod cell_type; +mod unicode_placeholder; use crate::appearance::Appearance; use crate::terminal::grid_size_util::calculate_grid_baseline_position; @@ -18,8 +19,11 @@ use crate::util::color::{ContrastingColor, MinimumAllowedContrast}; use core::mem; use lazy_static::lazy_static; use num_traits::Float as _; +use smallvec::SmallVec; use std::cmp::Ordering; use std::ops::Range; +use std::sync::{Arc, LazyLock}; +use std::time::{Duration, Instant}; use std::{collections::HashMap, ops::RangeInclusive}; use unicode_width::UnicodeWidthChar; use warp_core::features::FeatureFlag; @@ -29,7 +33,9 @@ use warpui::elements::{Border, CornerRadius, Fill, Radius, DEFAULT_UI_LINE_HEIGH use warpui::fonts::{Cache as FontCache, FamilyId, FontId, Properties, Style, Weight}; use warpui::geometry::rect::RectF; use warpui::geometry::vector::{vec2f, Vector2F}; -use warpui::image_cache::{AnimatedImageBehavior, CacheOption, FitType, Image, ImageCache}; +use warpui::image_cache::{ + AnimatedImageBehavior, CacheOption, FitType, Image, ImageCache, StaticImage, +}; use warpui::platform::LineStyle; use warpui::text_layout::{Line, StyleAndFont, TextStyle, DEFAULT_TOP_BOTTOM_RATIO}; use warpui::units::{IntoLines as _, Lines, Pixels}; @@ -37,6 +43,9 @@ use warpui::{AppContext, Element, EntityId, PaintContext, Scene, SingletonEntity pub use self::cell_glyph_cache::CellGlyphCache; use self::cell_type::{CellType, IsFocused, Secret}; +use self::unicode_placeholder::{ + build_runs, is_unicode_placeholder, parse_placeholder_cell, PlaceholderRun, +}; use super::block_filter::{BLOCK_FILTER_DOTTED_LINE_DASH, BLOCK_FILTER_DOTTED_LINE_WIDTH}; use super::blockgrid_renderer::GridRenderParams; @@ -47,6 +56,10 @@ use super::model::image_map::{ImagePlacementData, StoredImageMetadata}; use super::model::terminal_model::RangeInModel; use crate::settings::EnforceMinimumContrast; +/// Rows of a single grid render held without allocating. Sized for a typical +/// viewport; taller ones spill to the heap. +const VISIBLE_ROWS_INLINE_CAPACITY: usize = 64; + // The scale factor of the cursor relative to the cursor width. const CURSOR_THICKNESS_SCALE_FACTOR: f32 = 0.15; @@ -455,6 +468,12 @@ pub fn render_grid<'a>( } } +/// Whether inline terminal images should be rendered. Both the iTerm and the kitty +/// graphics protocols feed the same image map, so either flag enables rendering. +fn terminal_images_enabled() -> bool { + FeatureFlag::ITermImages.is_enabled() || FeatureFlag::KittyImages.is_enabled() +} + // TODO (kevin): Solidify highlighted_url and link_tool_tip into one single struct. #[allow(clippy::too_many_arguments)] fn render_grid_without_ligatures<'a>( @@ -551,7 +570,12 @@ fn render_grid_without_ligatures<'a>( .flat_map(|marked_text| marked_text.chars()); let mut next_marked_text_cell_is_wide_char_spacer = false; - if FeatureFlag::ITermImages.is_enabled() { + // Materialized because the unicode placeholder pass walks the same rows + // before the main loop does. Inline capacity covers a normal viewport, so + // this doesn't allocate per frame. + let visible_rows: SmallVec<[usize; VISIBLE_ROWS_INLINE_CAPACITY]> = visible_rows.collect(); + + if terminal_images_enabled() { let image_ids = grid.get_image_ids_in_range(start_row, end_row); let (background_image_ids, foreground_image_ids): (Vec<_>, Vec<_>) = image_ids @@ -613,9 +637,22 @@ fn render_grid_without_ligatures<'a>( } ctx.scene.stop_layer(); } + + // Virtual placements are anchored by their placeholder cells rather than + // by the image map, so they need their own pass over the visible cells. + render_unicode_placeholders( + grid, + &visible_rows, + start_row, + cell_size, + grid_origin, + image_metadata, + ctx, + app, + ); } - for (offset, row_idx) in visible_rows.enumerate() { + for (offset, row_idx) in visible_rows.iter().copied().enumerate() { let offset_row = start_row + offset; let Some(row) = grid.row(row_idx) else { @@ -1055,7 +1092,11 @@ fn render_grid_with_ligatures<'a>( .flat_map(|marked_text| marked_text.chars()) .peekable(); let mut next_marked_text_cell_is_wide_char_spacer = false; - if FeatureFlag::ITermImages.is_enabled() { + + // See the matching comment in `render_grid_without_ligatures`. + let visible_rows: SmallVec<[usize; VISIBLE_ROWS_INLINE_CAPACITY]> = visible_rows.collect(); + + if terminal_images_enabled() { let image_ids = grid.get_image_ids_in_range(start_row, end_row); let (background_image_ids, foreground_image_ids): (Vec<_>, Vec<_>) = image_ids @@ -1117,9 +1158,22 @@ fn render_grid_with_ligatures<'a>( } ctx.scene.stop_layer(); } + + // Virtual placements are anchored by their placeholder cells rather than + // by the image map, so they need their own pass over the visible cells. + render_unicode_placeholders( + grid, + &visible_rows, + start_row, + cell_size, + grid_origin, + image_metadata, + ctx, + app, + ); } - for (offset, row_idx) in visible_rows.enumerate() { + for (offset, row_idx) in visible_rows.iter().copied().enumerate() { let offset_row = start_row + offset; let mut string_builder = AttributedStringBuilder::new( font_family, @@ -1413,6 +1467,11 @@ fn render_grid_with_ligatures<'a>( ctx, ) { string_builder.append_content(secret_content, col); + } else if is_unicode_placeholder(cell) { + // Drawn as an image by `render_unicode_placeholders`. The + // cell is still reserved so columns stay aligned, and its + // grapheme never reaches the text layout cache. + string_builder.append_placeholder(col); } else if let Some(glyph_type) = native_glyph_for_cell(cell) { native_glyphs_to_render.push(NativeGlyph { cell_bounds: RectF::new(grid_origin + glyph_offset, actual_cell_size), @@ -1693,6 +1752,14 @@ fn render_cell_glyph( fallback_font_family: Option, ctx: &mut PaintContext, ) { + // Kitty unicode placeholders are drawn as images by + // `render_unicode_placeholders`. Returning before the glyph lookup also + // keeps their grapheme strings, which are unique per image cell, out of the + // `CellGlyphCache`. + if is_unicode_placeholder(cell) { + return; + } + let cell_size = if cell.flags().intersects(Flags::WIDE_CHAR) { // WIDE_CHAR takes up two cells. Vector2F::new(cell_size.x() * 2., cell_size.y()) @@ -1832,6 +1899,216 @@ fn render_glyph_svg( } } +/// Reference point for animated inline image playback. All animated images share this +/// phase, so a GIF starts on whichever frame is current when it is first drawn rather +/// than on frame zero. +static ANIMATION_EPOCH: LazyLock = LazyLock::new(Instant::now); + +/// Draws the images that kitty unicode placeholder cells (`U=1`) stand in for. +/// +/// Virtual placements are not in the image map — they have no anchor in the grid +/// — so this scans the visible cells instead. Each horizontal strip of an image +/// becomes a single quad that samples just the part of the image its cells +/// represent. +#[allow(clippy::too_many_arguments)] +fn render_unicode_placeholders( + grid: &GridHandler, + visible_rows: &[usize], + start_row: usize, + cell_size: Vector2F, + grid_origin: Vector2F, + image_metadata: &HashMap, + ctx: &mut PaintContext, + app: &AppContext, +) { + // The scan below touches every visible cell, so skip it (and its layer) + // outright unless some image actually has a virtual placement. + if !image_metadata + .values() + .any(|m| matches!(m, StoredImageMetadata::Kitty(k) if !k.virtual_placements.is_empty())) + { + return; + } + + ctx.scene.start_layer(warpui::ClipBounds::ActiveLayer); + + let mut placeholder_cells = Vec::new(); + + for (offset, row_idx) in visible_rows.iter().copied().enumerate() { + let Some(row) = grid.row(row_idx) else { + continue; + }; + + placeholder_cells.clear(); + placeholder_cells.extend((0..grid.columns()).filter_map(|col| { + parse_placeholder_cell(&row[col]).map(|placeholder| (col, placeholder)) + })); + + if placeholder_cells.is_empty() { + continue; + } + + let offset_row = start_row + offset; + for run in build_runs(placeholder_cells.iter().copied()) { + render_placeholder_run( + &run, + offset_row, + cell_size, + grid_origin, + image_metadata, + ctx, + app, + ); + } + } + + ctx.scene.stop_layer(); +} + +/// Matches `MAX_IMAGE_CELL_HEIGHT` in the anchored placement path. +const MAX_VIRTUAL_PLACEMENT_CELLS: u32 = 255; + +/// Draws one horizontal strip of a virtually placed image. +#[allow(clippy::too_many_arguments)] +fn render_placeholder_run( + run: &PlaceholderRun, + offset_row: usize, + cell_size: Vector2F, + grid_origin: Vector2F, + image_metadata: &HashMap, + ctx: &mut PaintContext, + app: &AppContext, +) { + let Some(StoredImageMetadata::Kitty(metadata)) = image_metadata.get(&run.image_id) else { + return; + }; + + // Placement ids cannot be recovered from the cells, so `run.placement_id` is + // always 0 (see `parse_placeholder_cell`). When the keyed lookup misses, + // degrade to the lowest-numbered placement so an image shown through + // several virtual placements still renders one of them. + let placement = metadata + .virtual_placements + .get(&run.placement_id) + .or_else(|| { + metadata + .virtual_placements + .iter() + .min_by_key(|(id, _)| **id) + .map(|(_, placement)| placement) + }); + let Some(placement) = placement else { + return; + }; + + // The placement's extent in cells is resolved here rather than when the + // placement was created, so that a font size change re-tiles the image + // against the new cell size. + // Clamped like the anchored placement path: `r=`/`c=` are client-supplied, + // and unclamped values would drive an arbitrarily large resize (and a cache + // entry per distinct size) in `ImageCache::image` below. + let cols = placement + .cols + .unwrap_or_else(|| cells_to_cover(metadata.image_size.x(), cell_size.x())) + .min(MAX_VIRTUAL_PLACEMENT_CELLS); + let rows = placement + .rows + .unwrap_or_else(|| cells_to_cover(metadata.image_size.y(), cell_size.y())) + .min(MAX_VIRTUAL_PLACEMENT_CELLS); + + if rows == 0 || cols == 0 || run.image_row >= rows || run.image_col_start >= cols { + return; + } + + // Cells are free to reference columns past the right edge of the placement. + let image_col_end = run.image_col_end.min(cols - 1); + let covered_cols = image_col_end - run.image_col_start + 1; + + let placement_size = cell_size * vec2f(cols as f32, rows as f32); + let bounds = (placement_size * ctx.scene.scale_factor()).to_i32(); + + let asset_cache = AssetCache::as_ref(app); + let image = ImageCache::as_ref(app).image( + AssetSource::Raw { + id: run.image_id.to_string(), + }, + bounds, + // `rows`/`cols` already define the box the image is tiled over. + FitType::Stretch, + AnimatedImageBehavior::FullAnimation, + CacheOption::BySize, + ctx.max_texture_dimension_2d, + asset_cache, + ); + + let image = match image { + AssetState::Loaded { data } => data, + AssetState::Evicted => return, + _ => { + log::warn!( + "Could not load image to render (image id = {})", + run.image_id + ); + return; + } + }; + + let Some(frame) = current_image_frame(image.as_ref(), run.image_id, ctx) else { + return; + }; + + let origin = grid_origin + cell_size * vec2f(run.col_start as f32, offset_row as f32); + let destination = RectF::new(origin, cell_size * vec2f(covered_cols as f32, 1.)); + let source_uv = RectF::new( + vec2f( + run.image_col_start as f32 / cols as f32, + run.image_row as f32 / rows as f32, + ), + vec2f(covered_cols as f32 / cols as f32, 1. / rows as f32), + ); + + ctx.scene + .draw_image_with_source(destination, frame, source_uv, 1., CornerRadius::default()); +} + +/// How many cells of `cell_extent` pixels an image extent of `image_extent` +/// pixels spans, rounding up. This is kitty's default when a virtual placement +/// doesn't say how many rows or columns it covers. +fn cells_to_cover(image_extent: f32, cell_extent: f32) -> u32 { + if cell_extent <= 0. || image_extent <= 0. { + return 0; + } + + (image_extent / cell_extent).ceil() as u32 +} + +/// The frame of an inline image to draw right now, scheduling a repaint if the +/// image is animated. +fn current_image_frame( + image: &Image, + image_id: u32, + ctx: &mut PaintContext, +) -> Option> { + match image { + Image::Static(image) => Some(image.clone()), + Image::Animated(animated_image) => { + let elapsed = ANIMATION_EPOCH.elapsed().as_millis() as u32; + match animated_image.get_current_frame(elapsed) { + Ok((frame, remaining_delay)) => { + ctx.repaint_after(Duration::from_millis(remaining_delay as u64)); + Some(frame) + } + Err(e) => { + log::error!( + "Unable to retrieve current frame from image (image id = {image_id}): {e:?}" + ); + None + } + } + } + } +} + #[allow(clippy::too_many_arguments)] fn render_image( image_id: u32, @@ -1885,8 +2162,29 @@ fn render_image( CornerRadius::default(), ); } - Image::Animated(_) => { - log::warn!("Image should be static"); + Image::Animated(animated_image) => { + // After about ~50 days of uptime, casting the elapsed time to a u32 will + // silently overflow. The animation may jump and resume from a different frame. + let elapsed = ANIMATION_EPOCH.elapsed().as_millis() as u32; + match animated_image.get_current_frame(elapsed) { + Ok((frame, remaining_delay)) => { + let logical_image_size = frame.size().to_f32() / ctx.scene.scale_factor(); + + let image_origin = grid_origin + glyph_offset; + ctx.scene.draw_image( + RectF::new(image_origin, logical_image_size), + frame, + 1., + CornerRadius::default(), + ); + ctx.repaint_after(Duration::from_millis(remaining_delay as u64)); + } + Err(e) => { + log::error!( + "Unable to retrieve current frame from image (image id = {image_id}): {e:?}" + ); + } + } } } } diff --git a/app/src/terminal/grid_renderer/unicode_placeholder.rs b/app/src/terminal/grid_renderer/unicode_placeholder.rs new file mode 100644 index 00000000000..ddbe8228df0 --- /dev/null +++ b/app/src/terminal/grid_renderer/unicode_placeholder.rs @@ -0,0 +1,519 @@ +//! Decoding of kitty's unicode placeholder cells (the `U=1` placement mode). +//! +//! A virtual placement has no anchor in the grid. Instead the application +//! prints cells containing U+10EEEE, and each such cell says which image, and +//! which cell of that image, it stands for. The image id travels in the cell's +//! foreground colour and the row/column travel as combining diacritics on the +//! placeholder character, indexed into [`ROWCOLUMN_DIACRITICS`]. +//! +//! This module only decodes cells and groups them into runs; the drawing lives +//! in [`super::render_unicode_placeholders`]. + +use crate::terminal::model::ansi::Color; +use crate::terminal::model::cell::Cell; +use crate::terminal::model::char_or_str::CharOrStr; + +/// The character every placeholder cell holds. +pub const PLACEHOLDER_CHAR: char = '\u{10EEEE}'; + +/// Combining characters that encode a row or column number, in kitty's +/// specified order: a character's index in this table *is* the number it +/// encodes. The table is sorted ascending, so [`diacritic_index`] can binary +/// search it directly instead of keeping a second sorted copy; the +/// `diacritic_table_is_sorted` test holds that invariant. +#[rustfmt::skip] +static ROWCOLUMN_DIACRITICS: [char; 297] = [ + '\u{0305}', '\u{030D}', '\u{030E}', '\u{0310}', '\u{0312}', '\u{033D}', '\u{033E}', '\u{033F}', + '\u{0346}', '\u{034A}', '\u{034B}', '\u{034C}', '\u{0350}', '\u{0351}', '\u{0352}', '\u{0357}', + '\u{035B}', '\u{0363}', '\u{0364}', '\u{0365}', '\u{0366}', '\u{0367}', '\u{0368}', '\u{0369}', + '\u{036A}', '\u{036B}', '\u{036C}', '\u{036D}', '\u{036E}', '\u{036F}', '\u{0483}', '\u{0484}', + '\u{0485}', '\u{0486}', '\u{0487}', '\u{0592}', '\u{0593}', '\u{0594}', '\u{0595}', '\u{0597}', + '\u{0598}', '\u{0599}', '\u{059C}', '\u{059D}', '\u{059E}', '\u{059F}', '\u{05A0}', '\u{05A1}', + '\u{05A8}', '\u{05A9}', '\u{05AB}', '\u{05AC}', '\u{05AF}', '\u{05C4}', '\u{0610}', '\u{0611}', + '\u{0612}', '\u{0613}', '\u{0614}', '\u{0615}', '\u{0616}', '\u{0617}', '\u{0657}', '\u{0658}', + '\u{0659}', '\u{065A}', '\u{065B}', '\u{065D}', '\u{065E}', '\u{06D6}', '\u{06D7}', '\u{06D8}', + '\u{06D9}', '\u{06DA}', '\u{06DB}', '\u{06DC}', '\u{06DF}', '\u{06E0}', '\u{06E1}', '\u{06E2}', + '\u{06E4}', '\u{06E7}', '\u{06E8}', '\u{06EB}', '\u{06EC}', '\u{0730}', '\u{0732}', '\u{0733}', + '\u{0735}', '\u{0736}', '\u{073A}', '\u{073D}', '\u{073F}', '\u{0740}', '\u{0741}', '\u{0743}', + '\u{0745}', '\u{0747}', '\u{0749}', '\u{074A}', '\u{07EB}', '\u{07EC}', '\u{07ED}', '\u{07EE}', + '\u{07EF}', '\u{07F0}', '\u{07F1}', '\u{07F3}', '\u{0816}', '\u{0817}', '\u{0818}', '\u{0819}', + '\u{081B}', '\u{081C}', '\u{081D}', '\u{081E}', '\u{081F}', '\u{0820}', '\u{0821}', '\u{0822}', + '\u{0823}', '\u{0825}', '\u{0826}', '\u{0827}', '\u{0829}', '\u{082A}', '\u{082B}', '\u{082C}', + '\u{082D}', '\u{0951}', '\u{0953}', '\u{0954}', '\u{0F82}', '\u{0F83}', '\u{0F86}', '\u{0F87}', + '\u{135D}', '\u{135E}', '\u{135F}', '\u{17DD}', '\u{193A}', '\u{1A17}', '\u{1A75}', '\u{1A76}', + '\u{1A77}', '\u{1A78}', '\u{1A79}', '\u{1A7A}', '\u{1A7B}', '\u{1A7C}', '\u{1B6B}', '\u{1B6D}', + '\u{1B6E}', '\u{1B6F}', '\u{1B70}', '\u{1B71}', '\u{1B72}', '\u{1B73}', '\u{1CD0}', '\u{1CD1}', + '\u{1CD2}', '\u{1CDA}', '\u{1CDB}', '\u{1CE0}', '\u{1DC0}', '\u{1DC1}', '\u{1DC3}', '\u{1DC4}', + '\u{1DC5}', '\u{1DC6}', '\u{1DC7}', '\u{1DC8}', '\u{1DC9}', '\u{1DCB}', '\u{1DCC}', '\u{1DD1}', + '\u{1DD2}', '\u{1DD3}', '\u{1DD4}', '\u{1DD5}', '\u{1DD6}', '\u{1DD7}', '\u{1DD8}', '\u{1DD9}', + '\u{1DDA}', '\u{1DDB}', '\u{1DDC}', '\u{1DDD}', '\u{1DDE}', '\u{1DDF}', '\u{1DE0}', '\u{1DE1}', + '\u{1DE2}', '\u{1DE3}', '\u{1DE4}', '\u{1DE5}', '\u{1DE6}', '\u{1DFE}', '\u{20D0}', '\u{20D1}', + '\u{20D4}', '\u{20D5}', '\u{20D6}', '\u{20D7}', '\u{20DB}', '\u{20DC}', '\u{20E1}', '\u{20E7}', + '\u{20E9}', '\u{20F0}', '\u{2CEF}', '\u{2CF0}', '\u{2CF1}', '\u{2DE0}', '\u{2DE1}', '\u{2DE2}', + '\u{2DE3}', '\u{2DE4}', '\u{2DE5}', '\u{2DE6}', '\u{2DE7}', '\u{2DE8}', '\u{2DE9}', '\u{2DEA}', + '\u{2DEB}', '\u{2DEC}', '\u{2DED}', '\u{2DEE}', '\u{2DEF}', '\u{2DF0}', '\u{2DF1}', '\u{2DF2}', + '\u{2DF3}', '\u{2DF4}', '\u{2DF5}', '\u{2DF6}', '\u{2DF7}', '\u{2DF8}', '\u{2DF9}', '\u{2DFA}', + '\u{2DFB}', '\u{2DFC}', '\u{2DFD}', '\u{2DFE}', '\u{2DFF}', '\u{A66F}', '\u{A67C}', '\u{A67D}', + '\u{A6F0}', '\u{A6F1}', '\u{A8E0}', '\u{A8E1}', '\u{A8E2}', '\u{A8E3}', '\u{A8E4}', '\u{A8E5}', + '\u{A8E6}', '\u{A8E7}', '\u{A8E8}', '\u{A8E9}', '\u{A8EA}', '\u{A8EB}', '\u{A8EC}', '\u{A8ED}', + '\u{A8EE}', '\u{A8EF}', '\u{A8F0}', '\u{A8F1}', '\u{AAB0}', '\u{AAB2}', '\u{AAB3}', '\u{AAB7}', + '\u{AAB8}', '\u{AABE}', '\u{AABF}', '\u{AAC1}', '\u{FE20}', '\u{FE21}', '\u{FE22}', '\u{FE23}', + '\u{FE24}', '\u{FE25}', '\u{FE26}', '\u{10A0F}', '\u{10A38}', '\u{1D185}', '\u{1D186}', + '\u{1D187}', '\u{1D188}', '\u{1D189}', '\u{1D1AA}', '\u{1D1AB}', '\u{1D1AC}', '\u{1D1AD}', + '\u{1D242}', '\u{1D243}', '\u{1D244}', +]; + +/// The number a row/column diacritic encodes, or `None` if `c` isn't one. +pub fn diacritic_index(c: char) -> Option { + ROWCOLUMN_DIACRITICS + .binary_search(&c) + .ok() + .map(|index| index as u32) +} + +/// Whether this cell stands in for part of a virtually placed image. Cheap +/// enough to call for every cell on every frame. +pub fn is_unicode_placeholder(cell: &Cell) -> bool { + cell.c == PLACEHOLDER_CHAR +} + +/// One decoded placeholder cell. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PlaceholderCell { + pub image_id: u32, + pub placement_id: u32, + /// Row of the image this cell shows, if the cell spelled it out. + pub row: Option, + /// Column of the image this cell shows, if the cell spelled it out. + pub col: Option, +} + +/// Decodes a placeholder cell, or returns `None` if this isn't one or its image +/// id can't be recovered. +pub fn parse_placeholder_cell(cell: &Cell) -> Option { + if !is_unicode_placeholder(cell) { + return None; + } + + let mut image_id = image_id_from_foreground(cell.fg)?; + + // The base character comes first in the grapheme; the rest are the + // diacritics, in the order they were printed. A cell with no zerowidth + // characters reports a bare `Char` and so carries no position at all. + let grapheme = match cell.raw_content() { + CharOrStr::Str(grapheme) => grapheme, + CharOrStr::Char(_) => "", + }; + let mut diacritics = grapheme.chars().skip(1).filter_map(diacritic_index); + + let row = diacritics.next(); + let col = diacritics.next(); + + // A third diacritic carries the top byte of a 32-bit image id, which does + // not fit in a 24-bit colour. + if let Some(most_significant_byte) = diacritics.next() { + // Only the first 256 diacritics can encode a byte. A higher index is a + // malformed cell, and decoding it anyway would silently name a + // different image. + if most_significant_byte > 0xff { + return None; + } + image_id |= most_significant_byte << 24; + } + + Some(PlaceholderCell { + image_id, + // Placement ids are carried in the cell's underline colour, which this + // fork's `Cell` has nowhere to store. Every placeholder therefore reads + // as placement 0, and the renderer falls back to an image's sole + // virtual placement when it has exactly one. Images displayed through + // several simultaneous virtual placements are consequently not + // distinguishable here. + placement_id: 0, + row, + col, + }) +} + +/// Recovers the image id a placeholder cell's foreground colour encodes. +fn image_id_from_foreground(foreground: Color) -> Option { + match foreground { + Color::Indexed(index) => Some(index as u32), + Color::Spec(color) => { + Some(((color.r as u32) << 16) | ((color.g as u32) << 8) | color.b as u32) + } + // Only the 16 palette colours name an image id; `Foreground`, `Cursor` + // and friends have no numeric meaning here. + Color::Named(named) => { + let index = named.into_color_index(); + (index <= 15).then_some(index as u32) + } + } +} + +/// A horizontal strip of an image to draw with a single quad: one image, one +/// placement, one image row, and consecutive image columns backed by +/// consecutive grid cells. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PlaceholderRun { + pub image_id: u32, + pub placement_id: u32, + /// Row of the image to sample. + pub image_row: u32, + /// First column of the image to sample. + pub image_col_start: u32, + /// Last column of the image to sample, inclusive. + pub image_col_end: u32, + /// First grid column the run covers. + pub col_start: usize, + /// How many grid cells the run covers. + pub len: usize, +} + +/// Resolved position of the placeholder cell to the left, used to fill in +/// diacritics the application left out. +#[derive(Clone, Copy)] +struct PreviousCell { + grid_col: usize, + image_id: u32, + placement_id: u32, + row: u32, + col: u32, +} + +/// Groups a row's placeholder cells into runs. +/// +/// `cells` must yield `(grid column, cell)` in ascending column order. Gaps in +/// the grid columns break a run, as do changes of image, placement or image +/// row. +/// +/// Applications may leave out diacritics for brevity: a cell without a column +/// continues the previous cell's column, and a cell without a row repeats the +/// previous cell's row. Inheritance only applies across immediately adjacent +/// cells of the same placement; anywhere else an omitted number means zero. +pub fn build_runs(cells: impl Iterator) -> Vec { + let mut runs = Vec::new(); + let mut current: Option = None; + let mut previous: Option = None; + + for (grid_col, cell) in cells { + let inherited = previous.filter(|previous| { + previous.grid_col + 1 == grid_col + && previous.image_id == cell.image_id + && previous.placement_id == cell.placement_id + }); + + let row = cell + .row + .or(inherited.map(|previous| previous.row)) + .unwrap_or(0); + let col = cell + .col + .or(inherited.map(|previous| previous.col + 1)) + .unwrap_or(0); + + let extends_current = current.is_some_and(|run| { + run.image_id == cell.image_id + && run.placement_id == cell.placement_id + && run.image_row == row + && run.col_start + run.len == grid_col + && run.image_col_end + 1 == col + }); + + match current.as_mut() { + Some(run) if extends_current => { + run.image_col_end = col; + run.len += 1; + } + _ => { + runs.extend(current.take()); + current = Some(PlaceholderRun { + image_id: cell.image_id, + placement_id: cell.placement_id, + image_row: row, + image_col_start: col, + image_col_end: col, + col_start: grid_col, + len: 1, + }); + } + } + + previous = Some(PreviousCell { + grid_col, + image_id: cell.image_id, + placement_id: cell.placement_id, + row, + col, + }); + } + + runs.extend(current); + runs +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::terminal::model::ansi::NamedColor; + use warpui::color::ColorU; + + /// A placeholder cell carrying `diacritics` after the base character. + fn placeholder_cell(foreground: Color, diacritics: &[char]) -> Cell { + let mut cell = Cell::default(); + cell.c = PLACEHOLDER_CHAR; + cell.fg = foreground; + for diacritic in diacritics { + cell.push_zerowidth(*diacritic, false); + } + cell + } + + /// The diacritic that encodes `index`. + fn diacritic(index: u32) -> char { + ROWCOLUMN_DIACRITICS[index as usize] + } + + #[test] + fn diacritic_table_is_sorted() { + assert_eq!(ROWCOLUMN_DIACRITICS.len(), 297); + assert!( + ROWCOLUMN_DIACRITICS + .windows(2) + .all(|pair| pair[0] < pair[1]), + "binary_search in diacritic_index requires a sorted, duplicate-free table" + ); + } + + #[test] + fn diacritic_index_round_trips() { + for index in [0, 1, 2, 42, 150, 295, 296] { + assert_eq!( + diacritic_index(diacritic(index)), + Some(index), + "index {index} did not round trip" + ); + } + + // Spot-check the first and last entries against the spec's values. + assert_eq!(diacritic(0), '\u{0305}'); + assert_eq!(diacritic(296), '\u{1D244}'); + } + + #[test] + fn non_diacritics_have_no_index() { + for c in ['a', ' ', PLACEHOLDER_CHAR, '\u{0301}'] { + assert_eq!(diacritic_index(c), None, "{c:?} should not be a diacritic"); + } + } + + #[test] + fn non_placeholder_cells_are_rejected() { + let mut cell = Cell::default(); + cell.c = 'a'; + cell.fg = Color::Indexed(7); + + assert_eq!(parse_placeholder_cell(&cell), None); + } + + #[test] + fn indexed_foreground_gives_image_id() { + let cell = placeholder_cell(Color::Indexed(42), &[diacritic(3), diacritic(5)]); + + let parsed = parse_placeholder_cell(&cell).expect("cell should parse"); + assert_eq!(parsed.image_id, 42); + assert_eq!(parsed.row, Some(3)); + assert_eq!(parsed.col, Some(5)); + assert_eq!(parsed.placement_id, 0); + } + + #[test] + fn direct_foreground_gives_24_bit_image_id() { + let cell = placeholder_cell( + Color::Spec(ColorU::new(0x12, 0x34, 0x56, 0xff)), + &[diacritic(0)], + ); + + let parsed = parse_placeholder_cell(&cell).expect("cell should parse"); + assert_eq!(parsed.image_id, 0x123456); + assert_eq!(parsed.row, Some(0)); + // Only one diacritic was printed, so the column is left to inheritance. + assert_eq!(parsed.col, None); + } + + #[test] + fn third_diacritic_supplies_the_image_id_high_byte() { + let cell = placeholder_cell( + Color::Spec(ColorU::new(0x00, 0x00, 0x01, 0xff)), + &[diacritic(0), diacritic(0), diacritic(0xAB)], + ); + + let parsed = parse_placeholder_cell(&cell).expect("cell should parse"); + assert_eq!(parsed.image_id, 0xAB00_0001); + } + + #[test] + fn only_palette_colors_name_an_image_id() { + let palette = placeholder_cell(Color::Named(NamedColor::BrightWhite), &[diacritic(0)]); + assert_eq!( + parse_placeholder_cell(&palette).map(|cell| cell.image_id), + Some(15) + ); + + let default = placeholder_cell(Color::Named(NamedColor::Foreground), &[diacritic(0)]); + assert_eq!(parse_placeholder_cell(&default), None); + } + + /// Shorthand for a parsed cell at a grid column. + fn parsed(grid_col: usize, row: Option, col: Option) -> (usize, PlaceholderCell) { + ( + grid_col, + PlaceholderCell { + image_id: 1, + placement_id: 0, + row, + col, + }, + ) + } + + #[test] + fn fully_specified_cells_batch_into_one_run() { + let cells = vec![ + parsed(0, Some(0), Some(0)), + parsed(1, Some(0), Some(1)), + parsed(2, Some(0), Some(2)), + ]; + + let runs = build_runs(cells.into_iter()); + + assert_eq!(runs.len(), 1); + assert_eq!(runs[0].image_row, 0); + assert_eq!(runs[0].image_col_start, 0); + assert_eq!(runs[0].image_col_end, 2); + assert_eq!(runs[0].col_start, 0); + assert_eq!(runs[0].len, 3); + } + + #[test] + fn omitted_diacritics_inherit_from_the_left() { + // The common shorthand: only the first cell spells out its position. + let cells = vec![ + parsed(4, Some(2), Some(0)), + parsed(5, None, None), + parsed(6, None, None), + ]; + + let runs = build_runs(cells.into_iter()); + + assert_eq!( + runs.len(), + 1, + "inherited cells should join the run: {runs:?}" + ); + assert_eq!(runs[0].image_row, 2); + assert_eq!(runs[0].image_col_start, 0); + assert_eq!(runs[0].image_col_end, 2); + assert_eq!(runs[0].col_start, 4); + assert_eq!(runs[0].len, 3); + } + + #[test] + fn a_row_only_cell_inherits_the_column() { + let cells = vec![parsed(0, Some(1), Some(7)), parsed(1, Some(1), None)]; + + let runs = build_runs(cells.into_iter()); + + assert_eq!(runs.len(), 1); + assert_eq!(runs[0].image_col_start, 7); + assert_eq!(runs[0].image_col_end, 8); + } + + #[test] + fn a_gap_in_grid_columns_splits_runs() { + let cells = vec![parsed(0, Some(0), Some(0)), parsed(2, Some(0), Some(1))]; + + let runs = build_runs(cells.into_iter()); + + assert_eq!(runs.len(), 2); + assert_eq!(runs[0].col_start, 0); + assert_eq!(runs[1].col_start, 2); + // The gap also stops inheritance, so cell 2 keeps its explicit column. + assert_eq!(runs[1].image_col_start, 1); + } + + #[test] + fn a_new_row_splits_runs() { + let cells = vec![ + parsed(0, Some(0), Some(0)), + parsed(1, Some(1), Some(1)), + parsed(2, None, None), + ]; + + let runs = build_runs(cells.into_iter()); + + assert_eq!(runs.len(), 2); + assert_eq!(runs[0].image_row, 0); + assert_eq!(runs[1].image_row, 1); + assert_eq!(runs[1].len, 2, "the third cell inherits row 1: {runs:?}"); + } + + #[test] + fn a_different_image_splits_runs() { + let mut cells = vec![parsed(0, Some(0), Some(0)), parsed(1, Some(0), Some(1))]; + cells[1].1.image_id = 9; + + let runs = build_runs(cells.into_iter()); + + assert_eq!(runs.len(), 2); + assert_eq!(runs[0].image_id, 1); + assert_eq!(runs[1].image_id, 9); + } + + #[test] + fn no_cells_means_no_runs() { + assert!(build_runs(std::iter::empty()).is_empty()); + } + + /// The decoder is only useful if the print path keeps a placeholder cell + /// whole: the base character in `Cell::c`, the diacritics as zerowidth + /// characters, and the image id in the foreground colour. + #[test] + fn printed_placeholder_cells_survive_the_grid() { + let mut terminal = crate::terminal::model::TerminalModel::mock(None, None); + terminal.simulate_cmd("kitty"); + + // A 24-bit foreground carrying image id 0x010203, then two placeholder + // cells: the first spells out row 0 and column 0, the second omits both. + let mut printed = String::from("\x1b[38;2;1;2;3m"); + printed.push(PLACEHOLDER_CHAR); + printed.push(diacritic(0)); + printed.push(diacritic(0)); + printed.push(PLACEHOLDER_CHAR); + terminal.process_bytes(printed.as_str()); + + let grid = terminal + .block_list() + .active_block() + .output_grid() + .grid_handler(); + let row = grid.row(0).expect("the printed row should exist"); + + let first = parse_placeholder_cell(&row[0]).expect("first cell should parse"); + assert_eq!(first.image_id, 0x01_0203); + assert_eq!(first.row, Some(0)); + assert_eq!(first.col, Some(0)); + + let second = parse_placeholder_cell(&row[1]).expect("second cell should parse"); + assert_eq!(second.image_id, 0x01_0203); + assert_eq!(second.row, None, "trailing diacritics were omitted"); + assert_eq!(second.col, None, "trailing diacritics were omitted"); + + // Together they are a single quad covering image columns 0 and 1. + let runs = build_runs([(0, first), (1, second)].into_iter()); + assert_eq!(runs.len(), 1, "{runs:?}"); + assert_eq!(runs[0].image_col_start, 0); + assert_eq!(runs[0].image_col_end, 1); + assert_eq!(runs[0].len, 2); + } +} diff --git a/app/src/terminal/model/grid/ansi_handler.rs b/app/src/terminal/model/grid/ansi_handler.rs index 4d673750b47..58caa3ecb61 100644 --- a/app/src/terminal/model/grid/ansi_handler.rs +++ b/app/src/terminal/model/grid/ansi_handler.rs @@ -4,7 +4,7 @@ mod tab_stops; use std::cmp::min; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::io; use std::ops::Range; use std::sync::Arc; @@ -35,7 +35,8 @@ use crate::terminal::model::image_map::{ImagePlacementData, ImageType, StoredIma use crate::terminal::model::index::{Point, VisibleRow}; use crate::terminal::model::iterm_image::{ITermImage, ITermImageDimensionUnit}; use crate::terminal::model::kitty::{ - CursorMovementPolicy, KittyAction, KittyError, KittyResponse, StorageError, + CursorMovementPolicy, DeletionType, InvalidKittyAction, KittyAction, KittyError, KittyResponse, + StorageError, }; use crate::terminal::model::selection::ScrollDelta; use crate::terminal::model::ObfuscateSecrets; @@ -1343,6 +1344,8 @@ impl ansi::Handler for GridHandler { // Convert the visual dimension in pixels to cells. We want to round up if this doesn't perfectly fit within an amount of cells. let height_cells = (height_px as f32 / (self.ansi_handler_state.cell_height as f32)).ceil() as usize; + let width_cells = + (width_px as f32 / (self.ansi_handler_state.cell_width as f32)).ceil() as usize; // Convert the user requested dimension in pixels to cells. This is needed to scroll the cursor by the space the user requested. // If preserve_aspect_ratio is true, this may be larger than the visual dimension of the image. @@ -1377,6 +1380,7 @@ impl ansi::Handler for GridHandler { ImagePlacementData { z_index: 0, height_cells, + width_cells, image_size, }, ); @@ -1450,6 +1454,12 @@ impl ansi::Handler for GridHandler { } } +/// Converts a kitty cell coordinate into a 0-based grid index. Kitty numbers +/// cells from one, so a zero means the key was absent and selects no cell. +fn cell_index(coordinate: u32) -> Option { + coordinate.checked_sub(1).map(|index| index as usize) +} + /// Helper functions for the [`ansi::Handler`] implementation. impl GridHandler { /// Advances the cursor by one cell, handling wrapping appropriately. @@ -1712,6 +1722,34 @@ impl GridHandler { } } + /// Maps a kitty delete coordinate pair onto the absolute cell that + /// placements are keyed by, using the same translation that placing an image + /// at the cursor uses. + fn delete_cell(&self, col: u32, row: u32) -> Option { + let col = cell_index(col)?; + let row = cell_index(row)?; + Some(AbsolutePoint::from_point(Point::new(row, col), self)) + } + + /// Removes every placement drawn at `z_index`, returning what was removed so + /// that callers can also free the images that no longer have a placement. + pub fn evict_placements_with_z(&mut self, z_index: i32) -> Vec<(u32, u32)> { + let placements = self.images.placements_with_z(z_index); + self.images.evict_placements(&placements); + placements + } + + /// Removes every placement whose image id lies in the inclusive range. + pub fn evict_placements_in_id_range(&mut self, start: u32, end: u32) { + let placements = self.images.placements_for_id_range(start, end); + self.images.evict_placements(&placements); + } + + /// Removes every placement of an image that is not in `live`. + pub fn evict_images_absent_from(&mut self, live: &HashSet) { + self.images.evict_images_absent_from(live); + } + fn handle_completed_kitty_action_internal( &mut self, action: KittyAction, @@ -1768,8 +1806,24 @@ impl GridHandler { return Ok(()); } - let max_width = - (self.columns() - self.cursor_point().col) * self.ansi_handler_state.cell_width; + // A virtual placement has no anchor in the grid and must not + // disturb the cursor: the unicode placeholder cells that + // reference it decide where it is drawn. The image data still + // has to reach the asset cache. + if action.placement_data.unicode_placeholder { + self.ansi_handler_state + .event_proxy + .send_terminal_event(Event::ImageReceived { + image_id: action.image_id, + image_data: action.image.data, + image_protocol: ImageProtocol::Kitty, + }); + + return Ok(()); + } + + let max_width = self.columns().saturating_sub(self.cursor_point().col) + * self.ansi_handler_state.cell_width; let max_height = MAX_IMAGE_CELL_HEIGHT as usize * self.ansi_handler_state.cell_height; @@ -1812,6 +1866,7 @@ impl GridHandler { ImagePlacementData { z_index: action.placement_data.z_index, height_cells, + width_cells, image_size, }, ); @@ -1832,7 +1887,7 @@ impl GridHandler { } // Create the whitespace to fit the image on. - for _ in 0..height_cells - 1 { + for _ in 0..height_cells.saturating_sub(1) { self.newline(); } @@ -1858,6 +1913,10 @@ impl GridHandler { } }; + if metadata.image_size.x() == 0.0 || metadata.image_size.y() == 0.0 { + return Ok(()); + } + if let Some(0) = action.placement_data.cols { return Ok(()); } @@ -1866,8 +1925,14 @@ impl GridHandler { return Ok(()); } - let max_width = - (self.columns() - self.cursor_point().col) * self.ansi_handler_state.cell_width; + // See the matching branch in `StoreAndDisplay`: a virtual + // placement is anchored by its placeholder cells, not the grid. + if action.placement_data.unicode_placeholder { + return Ok(()); + } + + let max_width = self.columns().saturating_sub(self.cursor_point().col) + * self.ansi_handler_state.cell_width; let max_height = MAX_IMAGE_CELL_HEIGHT as usize * self.ansi_handler_state.cell_height; @@ -1902,6 +1967,7 @@ impl GridHandler { ImagePlacementData { z_index: action.placement_data.z_index, height_cells, + width_cells, image_size, }, ); @@ -1922,7 +1988,7 @@ impl GridHandler { } // Create the whitespace to fit the image on. - for _ in 0..height_cells - 1 { + for _ in 0..height_cells.saturating_sub(1) { self.newline(); } @@ -1938,7 +2004,66 @@ impl GridHandler { }); } KittyAction::QuerySupport(_) => {} - KittyAction::Delete { .. } => {} + KittyAction::Delete { + delete_placements_only, + deletion_type, + } => { + // Only the positional specifiers are handled here: they are the + // ones that need this grid's cursor and placement geometry. The + // rest are applied to every grid by the terminal model. + let placements = match deletion_type { + DeletionType::AtCursor => { + let cursor = AbsolutePoint::from_point(self.cursor_point(), self); + self.images.placements_at_point(cursor.col, cursor.row) + } + DeletionType::AtPoint { col, row } => match self.delete_cell(col, row) { + Some(cell) => self.images.placements_at_point(cell.col, cell.row), + None => vec![], + }, + DeletionType::AtPointZ { col, row, z } => match self.delete_cell(col, row) { + Some(cell) => self + .images + .placements_at_point(cell.col, cell.row) + .into_iter() + .filter(|&(image_id, placement_id)| { + self.images + .get_image_placement_data(image_id, placement_id) + .is_some_and(|data| data.z_index == z) + }) + .collect(), + None => vec![], + }, + DeletionType::Column(col) => match cell_index(col) { + Some(col) => self.images.placements_intersecting_col(col), + None => vec![], + }, + DeletionType::Row(row) => match self.delete_cell(1, row) { + Some(cell) => self.images.placements_intersecting_row(cell.row), + None => vec![], + }, + DeletionType::Frames { .. } => { + return Err(InvalidKittyAction::UnsupportedAction.into()) + } + DeletionType::All + | DeletionType::ById { .. } + | DeletionType::ByNumber { .. } + | DeletionType::ZIndex(_) + | DeletionType::IdRange { .. } => return Ok(()), + }; + + // An uppercase specifier frees the image data as well, so the + // model must forget the metadata it looks placements up by. + if !delete_placements_only { + for (image_id, _) in &placements { + metadata.remove(image_id); + } + } + + self.images.evict_placements(&placements); + } + // Animation actions carry no placement: the terminal model applies + // them to the image itself before any grid sees them. + KittyAction::TransmitFrame { .. } | KittyAction::AnimationControl { .. } => {} } Ok(()) diff --git a/app/src/terminal/model/image_map.rs b/app/src/terminal/model/image_map.rs index d57068c66aa..36c7428d986 100644 --- a/app/src/terminal/model/image_map.rs +++ b/app/src/terminal/model/image_map.rs @@ -268,6 +268,92 @@ impl ImageMap { } } + /// Removes every kitty placement of an image that is not in `live`. Only + /// kitty placements: a kitty delete command must be structurally incapable + /// of touching an iTerm image, whose metadata lifecycle is independent. + pub fn evict_images_absent_from(&mut self, live: &HashSet) { + let orphaned: Vec<(u32, u32)> = self + .point_by_image_id + .keys() + .copied() + .filter(|(image_id, _)| { + !live.contains(image_id) + && self.image_type_by_image_id.get(image_id) == Some(&ImageType::Kitty) + }) + .collect(); + + self.evict_placements(&orphaned); + } + + /// Every placement in the map, paired with its anchor cell and its geometry. + /// Placements without recorded geometry are skipped, matching how the + /// rendering queries treat them. + fn placements( + &self, + ) -> impl Iterator + '_ { + let placement_data = &self.image_placement_data; + self.image_ids_by_point + .iter() + .flat_map(move |(&top_left, ids)| { + ids.iter().filter_map(move |&id| { + let data = placement_data.get(&id)?; + Some((top_left, id, data)) + }) + }) + } + + /// The placements whose cell footprint contains the given absolute cell. + pub fn placements_at_point(&self, col: usize, row: u64) -> Vec<(u32, u32)> { + self.placements() + .filter(|(top_left, _, data)| { + data.covers_row(*top_left, row) && data.covers_col(*top_left, col) + }) + .map(|(_, id, _)| id) + .collect_vec() + } + + /// The placements whose cell footprint intersects the given absolute row. + pub fn placements_intersecting_row(&self, row: u64) -> Vec<(u32, u32)> { + self.placements() + .filter(|(top_left, _, data)| data.covers_row(*top_left, row)) + .map(|(_, id, _)| id) + .collect_vec() + } + + /// The placements whose cell footprint intersects the given column. + pub fn placements_intersecting_col(&self, col: usize) -> Vec<(u32, u32)> { + self.placements() + .filter(|(top_left, _, data)| data.covers_col(*top_left, col)) + .map(|(_, id, _)| id) + .collect_vec() + } + + /// The placements drawn at the given z-index. + pub fn placements_with_z(&self, z_index: i32) -> Vec<(u32, u32)> { + self.placements() + .filter(|(_, _, data)| data.z_index == z_index) + .map(|(_, id, _)| id) + .collect_vec() + } + + /// The placements of every image whose id lies in the inclusive range. + pub fn placements_for_id_range(&self, start: u32, end: u32) -> Vec<(u32, u32)> { + if start > end { + return vec![]; + } + + self.point_by_image_id + .range((Included(&(start, u32::MIN)), Included(&(end, u32::MAX)))) + .map(|(&id, _)| id) + .collect_vec() + } + + pub fn evict_placements(&mut self, placements: &[(u32, u32)]) { + for &(image_id, placement_id) in placements { + self.evict_placement(image_id, placement_id); + } + } + pub fn evict_placement(&mut self, image_id: u32, placement_id: u32) { self.image_placement_data.remove(&(image_id, placement_id)); if let Some(point) = self.point_by_image_id.get(&(image_id, placement_id)) { @@ -322,5 +408,25 @@ impl StoredImageMetadata { pub struct ImagePlacementData { pub z_index: i32, pub height_cells: usize, + /// The placement's width in cells. Needed to decide whether a placement + /// intersects a given cell or column. + pub width_cells: usize, pub image_size: Vector2F, } + +impl ImagePlacementData { + /// Whether the placement anchored at `top_left` covers `row`. A placement + /// with no recorded extent still covers the cell it is anchored on, so that + /// it stays deletable. + fn covers_row(&self, top_left: AbsolutePoint, row: u64) -> bool { + let height = self.height_cells.max(1) as u64; + (top_left.row..top_left.row + height).contains(&row) + } + + /// Whether the placement anchored at `top_left` covers `col`. See + /// [`ImagePlacementData::covers_row`]. + fn covers_col(&self, top_left: AbsolutePoint, col: usize) -> bool { + let width = self.width_cells.max(1); + (top_left.col..top_left.col + width).contains(&col) + } +} diff --git a/app/src/terminal/model/kitty.rs b/app/src/terminal/model/kitty.rs index e8b84096254..8a94c60209d 100644 --- a/app/src/terminal/model/kitty.rs +++ b/app/src/terminal/model/kitty.rs @@ -4,6 +4,7 @@ use flate2::read::ZlibDecoder; use pathfinder_geometry::vector::Vector2F; use rand::Rng; use std::cmp::min; +use std::collections::HashMap; use std::io::Read; #[cfg(feature = "local_fs")] use std::{env, fs, str}; @@ -27,6 +28,52 @@ pub enum KittyAction { delete_placements_only: bool, deletion_type: DeletionType, }, + /// `a=f`: an animation frame for an already transmitted image. Only + /// full-canvas frames are accepted; see [`KittyImageMetadata::frames`]. + TransmitFrame { + image_id: u32, + /// The `r=` frame number this frame replaces, if the client gave one. + /// Absent means "append". + frame_number: Option, + /// The `z=` gap in milliseconds until the following frame. + gap_ms: u32, + image: KittyImage, + }, + /// `a=a`: start or stop an image's animation, and/or change a frame's gap. + AnimationControl { + image_id: u32, + /// The playback state requested by `s=`, or `None` when the message only + /// edits a gap. + play: Option, + /// The `r=` frame number and its new `z=` gap in milliseconds. + gap_edit: Option<(u32, u32)>, + }, +} + +/// The gap kitty falls back to for a frame that does not carry a usable one. +pub const DEFAULT_FRAME_GAP_MS: u32 = 40; + +/// Caps on the animation frames kept per image, so a runaway or malicious +/// `a=f` stream cannot grow memory without bound. Kitty itself keeps every +/// frame; the bound is a deliberate divergence, reported as `ENOTSUPP:`. +pub const MAX_ANIMATION_FRAMES: usize = 1024; +pub const MAX_ANIMATION_FRAME_BYTES: usize = 256 * 1024 * 1024; + +/// Reads a frame's `z=` gap. Kitty ignores a zero gap, which leaves the default +/// in place, and reads a negative one as "no gap at all". +fn frame_gap_ms(z: i32) -> u32 { + match z { + 0 => DEFAULT_FRAME_GAP_MS, + z if z.is_negative() => 0, + z => z as u32, + } +} + +/// Maps a protocol frame number onto an index into +/// [`KittyImageMetadata::frames`]. Frame 1 is the root image, which is not +/// stored in that list, so it has no index. +pub fn frame_index(frame_number: u32) -> Option { + frame_number.checked_sub(2).map(|index| index as usize) } #[derive(Debug, Default, Clone)] @@ -56,16 +103,42 @@ pub struct QuerySupport { pub image_id: u32, } +/// Which placements a `a=d` message asks the terminal to remove, as selected by +/// the message's `d=` key. #[derive(Debug, Clone)] pub enum DeletionType { - DeleteAll, - DeleteById(DeleteById), -} - -#[derive(Debug, Clone)] -pub struct DeleteById { - pub image_id: u32, - pub placement_id: Option, + /// `d=a`: every placement on screen. + All, + /// `d=i`: every placement of an image id, or a single placement when + /// `placement_id` is given. + ById { + image_id: u32, + placement_id: Option, + }, + /// `d=n`: like [`DeletionType::ById`], but the image is selected by the + /// client assigned image number (`I=`) rather than its id. + ByNumber { + image_number: u32, + placement_id: Option, + }, + /// `d=c`: every placement intersecting the cell the cursor is on. + AtCursor, + /// `d=p`: every placement intersecting the given cell. + AtPoint { col: u32, row: u32 }, + /// `d=q`: every placement intersecting the given cell that also has the + /// given z-index. + AtPointZ { col: u32, row: u32, z: i32 }, + /// `d=x`: every placement intersecting the given column. + Column(u32), + /// `d=y`: every placement intersecting the given row. + Row(u32), + /// `d=z`: every placement with the given z-index. + ZIndex(i32), + /// `d=r`: every placement whose image id lies in the inclusive range. + IdRange { start: u32, end: u32 }, + /// `d=f`: the animation frames of an image. Frames are not stored yet, so + /// this is reported as unsupported. + Frames { image_id: u32 }, } #[derive(Debug, Default, Clone)] @@ -172,7 +245,7 @@ impl From for KittyError { #[derive(Debug, Clone)] pub enum InvalidControlData { IdMissing, - UnicodePlaceholderUnsupported, + UnknownDeleteAction, } impl From for KittyError { @@ -318,10 +391,6 @@ impl TryFrom for KittyAction { Ok(KittyAction::StoreOnly(action)) } KittyPlacementAction::StoreAndDisplay => { - if message.control_data.unicode_placeholder { - return Err(InvalidControlData::UnicodePlaceholderUnsupported.into()); - } - let mut action = StoreAndDisplay { image_id: message .control_data @@ -336,6 +405,7 @@ impl TryFrom for KittyAction { cols: message.control_data.cols, rows: message.control_data.rows, cursor_movement_policy: message.control_data.cursor_movement_policy, + unicode_placeholder: message.control_data.unicode_placeholder, }, image: KittyImage::try_from(message)?, }; @@ -349,10 +419,6 @@ impl TryFrom for KittyAction { Ok(KittyAction::StoreAndDisplay(action)) } KittyPlacementAction::DisplayStoredImage => { - if message.control_data.unicode_placeholder { - return Err(InvalidControlData::UnicodePlaceholderUnsupported.into()); - } - let id = match message.control_data.image_id { Some(id) => id, None => return Err(InvalidControlData::IdMissing.into()), @@ -369,6 +435,7 @@ impl TryFrom for KittyAction { cols: message.control_data.cols, rows: message.control_data.rows, cursor_movement_policy: message.control_data.cursor_movement_policy, + unicode_placeholder: message.control_data.unicode_placeholder, }, })) } @@ -390,18 +457,51 @@ impl TryFrom for KittyAction { Ok(KittyAction::QuerySupport(action)) } KittyPlacementAction::Delete => { - let deletion_type = match message.control_data.delete_action { - DeleteAction::DeleteAll => DeletionType::DeleteAll, - DeleteAction::DeleteById => { - let image_id = match message.control_data.image_id { + let control_data = &message.control_data; + // `x=`/`y=` carry the cell coordinates for the positional + // specifiers and the id bounds for `d=r`. A missing key reads as + // 0, which is not a valid 1-based cell coordinate or image id, + // so such a message deletes nothing. + let (x, y) = ( + control_data.delete_x.unwrap_or(0), + control_data.delete_y.unwrap_or(0), + ); + + let deletion_type = match control_data.delete_action { + DeleteAction::All => DeletionType::All, + DeleteAction::ById => DeletionType::ById { + image_id: match control_data.image_id { Some(image_id) => image_id, None => return Err(InvalidControlData::IdMissing.into()), - }; - - DeletionType::DeleteById(DeleteById { - image_id, - placement_id: message.control_data.placement_id, - }) + }, + placement_id: control_data.placement_id, + }, + DeleteAction::ByNumber => DeletionType::ByNumber { + image_number: match control_data.image_number { + Some(image_number) => image_number, + None => return Err(InvalidControlData::IdMissing.into()), + }, + placement_id: control_data.placement_id, + }, + DeleteAction::AtCursor => DeletionType::AtCursor, + DeleteAction::AtPoint => DeletionType::AtPoint { col: x, row: y }, + DeleteAction::AtPointZ => DeletionType::AtPointZ { + col: x, + row: y, + z: control_data.z_index, + }, + DeleteAction::Column => DeletionType::Column(x), + DeleteAction::Row => DeletionType::Row(y), + DeleteAction::ZIndex => DeletionType::ZIndex(control_data.z_index), + DeleteAction::IdRange => DeletionType::IdRange { start: x, end: y }, + DeleteAction::Frames => DeletionType::Frames { + image_id: match control_data.image_id { + Some(image_id) => image_id, + None => return Err(InvalidControlData::IdMissing.into()), + }, + }, + DeleteAction::Unknown => { + return Err(InvalidControlData::UnknownDeleteAction.into()) } }; @@ -410,6 +510,71 @@ impl TryFrom for KittyAction { delete_placements_only: message.control_data.delete_placements_only, }) } + KittyPlacementAction::TransmitFrame => { + let image_id = match message.control_data.image_id { + Some(image_id) => image_id, + None => return Err(InvalidControlData::IdMissing.into()), + }; + + // On a frame, `x=`/`y=` place it inside the canvas and `c=` names + // a frame to use as its base. All three ask for compositing, + // which this terminal does not implement. + if message.control_data.delete_x.unwrap_or(0) != 0 + || message.control_data.delete_y.unwrap_or(0) != 0 + || message.control_data.cols.unwrap_or(0) != 0 + { + return Err(InvalidKittyAction::UnsupportedAction.into()); + } + + let frame_number = message.control_data.rows; + let gap_ms = frame_gap_ms(message.control_data.z_index); + + let mut image = KittyImage::try_from(message)?; + if image.metadata.pixel_data_format == KittyPixelDataFormat::Png { + image = set_kitty_png_size(image)?; + } else { + image = set_kitty_rgb_headers(image)?; + } + + Ok(KittyAction::TransmitFrame { + image_id, + frame_number, + gap_ms, + image, + }) + } + KittyPlacementAction::AnimationControl => { + let control_data = &message.control_data; + + let image_id = match control_data.image_id { + Some(image_id) => image_id, + None => return Err(InvalidControlData::IdMissing.into()), + }; + + // `s=` carries the playback state here rather than a width. Kitty + // separates "run until the loops run out" (2) from "run" (3); + // loops are not tracked, so both simply play. + let play = match control_data.width { + 1 => Some(false), + 2 | 3 => Some(true), + _ => None, + }; + + // A frame number without a gap edits nothing: kitty ignores a + // zero gap. + let gap_edit = match (control_data.rows, control_data.z_index) { + (Some(frame_number), gap) if gap != 0 => { + Some((frame_number, frame_gap_ms(gap))) + } + _ => None, + }; + + Ok(KittyAction::AnimationControl { + image_id, + play, + gap_edit, + }) + } KittyPlacementAction::Unknown => Err(InvalidKittyAction::UnsupportedAction.into()), } } @@ -420,6 +585,32 @@ pub struct KittyImageMetadata { pub pixel_data_format: KittyPixelDataFormat, pub transmission_medium: KittyTransmissionMedium, pub image_size: Vector2F, + /// The `I=` number the client transmitted this image under, if any. Delete + /// messages using `d=n` select an image by this number instead of by id. + pub image_number: Option, + /// Placements created with `U=1`, keyed by placement id. These have no + /// anchor in the grid: the cells that reference them carry the row/column + /// to sample, so the renderer resolves their geometry per frame. Dropping + /// this image's metadata drops its virtual placements with it, which is why + /// no other copy of this state is kept. + pub virtual_placements: HashMap, + /// The animation frames transmitted with `a=f`, each paired with the gap in + /// milliseconds until the frame after it. The root image is frame 1 of the + /// animation and is not copied here — the asset cache already holds it — so + /// the protocol's frame number `n` is `frames[n - 2]` (see [`frame_index`]). + pub frames: Vec<(Vec, u32)>, + /// Whether this image's animation is running. Set by `a=a,s=`, and by the + /// first `a=f`: frames are transmitted in order to be played. + pub playing: bool, +} + +/// A `U=1` placement. `rows`/`cols` stay unresolved so that a font-size change +/// re-tiles the image against the new cell size instead of stretching it. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct VirtualPlacement { + pub rows: Option, + pub cols: Option, + pub z_index: i32, } #[derive(Debug, Default, Clone)] @@ -428,9 +619,21 @@ pub struct KittyPlacementData { pub cols: Option, pub rows: Option, pub cursor_movement_policy: CursorMovementPolicy, + /// Set by `U=1`: the image is not anchored at the cursor, it is drawn + /// wherever unicode placeholder cells reference it. + pub unicode_placeholder: bool, } impl KittyPlacementData { + /// The renderer-facing view of a `U=1` placement. + pub fn virtual_placement(&self) -> VirtualPlacement { + VirtualPlacement { + rows: self.rows, + cols: self.cols, + z_index: self.z_index, + } + } + pub fn get_desired_dimensions( &self, image_size: Vector2F, @@ -475,6 +678,10 @@ impl From for KittyImageMetadata { pixel_data_format: control_data.pixel_data_format, image_size: Vector2F::new(control_data.width as f32, control_data.height as f32), transmission_medium: control_data.transmission_medium, + image_number: control_data.image_number, + virtual_placements: HashMap::new(), + frames: Vec::new(), + playing: false, } } } @@ -505,6 +712,8 @@ pub enum KittyPlacementAction { DisplayStoredImage, QuerySupport, Delete, + TransmitFrame, + AnimationControl, Unknown, } @@ -517,11 +726,21 @@ pub struct KittyControlData { transmission_medium: KittyTransmissionMedium, pub further_chunks: bool, pub image_id: Option, + /// The client assigned image number (`I=`), which delete messages can use in + /// place of an image id. + pub image_number: Option, pub placement_id: Option, pub placement_action: KittyPlacementAction, pub verbosity: KittyResponseVerbosity, pub z_index: i32, pub delete_action: DeleteAction, + /// The `x=` key. It is a crop offset on transmission, a frame offset on + /// `a=f`, and either a cell column or an image id bound on deletion. Only the + /// delete reading is implemented; a frame offset is read just far enough to + /// reject it. + pub delete_x: Option, + /// The `y=` key. See [`KittyControlData::delete_x`]. + pub delete_y: Option, delete_placements_only: bool, pub rows: Option, pub cols: Option, @@ -539,11 +758,14 @@ impl Default for KittyControlData { transmission_medium: KittyTransmissionMedium::default(), further_chunks: false, image_id: None, + image_number: None, placement_id: None, placement_action: KittyPlacementAction::default(), verbosity: KittyResponseVerbosity::default(), z_index: 0, delete_action: DeleteAction::default(), + delete_x: None, + delete_y: None, delete_placements_only: true, rows: None, cols: None, @@ -553,11 +775,24 @@ impl Default for KittyControlData { } } +/// The `d=` key of a delete message. The kitty protocol defaults it to `a`. #[derive(Default, Debug, PartialEq, Clone, Copy)] pub enum DeleteAction { #[default] - DeleteAll, - DeleteById, + All, + ById, + ByNumber, + AtCursor, + AtPoint, + AtPointZ, + Column, + Row, + ZIndex, + IdRange, + Frames, + /// A `d=` letter this terminal does not recognize. Kept distinct from + /// [`DeleteAction::All`] so that a typo cannot wipe out every image. + Unknown, } #[derive(Default, Debug, PartialEq, Clone, Copy)] @@ -652,6 +887,11 @@ fn parse_kitty_control_data(control_data: &[u8]) -> KittyControlData { parsed_control_data.image_id = Some(value); } } + b"I" => { + if let Some(value) = parse_u32(value) { + parsed_control_data.image_number = Some(value); + } + } b"p" => { if let Some(value) = parse_u32(value) { parsed_control_data.placement_id = Some(value); @@ -664,6 +904,8 @@ fn parse_kitty_control_data(control_data: &[u8]) -> KittyControlData { b"p" => KittyPlacementAction::DisplayStoredImage, b"q" => KittyPlacementAction::QuerySupport, b"d" => KittyPlacementAction::Delete, + b"f" => KittyPlacementAction::TransmitFrame, + b"a" => KittyPlacementAction::AnimationControl, _ => KittyPlacementAction::Unknown, } } @@ -681,16 +923,28 @@ fn parse_kitty_control_data(control_data: &[u8]) -> KittyControlData { } } b"d" => { + // An uppercase specifier additionally frees the stored image data, + // not just the placements. if value.iter().all(|x| x.is_ascii_uppercase()) { parsed_control_data.delete_placements_only = false; } parsed_control_data.delete_action = match &value.to_ascii_lowercase()[..] { - // Note: Kitty Protocol specifies "a" as "Delete all placements on screen", - // we're just starting with DeleteAll placements to expedite the launch. - b"a" => DeleteAction::DeleteAll, - b"i" => DeleteAction::DeleteById, - _ => DeleteAction::default(), + b"a" => DeleteAction::All, + b"i" => DeleteAction::ById, + b"n" => DeleteAction::ByNumber, + b"c" => DeleteAction::AtCursor, + b"p" => DeleteAction::AtPoint, + b"q" => DeleteAction::AtPointZ, + b"x" => DeleteAction::Column, + b"y" => DeleteAction::Row, + b"z" => DeleteAction::ZIndex, + b"r" => DeleteAction::IdRange, + b"f" => DeleteAction::Frames, + // Deliberately not `DeleteAction::default()`: falling back to + // "delete everything" would let one unrecognized letter erase + // images the client never asked to remove. + _ => DeleteAction::Unknown, } } b"c" => { @@ -710,6 +964,16 @@ fn parse_kitty_control_data(control_data: &[u8]) -> KittyControlData { _ => CursorMovementPolicy::default(), } } + b"x" => { + if let Some(value) = parse_u32(value) { + parsed_control_data.delete_x = Some(value); + } + } + b"y" => { + if let Some(value) = parse_u32(value) { + parsed_control_data.delete_y = Some(value); + } + } b"U" => { parsed_control_data.unicode_placeholder = value == b"1"; } @@ -953,20 +1217,71 @@ pub fn set_kitty_rgb_headers(mut image: KittyImage) -> Result Vec { +/// The error code prefix the kitty graphics protocol expects a failed response +/// to start with, including its trailing colon. +fn kitty_error_code(err: &KittyError) -> String { + let code = match err { + KittyError::KittyFeatureDisabled => "ENOTSUPP", + KittyError::StorageError(StorageError::UnknownId { .. }) => "ENOENT", + KittyError::InvalidKittyAction(action) => match action { + InvalidKittyAction::UnsupportedAction => "ENOTSUPP", + InvalidKittyAction::InvalidControlData(_) => "EINVAL", + InvalidKittyAction::InvalidKittyPayload(payload) => match payload { + InvalidKittyPayload::InvalidTransmissionMedium(_) + | InvalidKittyPayload::KittyDecodeError(_) + | InvalidKittyPayload::InvalidKittyImage(_) => "EINVAL", + InvalidKittyPayload::FileError(FileError::UnsupportedPlatform) + | InvalidKittyPayload::ShmError(ShmError::UnsupportedPlatform) => "ENOTSUPP", + InvalidKittyPayload::FileError(_) | InvalidKittyPayload::ShmError(_) => "EBADF", + }, + }, + }; + + format!("{code}:") +} + +fn create_kitty_reply( + image_id: u32, + placement_id: Option, + image_number: Option, + message: String, +) -> Vec { + let mut identifiers = format!("i={image_id}"); + // A client that numbers its images needs `I=` echoed back to map the + // number onto the id the terminal assigned. + if let Some(image_number) = image_number.filter(|&number| number != 0) { + identifiers.push_str(&format!(",I={image_number}")); + } + // 0 is not a valid placement id, so a client that left `p` at its zero + // default gets it omitted from the reply, matching kitty. Exact-match ack + // parsers rely on this. + if let Some(placement_id) = placement_id.filter(|&id| id != 0) { + identifiers.push_str(&format!(",p={placement_id}")); + } + [ C1::APC, b"G", - format!("i={image_id};{message}").as_bytes(), + format!("{identifiers};{message}").as_bytes(), C1::ST, ] .concat() } -pub fn create_kitty_ok_reply(image_id: u32) -> Vec { - create_kitty_reply(image_id, "OK".to_string()) +pub fn create_kitty_ok_reply( + image_id: u32, + placement_id: Option, + image_number: Option, +) -> Vec { + create_kitty_reply(image_id, placement_id, image_number, "OK".to_string()) } -pub fn create_kitty_error_reply(image_id: u32, err: KittyError) -> Vec { - create_kitty_reply(image_id, format!("{err:?}")) +pub fn create_kitty_error_reply( + image_id: u32, + placement_id: Option, + image_number: Option, + err: KittyError, +) -> Vec { + let message = format!("{}{err:?}", kitty_error_code(&err)); + create_kitty_reply(image_id, placement_id, image_number, message) } diff --git a/app/src/terminal/model/kitty_test.rs b/app/src/terminal/model/kitty_test.rs new file mode 100644 index 00000000000..03e0e47d9f5 --- /dev/null +++ b/app/src/terminal/model/kitty_test.rs @@ -0,0 +1,790 @@ +use base64::engine::general_purpose::STANDARD as BASE64; +use warp_core::features::FeatureFlag; + +use crate::terminal::model::image_map::StoredImageMetadata; +use crate::terminal::model::index::Point; +use crate::terminal::model::kitty::MAX_ANIMATION_FRAMES; +use crate::terminal::model::TerminalModel; + +/// Builds a single-chunk kitty graphics APC message. +fn kitty_apc(control_data: &str, payload: &[u8]) -> String { + format!( + "\x1b_G{};{}\x1b\\", + control_data, + base64::Engine::encode(&BASE64, payload) + ) +} + +/// A one pixel, 24-bit RGB image, which is the smallest payload that passes +/// kitty's RGB size validation. +fn one_pixel_rgb() -> &'static [u8] { + &[0xff, 0x00, 0x00] +} + +/// A terminal with a running command, so that graphics land in a block's output +/// grid. Blocks that haven't started executing route to their header grid, which +/// doesn't handle kitty actions at all. +fn kitty_terminal() -> TerminalModel { + let mut terminal = TerminalModel::mock(None, None); + terminal.simulate_cmd("kitty"); + terminal +} + +/// Where the cursor sits in the block that graphics are landing in. +fn cursor_point(terminal: &TerminalModel) -> Point { + terminal + .block_list() + .active_block() + .grid_handler() + .cursor_point() +} + +/// The virtual (`U=1`) placements recorded for an image. +fn virtual_placement_ids(terminal: &TerminalModel, image_id: u32) -> Vec { + let Some(StoredImageMetadata::Kitty(metadata)) = terminal.image_id_to_metadata.get(&image_id) + else { + return Vec::new(); + }; + + let mut ids: Vec = metadata.virtual_placements.keys().copied().collect(); + ids.sort_unstable(); + ids +} + +fn reply_for(control_data: &str, payload: &[u8]) -> String { + let mut terminal = kitty_terminal(); + let written = terminal.process_bytes_capturing(kitty_apc(control_data, payload).as_str()); + String::from_utf8_lossy(&written).into_owned() +} + +/// Transmits and displays a one cell image at the cursor without moving the +/// cursor afterwards (`C=1`), so that tests control which cell it lands on. +fn place_image(terminal: &mut TerminalModel, control_data: &str) { + terminal.process_bytes( + kitty_apc( + &format!("a=T,f=24,s=1,v=1,C=1,{control_data}"), + one_pixel_rgb(), + ) + .as_str(), + ); +} + +/// A terminal holding a stored one pixel image with id 1, which animation +/// messages can then address. +fn terminal_with_stored_image() -> TerminalModel { + let mut terminal = kitty_terminal(); + terminal.process_bytes(kitty_apc("a=t,i=1,f=24,s=1,v=1", one_pixel_rgb()).as_str()); + terminal +} + +/// Sends an animation message and returns whatever was replied to the shell. +fn animate(terminal: &mut TerminalModel, control_data: &str, payload: &[u8]) -> String { + let written = terminal.process_bytes_capturing(kitty_apc(control_data, payload).as_str()); + String::from_utf8_lossy(&written).into_owned() +} + +/// The gaps of the animation frames recorded for an image, in playback order. +fn frame_gaps(terminal: &TerminalModel, image_id: u32) -> Vec { + let Some(StoredImageMetadata::Kitty(metadata)) = terminal.image_id_to_metadata.get(&image_id) + else { + return Vec::new(); + }; + + metadata.frames.iter().map(|&(_, gap)| gap).collect() +} + +/// Whether an image's animation is running. +fn is_playing(terminal: &TerminalModel, image_id: u32) -> bool { + matches!( + terminal.image_id_to_metadata.get(&image_id), + Some(StoredImageMetadata::Kitty(metadata)) if metadata.playing + ) +} + +/// Sends a delete message and returns whatever was replied to the shell. +fn delete(terminal: &mut TerminalModel, control_data: &str) -> String { + let written = + terminal.process_bytes_capturing(kitty_apc(&format!("a=d,{control_data}"), &[]).as_str()); + String::from_utf8_lossy(&written).into_owned() +} + +/// Whether the active block's grid still holds the given placement. +fn has_placement(terminal: &TerminalModel, image_id: u32, placement_id: u32) -> bool { + terminal + .block_list() + .active_block() + .grid_handler() + .get_image_placement_data(image_id, placement_id) + .is_some() +} + +/// A terminal holding two one cell placements: image 1 at the cell kitty calls +/// `x=1,y=1` and image 2 at `x=3,y=2`. The cursor is left on image 2's cell. +fn terminal_with_two_placements() -> TerminalModel { + let mut terminal = kitty_terminal(); + place_image(&mut terminal, "i=1,p=1"); + terminal.process_bytes("\r\n "); + place_image(&mut terminal, "i=2,p=2"); + terminal +} + +#[test] +fn zero_size_transmit_and_display_does_not_panic() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let reply = reply_for("a=T,i=1,f=24,s=0,v=0", &[]); + + // The action is a no-op, but it must still be acknowledged rather than panic. + assert!(reply.contains("i=1;OK"), "unexpected reply: {reply:?}"); +} + +#[test] +fn zero_size_display_of_stored_image_does_not_panic() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + terminal.process_bytes(kitty_apc("a=t,i=1,f=24,s=0,v=0", &[]).as_str()); + let written = terminal.process_bytes_capturing(kitty_apc("a=p,i=1", &[]).as_str()); + + let reply = String::from_utf8_lossy(&written); + assert!(reply.contains("i=1;OK"), "unexpected reply: {reply:?}"); +} + +#[test] +fn query_reply_is_sent_despite_quiet_mode() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let reply = reply_for("a=q,i=1,q=1,f=24,s=1,v=1", one_pixel_rgb()); + + assert!(reply.contains("i=1;OK"), "unexpected reply: {reply:?}"); +} + +#[test] +fn unknown_image_id_error_reply_uses_enoent() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let reply = reply_for("a=p,i=999", &[]); + + assert!( + reply.contains("i=999;ENOENT:"), + "unexpected reply: {reply:?}" + ); +} + +#[test] +fn ok_reply_echoes_image_and_placement_ids() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let reply = reply_for("a=T,i=7,p=3,f=24,s=1,v=1", one_pixel_rgb()); + + assert!(reply.contains("i=7,p=3;OK"), "unexpected reply: {reply:?}"); +} + +#[test] +fn quiet_mode_one_suppresses_ok_but_not_errors() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let ok_reply = reply_for("a=T,i=1,q=1,f=24,s=1,v=1", one_pixel_rgb()); + assert!(ok_reply.is_empty(), "unexpected reply: {ok_reply:?}"); + + let error_reply = reply_for("a=p,i=999,q=1", &[]); + assert!( + error_reply.contains("ENOENT:"), + "unexpected reply: {error_reply:?}" + ); +} + +#[test] +fn quiet_mode_two_suppresses_errors() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let error_reply = reply_for("a=p,i=999,q=2", &[]); + assert!(error_reply.is_empty(), "unexpected reply: {error_reply:?}"); + + let ok_reply = reply_for("a=T,i=1,q=2,f=24,s=1,v=1", one_pixel_rgb()); + assert!(ok_reply.is_empty(), "unexpected reply: {ok_reply:?}"); +} + +#[test] +fn delete_all_removes_every_placement() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = terminal_with_two_placements(); + delete(&mut terminal, "d=a"); + + assert!(!has_placement(&terminal, 1, 1)); + assert!(!has_placement(&terminal, 2, 2)); +} + +#[test] +fn delete_by_id_removes_only_that_image() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = terminal_with_two_placements(); + // A second placement of image 1, so we can tell "every placement of an + // image" apart from "one placement". + place_image(&mut terminal, "i=1,p=9"); + + delete(&mut terminal, "d=i,i=1"); + + assert!(!has_placement(&terminal, 1, 1)); + assert!(!has_placement(&terminal, 1, 9)); + assert!(has_placement(&terminal, 2, 2)); +} + +#[test] +fn delete_by_id_with_placement_id_removes_only_that_placement() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = terminal_with_two_placements(); + place_image(&mut terminal, "i=1,p=9"); + + delete(&mut terminal, "d=i,i=1,p=9"); + + assert!(has_placement(&terminal, 1, 1)); + assert!(!has_placement(&terminal, 1, 9)); + assert!(has_placement(&terminal, 2, 2)); +} + +#[test] +fn delete_by_number_removes_the_numbered_image() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + // `i=` still identifies the image; `I=` is the client's own number, which is + // what `d=n` selects on. + place_image(&mut terminal, "i=5,p=5,I=7"); + place_image(&mut terminal, "i=6,p=6"); + + delete(&mut terminal, "d=n,I=7"); + + assert!(!has_placement(&terminal, 5, 5)); + assert!(has_placement(&terminal, 6, 6)); +} + +#[test] +fn delete_at_cursor_removes_the_placement_under_the_cursor() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + // The helper leaves the cursor on image 2's cell. + let mut terminal = terminal_with_two_placements(); + delete(&mut terminal, "d=c"); + + assert!(has_placement(&terminal, 1, 1)); + assert!(!has_placement(&terminal, 2, 2)); +} + +#[test] +fn delete_at_point_removes_the_placement_in_that_cell() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = terminal_with_two_placements(); + delete(&mut terminal, "d=p,x=1,y=1"); + + assert!(!has_placement(&terminal, 1, 1)); + assert!(has_placement(&terminal, 2, 2)); +} + +#[test] +fn delete_at_point_with_z_index_only_removes_matching_depths() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + // Two placements stacked on the same cell at different depths. + let mut terminal = kitty_terminal(); + place_image(&mut terminal, "i=1,p=1,z=5"); + place_image(&mut terminal, "i=2,p=2,z=6"); + + delete(&mut terminal, "d=q,x=1,y=1,z=6"); + + assert!(has_placement(&terminal, 1, 1)); + assert!(!has_placement(&terminal, 2, 2)); +} + +#[test] +fn delete_in_column_removes_placements_in_that_column() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = terminal_with_two_placements(); + delete(&mut terminal, "d=x,x=3"); + + assert!(has_placement(&terminal, 1, 1)); + assert!(!has_placement(&terminal, 2, 2)); +} + +#[test] +fn delete_in_row_removes_placements_in_that_row() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = terminal_with_two_placements(); + delete(&mut terminal, "d=y,y=2"); + + assert!(has_placement(&terminal, 1, 1)); + assert!(!has_placement(&terminal, 2, 2)); +} + +#[test] +fn delete_by_z_index_removes_placements_at_that_depth() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + place_image(&mut terminal, "i=1,p=1,z=5"); + place_image(&mut terminal, "i=2,p=2"); + + delete(&mut terminal, "d=z,z=5"); + + assert!(!has_placement(&terminal, 1, 1)); + assert!(has_placement(&terminal, 2, 2)); +} + +#[test] +fn delete_by_id_range_removes_ids_within_the_range() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + for image_id in 1..=4 { + place_image(&mut terminal, &format!("i={image_id},p={image_id}")); + } + + delete(&mut terminal, "d=r,x=2,y=3"); + + assert!(has_placement(&terminal, 1, 1)); + assert!(!has_placement(&terminal, 2, 2)); + assert!(!has_placement(&terminal, 3, 3)); + assert!(has_placement(&terminal, 4, 4)); +} + +#[test] +fn unknown_delete_specifier_is_rejected_and_deletes_nothing() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + // Regression test: an unrecognized `d=` used to fall through to "delete all". + for specifier in ["d=w", "d=!"] { + let mut terminal = terminal_with_two_placements(); + let reply = delete(&mut terminal, &format!("{specifier},i=1")); + + assert!( + reply.contains("i=1;EINVAL:"), + "unexpected reply for {specifier}: {reply:?}" + ); + assert!( + has_placement(&terminal, 1, 1), + "{specifier} deleted image 1" + ); + assert!( + has_placement(&terminal, 2, 2), + "{specifier} deleted image 2" + ); + } +} + +#[test] +fn uppercase_delete_also_frees_the_image_data() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + place_image(&mut terminal, "i=1,p=1"); + delete(&mut terminal, "d=I,i=1"); + + // The image data is gone, so it can no longer be placed again. + let reply = String::from_utf8_lossy( + &terminal.process_bytes_capturing(kitty_apc("a=p,i=1", &[]).as_str()), + ) + .into_owned(); + assert!(reply.contains("i=1;ENOENT:"), "unexpected reply: {reply:?}"); + + // The lowercase form only drops placements, so the image survives. + let mut terminal = kitty_terminal(); + place_image(&mut terminal, "i=1,p=1"); + delete(&mut terminal, "d=i,i=1"); + + let reply = String::from_utf8_lossy( + &terminal.process_bytes_capturing(kitty_apc("a=p,i=1", &[]).as_str()), + ) + .into_owned(); + assert!(reply.contains("i=1;OK"), "unexpected reply: {reply:?}"); +} + +#[test] +fn uppercase_positional_delete_also_frees_the_image_data() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + place_image(&mut terminal, "i=1,p=1"); + delete(&mut terminal, "d=P,x=1,y=1"); + + let reply = String::from_utf8_lossy( + &terminal.process_bytes_capturing(kitty_apc("a=p,i=1", &[]).as_str()), + ) + .into_owned(); + assert!(reply.contains("i=1;ENOENT:"), "unexpected reply: {reply:?}"); +} + +#[test] +fn delete_frames_is_reported_as_unsupported() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + place_image(&mut terminal, "i=1,p=1"); + + let reply = delete(&mut terminal, "d=f,i=1"); + + assert!( + reply.contains("i=1;ENOTSUPP:"), + "unexpected reply: {reply:?}" + ); + // Animation frames are not stored yet, so nothing may be removed. + assert!(has_placement(&terminal, 1, 1)); +} + +#[test] +fn unicode_placeholder_transmit_and_display_is_accepted() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let reply = reply_for("a=T,U=1,i=1,p=2,f=24,s=1,v=1", one_pixel_rgb()); + + assert!(reply.contains("i=1,p=2;OK"), "unexpected reply: {reply:?}"); + assert!(!reply.contains("EINVAL"), "unexpected reply: {reply:?}"); +} + +#[test] +fn unicode_placeholder_display_of_stored_image_is_accepted() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + terminal.process_bytes(kitty_apc("a=t,i=1,f=24,s=1,v=1", one_pixel_rgb()).as_str()); + let written = terminal.process_bytes_capturing(kitty_apc("a=p,U=1,i=1,p=5", &[]).as_str()); + + let reply = String::from_utf8_lossy(&written); + assert!(reply.contains("i=1,p=5;OK"), "unexpected reply: {reply:?}"); + assert_eq!(virtual_placement_ids(&terminal, 1), vec![5]); +} + +#[test] +fn unicode_placeholder_placement_does_not_move_the_cursor() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + let before = cursor_point(&terminal); + + // A tall image: an anchored placement would scroll in rows of whitespace + // and leave the cursor past the image. + terminal.process_bytes(kitty_apc("a=T,U=1,i=1,r=4,c=4,f=24,s=1,v=1", one_pixel_rgb()).as_str()); + + assert_eq!( + cursor_point(&terminal), + before, + "a virtual placement must not move the cursor" + ); +} + +#[test] +fn unicode_placeholder_records_rows_and_columns_unresolved() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + terminal + .process_bytes(kitty_apc("a=T,U=1,i=3,p=1,r=2,c=6,f=24,s=1,v=1", one_pixel_rgb()).as_str()); + + let Some(StoredImageMetadata::Kitty(metadata)) = terminal.image_id_to_metadata.get(&3) else { + panic!("image 3 should have kitty metadata"); + }; + let placement = metadata + .virtual_placements + .get(&1) + .expect("placement 1 should be recorded"); + + // Left unresolved so that a font size change re-tiles the image. + assert_eq!(placement.rows, Some(2)); + assert_eq!(placement.cols, Some(6)); +} + +#[test] +fn anchored_placement_records_no_virtual_placement() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + terminal.process_bytes(kitty_apc("a=T,i=1,p=2,f=24,s=1,v=1", one_pixel_rgb()).as_str()); + + assert!(virtual_placement_ids(&terminal, 1).is_empty()); +} + +#[test] +fn extreme_aspect_ratio_display_does_not_underflow() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + // A 4000x1 image squeezed into one column: the desired height truncates to + // zero cells, which used to compute `0usize - 1` in the newline loop and + // turn it into an unbounded line-append in release builds. + let pixels = vec![0u8; 4000 * 3]; + let written = terminal + .process_bytes_capturing(kitty_apc("a=T,i=7,c=1,f=24,s=4000,v=1", &pixels).as_str()); + + let reply = String::from_utf8_lossy(&written); + assert!(reply.contains("i=7"), "unexpected reply: {reply:?}"); +} + +#[test] +fn query_is_answered_before_a_command_starts_executing() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + // No `simulate_cmd`: the block is still before `preexec`, so grid-bound + // actions route to the header grid. A support probe must be answered anyway. + let mut terminal = TerminalModel::mock(None, None); + let written = terminal + .process_bytes_capturing(kitty_apc("a=q,i=31,f=24,s=1,v=1", one_pixel_rgb()).as_str()); + + let reply = String::from_utf8_lossy(&written); + assert!(reply.contains("i=31;OK"), "unexpected reply: {reply:?}"); +} + +#[test] +fn transmitted_frames_accumulate_with_their_gaps() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = terminal_with_stored_image(); + + for gap in [100, 200, 300] { + let reply = animate( + &mut terminal, + &format!("a=f,i=1,f=24,s=1,v=1,z={gap}"), + one_pixel_rgb(), + ); + assert!(reply.contains("i=1;OK"), "unexpected reply: {reply:?}"); + } + + // The stored image is frame 1 of the animation, so only the three + // transmitted frames are recorded. + assert_eq!(frame_gaps(&terminal, 1), vec![100, 200, 300]); +} + +#[test] +fn animation_control_starts_and_stops_playback() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = terminal_with_stored_image(); + animate(&mut terminal, "a=f,i=1,f=24,s=1,v=1,z=100", one_pixel_rgb()); + + // Frames are transmitted to be played, so the first one starts the animation. + assert!(is_playing(&terminal, 1)); + + let reply = animate(&mut terminal, "a=a,i=1,s=1", &[]); + assert!(reply.contains("i=1;OK"), "unexpected reply: {reply:?}"); + assert!(!is_playing(&terminal, 1)); + + animate(&mut terminal, "a=a,i=1,s=2", &[]); + assert!(is_playing(&terminal, 1)); +} + +#[test] +fn animation_control_edits_a_frame_gap() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = terminal_with_stored_image(); + animate(&mut terminal, "a=f,i=1,f=24,s=1,v=1,z=100", one_pixel_rgb()); + animate(&mut terminal, "a=f,i=1,f=24,s=1,v=1,z=200", one_pixel_rgb()); + + // Frame 2 is the first transmitted frame; frame 1 is the stored image. + let reply = animate(&mut terminal, "a=a,i=1,r=2,z=500", &[]); + + assert!(reply.contains("i=1;OK"), "unexpected reply: {reply:?}"); + assert_eq!(frame_gaps(&terminal, 1), vec![500, 200]); +} + +#[test] +fn frames_needing_compositing_are_unsupported() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = terminal_with_stored_image(); + + // A frame placed at an offset, a frame built on top of another frame, and a + // frame smaller than the canvas all have to be composited. + for control_data in [ + "a=f,i=1,f=24,s=1,v=1,x=4", + "a=f,i=1,f=24,s=1,v=1,y=4", + "a=f,i=1,f=24,s=1,v=1,c=1", + ] { + let reply = animate(&mut terminal, control_data, one_pixel_rgb()); + assert!( + reply.contains("i=1;ENOTSUPP:"), + "unexpected reply to {control_data:?}: {reply:?}" + ); + } + + let reply = animate( + &mut terminal, + "a=f,i=1,f=24,s=2,v=1", + &[0xff, 0x00, 0x00, 0x00, 0xff, 0x00], + ); + assert!( + reply.contains("i=1;ENOTSUPP:"), + "unexpected reply: {reply:?}" + ); + + assert!(frame_gaps(&terminal, 1).is_empty()); +} + +#[test] +fn compose_action_is_unsupported() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let reply = reply_for("a=c,i=1", &[]); + + assert!( + reply.contains("i=1;ENOTSUPP:"), + "unexpected reply: {reply:?}" + ); +} + +#[test] +fn zero_image_number_gets_no_reply() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + // `I=0` is the unset default of client libraries that always emit the key; + // replying would hand them an id they never asked about. + let reply = reply_for("a=T,I=0,f=24,s=1,v=1", one_pixel_rgb()); + + assert!(reply.is_empty(), "unexpected reply: {reply:?}"); +} + +#[test] +fn animation_messages_resolve_the_client_image_number() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + terminal.process_bytes(kitty_apc("a=t,i=1,I=5,f=24,s=1,v=1", one_pixel_rgb()).as_str()); + + let reply = animate(&mut terminal, "a=f,I=5,f=24,s=1,v=1", one_pixel_rgb()); + + assert!(reply.contains("i=1,I=5;OK"), "unexpected reply: {reply:?}"); + assert_eq!(frame_gaps(&terminal, 1).len(), 1); +} + +#[test] +fn frame_edit_of_a_missing_frame_is_an_error_not_an_append() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = terminal_with_stored_image(); + + let reply = animate(&mut terminal, "a=f,i=1,r=7,f=24,s=1,v=1", one_pixel_rgb()); + + assert!(reply.contains("i=1;ENOENT:"), "unexpected reply: {reply:?}"); + assert!(frame_gaps(&terminal, 1).is_empty()); +} + +#[test] +fn explicit_zero_base_frame_is_accepted() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = terminal_with_stored_image(); + + // `c=0` means "no base frame": nothing to composite, so nothing to reject. + let reply = animate(&mut terminal, "a=f,i=1,c=0,f=24,s=1,v=1", one_pixel_rgb()); + + assert!(reply.contains("i=1;OK"), "unexpected reply: {reply:?}"); + assert_eq!(frame_gaps(&terminal, 1).len(), 1); +} + +#[test] +fn animation_frames_are_capped() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = terminal_with_stored_image(); + for _ in 0..MAX_ANIMATION_FRAMES { + terminal.process_bytes(kitty_apc("a=f,i=1,q=1,f=24,s=1,v=1", one_pixel_rgb()).as_str()); + } + assert_eq!(frame_gaps(&terminal, 1).len(), MAX_ANIMATION_FRAMES); + + let reply = animate(&mut terminal, "a=f,i=1,f=24,s=1,v=1", one_pixel_rgb()); + + assert!( + reply.contains("i=1;ENOTSUPP:"), + "unexpected reply: {reply:?}" + ); + assert_eq!(frame_gaps(&terminal, 1).len(), MAX_ANIMATION_FRAMES); +} + +#[test] +fn reply_echoes_the_client_image_number() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + // No `i=`: the reply is how the client learns the id the terminal assigned + // to its number. + let reply = reply_for("a=T,I=9,f=24,s=1,v=1", one_pixel_rgb()); + + assert!(reply.contains(",I=9;OK"), "unexpected reply: {reply:?}"); + assert!(reply.contains("i="), "unexpected reply: {reply:?}"); +} + +#[test] +fn delete_all_clears_virtual_placements_without_freeing_the_image() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + terminal.process_bytes(kitty_apc("a=T,U=1,i=1,p=5,f=24,s=1,v=1", one_pixel_rgb()).as_str()); + + delete(&mut terminal, "d=a"); + + assert!(virtual_placement_ids(&terminal, 1).is_empty()); + // Lowercase `d=a` removes placements, not the stored image data. + assert!(terminal.image_id_to_metadata.contains_key(&1)); +} + +#[test] +fn delete_by_id_clears_virtual_placements() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + terminal.process_bytes(kitty_apc("a=T,U=1,i=1,p=5,f=24,s=1,v=1", one_pixel_rgb()).as_str()); + terminal.process_bytes(kitty_apc("a=p,U=1,i=1,p=6", &[]).as_str()); + + delete(&mut terminal, "d=i,i=1,p=5"); + assert_eq!(virtual_placement_ids(&terminal, 1), vec![6]); + + delete(&mut terminal, "d=i,i=1"); + assert!(virtual_placement_ids(&terminal, 1).is_empty()); +} + +#[test] +fn delete_by_z_index_clears_matching_virtual_placements() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + terminal.process_bytes(kitty_apc("a=T,U=1,i=1,p=5,z=3,f=24,s=1,v=1", one_pixel_rgb()).as_str()); + terminal.process_bytes(kitty_apc("a=p,U=1,i=1,p=6,z=4", &[]).as_str()); + + delete(&mut terminal, "d=z,z=3"); + + assert_eq!(virtual_placement_ids(&terminal, 1), vec![6]); +} + +#[test] +fn uppercase_z_delete_evicts_every_placement_of_a_freed_image() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + place_image(&mut terminal, "i=1,p=1,z=5"); + place_image(&mut terminal, "i=1,p=2,z=6"); + + // Freeing image 1 by z-index must not leave the z=6 placement behind as a + // blank hole. + delete(&mut terminal, "d=Z,z=5"); + + assert!(!has_placement(&terminal, 1, 1)); + assert!(!has_placement(&terminal, 1, 2)); + assert!(!terminal.image_id_to_metadata.contains_key(&1)); +} + +#[test] +fn uppercase_delete_by_id_with_placement_frees_the_whole_image() { + let _kitty_images = FeatureFlag::KittyImages.override_enabled(true); + + let mut terminal = kitty_terminal(); + place_image(&mut terminal, "i=1,p=1"); + place_image(&mut terminal, "i=1,p=2"); + + delete(&mut terminal, "d=I,i=1,p=1"); + + assert!(!has_placement(&terminal, 1, 1)); + assert!(!has_placement(&terminal, 1, 2)); +} diff --git a/app/src/terminal/model/mod.rs b/app/src/terminal/model/mod.rs index dc074d94e12..faee9e22571 100644 --- a/app/src/terminal/model/mod.rs +++ b/app/src/terminal/model/mod.rs @@ -31,6 +31,9 @@ pub mod image_map; pub mod index; pub mod iterm_image; pub mod kitty; +#[cfg(test)] +#[path = "kitty_test.rs"] +mod kitty_test; pub mod secrets; pub mod selection; pub mod session; diff --git a/app/src/terminal/model/terminal_model.rs b/app/src/terminal/model/terminal_model.rs index cae00e73b9a..c152c01068d 100644 --- a/app/src/terminal/model/terminal_model.rs +++ b/app/src/terminal/model/terminal_model.rs @@ -39,8 +39,9 @@ use super::grid::grid_handler::{ use super::image_map::StoredImageMetadata; use super::index::Point; use super::kitty::{ - create_kitty_error_reply, create_kitty_ok_reply, DeletionType, KittyAction, KittyChunk, - KittyMessage, KittyResponse, PendingKittyMessage, + create_kitty_error_reply, create_kitty_ok_reply, frame_index, DeletionType, InvalidKittyAction, + KittyAction, KittyChunk, KittyError, KittyMessage, KittyPlacementAction, KittyResponse, + PendingKittyMessage, StorageError, MAX_ANIMATION_FRAMES, MAX_ANIMATION_FRAME_BYTES, }; use super::secrets::{RespectObfuscatedSecrets, SecretAndHandle}; use super::selection::ScrollDelta; @@ -76,7 +77,7 @@ use instant::Instant; use itertools::{Either, Itertools}; use serde::Serialize; use std::cmp::{max, min}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::num::ParseIntError; use std::ops::{Range, RangeInclusive}; use std::path::PathBuf; @@ -1679,6 +1680,173 @@ impl TerminalModel { self.image_id_to_metadata.remove(&image_id); } + /// Runs `evict` against every grid that can hold images: the alternate screen + /// while it is active, otherwise every block's grid. + fn for_each_image_grid(&mut self, mut evict: impl FnMut(&mut GridHandler)) { + if self.alt_screen_active { + evict(self.alt_screen.grid_handler_mut()); + } else { + for block in self.block_list_mut().blocks_mut() { + evict(block.grid_handler_mut()); + } + } + } + + /// Removes a kitty image's placements, or only the given placement, and + /// frees the stored image data when `delete_image_data` is set. + fn delete_kitty_image( + &mut self, + image_id: u32, + placement_id: Option, + delete_image_data: bool, + ) { + if delete_image_data { + // Freeing the image data frees every placement of the image: one + // left in a grid would draw nothing once the metadata is gone. + self.image_id_to_metadata.remove(&image_id); + self.for_each_image_grid(|grid| grid.evict_image(image_id)); + return; + } + + // Virtual (`U=1`) placements live in the metadata rather than any grid. + if let Some(StoredImageMetadata::Kitty(metadata)) = + self.image_id_to_metadata.get_mut(&image_id) + { + match placement_id { + Some(placement_id) => { + metadata.virtual_placements.remove(&placement_id); + } + None => metadata.virtual_placements.clear(), + } + } + + self.for_each_image_grid(|grid| match placement_id { + Some(placement_id) => grid.evict_placement(image_id, placement_id), + None => grid.evict_image(image_id), + }); + } + + /// The id of the newest image transmitted under the given `I=` number. Ids + /// are handed out in increasing order, so the newest is the largest. + fn newest_kitty_image_with_number(&self, image_number: u32) -> Option { + self.image_id_to_metadata + .iter() + .filter(|(_, metadata)| match metadata { + StoredImageMetadata::Kitty(metadata) => metadata.image_number == Some(image_number), + StoredImageMetadata::ITerm(_) => false, + }) + .map(|(image_id, _)| *image_id) + .max() + } + + /// Applies an `a=f` frame transmission or an `a=a` playback control to the + /// image it names, then hands the resulting frame list to the view so it can + /// rebuild the image's asset. Sending no frames leaves the image on frame 1, + /// which is how a stopped animation is expressed. + fn handle_kitty_animation_action(&mut self, action: KittyAction) -> Result<(), KittyError> { + if !FeatureFlag::KittyImages.is_enabled() { + return Err(KittyError::KittyFeatureDisabled); + } + + let image_id = match &action { + KittyAction::TransmitFrame { image_id, .. } + | KittyAction::AnimationControl { image_id, .. } => *image_id, + // No other action reaches this function. + _ => return Ok(()), + }; + + let Some(StoredImageMetadata::Kitty(metadata)) = + self.image_id_to_metadata.get_mut(&image_id) + else { + return Err(StorageError::UnknownId { id: image_id }.into()); + }; + + match action { + KittyAction::TransmitFrame { + frame_number, + gap_ms, + image, + .. + } => { + // A frame smaller than the canvas has to be composited onto what + // is already there, which is not implemented. + if image.metadata.image_size != metadata.image_size { + return Err(InvalidKittyAction::UnsupportedAction.into()); + } + + let edited = match frame_number.filter(|&number| number != 0) { + // Frame 1 is the root image; replacing its pixels means + // replacing the image itself, not a frame. + Some(1) => return Err(InvalidKittyAction::UnsupportedAction.into()), + Some(frame_number) => { + // `r=` names an existing frame; a number past the end + // is an error, not an append. + let index = frame_index(frame_number) + .filter(|&index| index < metadata.frames.len()) + .ok_or(KittyError::from(StorageError::UnknownId { + id: frame_number, + }))?; + metadata.frames.get_mut(index) + } + None => None, + }; + + match edited { + Some(frame) => *frame = (image.data, gap_ms), + None => { + let stored_bytes: usize = + metadata.frames.iter().map(|(data, _)| data.len()).sum(); + if metadata.frames.len() >= MAX_ANIMATION_FRAMES + || stored_bytes + image.data.len() > MAX_ANIMATION_FRAME_BYTES + { + return Err(InvalidKittyAction::UnsupportedAction.into()); + } + metadata.frames.push((image.data, gap_ms)); + } + } + + metadata.playing = true; + } + KittyAction::AnimationControl { play, gap_edit, .. } => { + let mut changed = false; + + if let Some(play) = play { + changed |= metadata.playing != play; + metadata.playing = play; + } + + // A gap edit naming a frame that was never transmitted is + // ignored, as is one naming the root frame, whose gap is not + // tracked. + if let Some((frame_number, gap_ms)) = gap_edit { + if let Some(frame) = + frame_index(frame_number).and_then(|index| metadata.frames.get_mut(index)) + { + changed |= frame.1 != gap_ms; + frame.1 = gap_ms; + } + } + + // A no-op control message must not trigger an asset rebuild. + if !changed { + return Ok(()); + } + } + _ => return Ok(()), + } + + let frames = if metadata.playing { + metadata.frames.clone() + } else { + Vec::new() + }; + + self.event_proxy + .send_terminal_event(Event::AnimatedImageReceived { image_id, frames }); + + Ok(()) + } + /// Starts the active block and resets block-to-block state. For local sessions, this is called /// from the input editor when it sends user bytes to the pty (usually the /// next command to run, but also ctrl-d). Once we've written to the pty on @@ -3405,9 +3573,36 @@ impl ansi::Handler for TerminalModel { }; let message_id = pending.control_data.image_id; + let placement_id = pending.control_data.placement_id; + let image_number = pending.control_data.image_number; let verbosity = pending.control_data.verbosity; + // Query replies are the only way a client can detect support, so they are + // sent regardless of the requested verbosity. This is a deliberate + // deviation: kitty itself honors `q=2` even for queries. + let is_query = matches!( + pending.control_data.placement_action, + KittyPlacementAction::QuerySupport + ); + let send_ok = is_query || verbosity.send_ok(); + let send_error = is_query || verbosity.send_error(); + + // A message that names its image only by client number (`I=`, no `i=`) + // refers to the newest image transmitted under that number, unless it + // is itself a transmission, which allocates a fresh image below. + if pending.control_data.image_id.is_none() { + if let Some(number) = image_number.filter(|&number| number != 0) { + if matches!( + pending.control_data.placement_action, + KittyPlacementAction::DisplayStoredImage + | KittyPlacementAction::TransmitFrame + | KittyPlacementAction::AnimationControl + ) { + pending.control_data.image_id = self.newest_kitty_image_with_number(number); + } + } + } - if message_id.is_none() { + if pending.control_data.image_id.is_none() { pending.control_data.image_id = Some(self.next_kitty_image_id); self.next_kitty_image_id = self.next_kitty_image_id.wrapping_add(1); // 0 is an invalid ID for kitty images @@ -3416,13 +3611,28 @@ impl ansi::Handler for TerminalModel { } } + // A message that carries only a client-side number (`I=`, no `i=`) + // still gets a reply: the reply is how the client learns the id the + // terminal assigned to its number. `I=0` is the unset default and gets + // no reply, matching how the reply builder omits it. + let message_id = message_id.or_else(|| { + image_number + .filter(|&number| number != 0) + .and(pending.control_data.image_id) + }); + let message = match KittyMessage::try_from(pending) { Ok(message) => message, Err(err) => { log::warn!("{err:?}"); if let Some(message_id) = message_id { - if verbosity.send_error() { - let _ = writer.write_all(&create_kitty_error_reply(message_id, err.into())); + if send_error { + let _ = writer.write_all(&create_kitty_error_reply( + message_id, + placement_id, + image_number, + err.into(), + )); } } return; @@ -3431,6 +3641,59 @@ impl ansi::Handler for TerminalModel { match KittyAction::try_from(message) { Ok(action) => { + // Queries never touch the grid, so answer them before the block + // delegate: a block still before `preexec` routes kitty actions + // to its header grid, which drops them without a reply, and a + // support probe must never go unanswered. Decode failures were + // already answered above. + if matches!(action, KittyAction::QuerySupport(_)) { + if let Some(message_id) = message_id { + let _ = writer.write_all(&create_kitty_ok_reply( + message_id, + placement_id, + image_number, + )); + } + return; + } + + // Animation actions change an image's frames, never the grid, so + // they are applied and answered here for the same reason queries + // are: a block that has not reached `preexec` routes grid-bound + // actions to its header grid, which drops them without a reply. + if matches!( + action, + KittyAction::TransmitFrame { .. } | KittyAction::AnimationControl { .. } + ) { + match self.handle_kitty_animation_action(action) { + Ok(()) => { + if let Some(message_id) = message_id { + if send_ok { + let _ = writer.write_all(&create_kitty_ok_reply( + message_id, + placement_id, + image_number, + )); + } + } + } + Err(err) => { + log::warn!("{err:?}"); + if let Some(message_id) = message_id { + if send_error { + let _ = writer.write_all(&create_kitty_error_reply( + message_id, + placement_id, + image_number, + err, + )); + } + } + } + } + return; + } + match &action { KittyAction::StoreOnly(action) => { self.image_id_to_metadata.insert( @@ -3439,85 +3702,197 @@ impl ansi::Handler for TerminalModel { ); } KittyAction::StoreAndDisplay(action) => { - self.image_id_to_metadata.insert( - action.image_id, - StoredImageMetadata::Kitty(action.image.metadata.clone()), - ); + let mut metadata = action.image.metadata.clone(); + if action.placement_data.unicode_placeholder { + metadata.virtual_placements.insert( + action.placement_id, + action.placement_data.virtual_placement(), + ); + } + self.image_id_to_metadata + .insert(action.image_id, StoredImageMetadata::Kitty(metadata)); + } + KittyAction::DisplayStoredImage(action) => { + // A virtual placement of an already-transmitted image only + // adds to that image's metadata; the cells referencing it + // supply the position. + if action.placement_data.unicode_placeholder { + if let Some(StoredImageMetadata::Kitty(metadata)) = + self.image_id_to_metadata.get_mut(&action.image_id) + { + metadata.virtual_placements.insert( + action.placement_id, + action.placement_data.virtual_placement(), + ); + } + } } - KittyAction::DisplayStoredImage(_) => {} - KittyAction::QuerySupport(_) => {} + // Both are answered above, before this match. + KittyAction::QuerySupport(_) + | KittyAction::TransmitFrame { .. } + | KittyAction::AnimationControl { .. } => {} KittyAction::Delete { delete_placements_only, deletion_type, - } => match deletion_type { - DeletionType::DeleteAll => { - if !delete_placements_only { - self.image_id_to_metadata.clear(); - } + } => { + let delete_image_data = !delete_placements_only; - if self.alt_screen_active { - self.alt_screen.grid_handler_mut().evict_all_images(); - } else { - for block in self.block_list_mut().blocks_mut() { - block.grid_handler_mut().evict_all_images(); + match deletion_type { + DeletionType::All => { + if delete_image_data { + self.image_id_to_metadata.clear(); + } else { + // `d=a` removes placements only, and virtual + // (`U=1`) placements live in the metadata + // rather than any grid. + for metadata in self.image_id_to_metadata.values_mut() { + if let StoredImageMetadata::Kitty(metadata) = metadata { + metadata.virtual_placements.clear(); + } + } } + + self.for_each_image_grid(|grid| grid.evict_all_images()); } - } - DeletionType::DeleteById(delete_by_id) => { - if !delete_placements_only { - self.image_id_to_metadata.remove(&delete_by_id.image_id); + DeletionType::ById { + image_id, + placement_id, + } => { + self.delete_kitty_image( + *image_id, + *placement_id, + delete_image_data, + ); } + DeletionType::ByNumber { + image_number, + placement_id, + } => { + // Nothing to do when no image was transmitted + // under that number; kitty treats deleting an + // absent image as a no-op. + if let Some(image_id) = + self.newest_kitty_image_with_number(*image_number) + { + self.delete_kitty_image( + image_id, + *placement_id, + delete_image_data, + ); + } + } + DeletionType::IdRange { start, end } => { + let range = *start..=*end; - if self.alt_screen_active { - if let Some(placement_id) = delete_by_id.placement_id { - self.alt_screen - .grid_handler_mut() - .evict_placement(delete_by_id.image_id, placement_id); - } else { - self.alt_screen - .grid_handler_mut() - .evict_image(delete_by_id.image_id); + if delete_image_data { + self.image_id_to_metadata + .retain(|image_id, _| !range.contains(image_id)); + } + + self.for_each_image_grid(|grid| { + grid.evict_placements_in_id_range(*start, *end) + }); + } + DeletionType::ZIndex(z_index) => { + let mut evicted = vec![]; + self.for_each_image_grid(|grid| { + evicted.extend(grid.evict_placements_with_z(*z_index)) + }); + let mut affected: Vec = + evicted.into_iter().map(|(image_id, _)| image_id).collect(); + + // Virtual (`U=1`) placements carry their own z + // and live in the metadata rather than any grid. + for (image_id, metadata) in self.image_id_to_metadata.iter_mut() { + if let StoredImageMetadata::Kitty(metadata) = metadata { + let before = metadata.virtual_placements.len(); + metadata + .virtual_placements + .retain(|_, placement| placement.z_index != *z_index); + if metadata.virtual_placements.len() != before { + affected.push(*image_id); + } + } } - } else { - for block in self.block_list_mut().blocks_mut() { - if let Some(placement_id) = delete_by_id.placement_id { - block - .grid_handler_mut() - .evict_placement(delete_by_id.image_id, placement_id); - } else { - block.grid_handler_mut().evict_image(delete_by_id.image_id); + + if delete_image_data { + for image_id in affected { + // Freeing an image frees every placement + // of it, not just the matched ones. + self.image_id_to_metadata.remove(&image_id); + self.for_each_image_grid(|grid| grid.evict_image(image_id)); } } } + // The positional specifiers need a cursor and cell + // geometry, so the grid handler applies them. + DeletionType::AtCursor + | DeletionType::AtPoint { .. } + | DeletionType::AtPointZ { .. } + | DeletionType::Column(_) + | DeletionType::Row(_) + | DeletionType::Frames { .. } => {} } - }, + } } match self.handle_completed_kitty_action(action.clone(), &mut HashMap::new()) { Some(Ok(_)) => { if let Some(message_id) = message_id { - if verbosity.send_ok() { - let _ = writer.write_all(&create_kitty_ok_reply(message_id)); + if send_ok { + let _ = writer.write_all(&create_kitty_ok_reply( + message_id, + placement_id, + image_number, + )); } } } Some(Err(err)) => { log::warn!("{err:?}"); if let Some(message_id) = message_id { - if verbosity.send_error() { - let _ = - writer.write_all(&create_kitty_error_reply(message_id, err)); + if send_error { + let _ = writer.write_all(&create_kitty_error_reply( + message_id, + placement_id, + image_number, + err, + )); } } } None => {} }; + + // An uppercase positional delete frees image data based on what + // the active grid held, but other placements of the freed + // images would draw as blank gaps. Sweep them out of the grids + // this delete can reach (the alt screen when active, the block + // grids otherwise — the same reach as every delete above). + if let KittyAction::Delete { + delete_placements_only: false, + deletion_type: + DeletionType::AtCursor + | DeletionType::AtPoint { .. } + | DeletionType::AtPointZ { .. } + | DeletionType::Column(_) + | DeletionType::Row(_), + } = &action + { + let live: HashSet = self.image_id_to_metadata.keys().copied().collect(); + self.for_each_image_grid(|grid| grid.evict_images_absent_from(&live)); + } } Err(err) => { log::warn!("{err:?}"); if let Some(message_id) = message_id { - if verbosity.send_error() { - let _ = writer.write_all(&create_kitty_error_reply(message_id, err)); + if send_error { + let _ = writer.write_all(&create_kitty_error_reply( + message_id, + placement_id, + image_number, + err, + )); } } } diff --git a/app/src/terminal/model/test_utils.rs b/app/src/terminal/model/test_utils.rs index 18f6a23f192..0b56c085e08 100644 --- a/app/src/terminal/model/test_utils.rs +++ b/app/src/terminal/model/test_utils.rs @@ -6,7 +6,7 @@ //! is marked as no_run only because it's currently not possible //! to reference `#[cfg(test)]` symbols from doctests. -use std::{io::sink, sync::Arc}; +use std::sync::Arc; use warp_core::command::ExitCode; use warpui::r#async::executor::Background; @@ -336,15 +336,17 @@ impl TerminalModel { /// Processes a set of `bytes` and applies them to the model, /// akin to what happens when reading bytes from a real PTY. pub fn process_bytes(&mut self, bytes: B) { + self.process_bytes_capturing(bytes); + } + + /// Same as [`Self::process_bytes`], but returns the bytes that would have + /// been written back to the shell, so that replies can be asserted on. + pub fn process_bytes_capturing(&mut self, bytes: B) -> Vec { let bytes = bytes.as_bytes(); let mut processor = Processor::new(); - processor.parse_bytes( - self, - bytes, - // For unit tests, there's no shell to write back to - // so the writes should no-op. - &mut sink(), - ); + let mut written = Vec::new(); + processor.parse_bytes(self, bytes, &mut written); + written } } diff --git a/app/src/terminal/model_events.rs b/app/src/terminal/model_events.rs index b76e5d4aca9..b43f078a4a6 100644 --- a/app/src/terminal/model_events.rs +++ b/app/src/terminal/model_events.rs @@ -292,6 +292,9 @@ impl ModelEventDispatcher { image_data, image_protocol, }, + Event::AnimatedImageReceived { image_id, frames } => { + ModelEvent::AnimatedImageReceived { image_id, frames } + } Event::BootstrapPrecmdDone => ModelEvent::BootstrapPrecmdDone, Event::AgentTaggedInChanged { is_tagged_in } => { ModelEvent::AgentTaggedInChanged { is_tagged_in } @@ -461,6 +464,11 @@ pub enum ModelEvent { image_data: Vec, image_protocol: ImageProtocol, }, + /// See [`Event::AnimatedImageReceived`]. + AnimatedImageReceived { + image_id: u32, + frames: Vec<(Vec, u32)>, + }, BootstrapPrecmdDone, AgentTaggedInChanged { is_tagged_in: bool, diff --git a/app/src/terminal/view.rs b/app/src/terminal/view.rs index 131b1e78c55..5e91ce6e6bc 100644 --- a/app/src/terminal/view.rs +++ b/app/src/terminal/view.rs @@ -376,8 +376,8 @@ use warpui::platform::{Cursor, OperatingSystem}; use warpui::r#async::Timer; use warpui::windowing::WindowManager; -use warpui::assets::asset_cache::{AssetCache, AssetCacheEvent}; -use warpui::image_cache::ImageType; +use warpui::assets::asset_cache::{Asset as _, AssetCache, AssetCacheEvent, AssetSource, AssetState}; +use warpui::image_cache::{AnimatedImage, ImageType, StaticImage}; use warpui::units::{IntoLines, IntoPixels, Lines, Pixels}; use warpui::{ accessibility::{AccessibilityContent, ActionAccessibilityContent, WarpA11yRole}, @@ -509,6 +509,7 @@ use super::model::block::{ }; use super::model::blocks::RichContentItem; use super::model::completions::ShellCompletion; +use super::model::kitty::DEFAULT_FRAME_GAP_MS; use super::model::rich_content::RichContentType; use super::model::secrets::RichContentSecretTooltipInfo; use super::model::selection::ExpandedSelectionRange; @@ -10853,6 +10854,16 @@ impl TerminalView { ctx ); } + ModelEvent::AnimatedImageReceived { image_id, frames } => { + AssetCache::handle(ctx).update(ctx, |asset_cache, ctx| { + let Some(asset) = build_animated_image(asset_cache, *image_id, frames) else { + return; + }; + + asset_cache.insert_asset::(image_id.to_string(), asset, ctx); + }); + ctx.notify(); + } ModelEvent::BootstrapPrecmdDone => { self.execute_pending_command((), ctx); } @@ -12109,6 +12120,81 @@ fn build_onboarding_keybindings(ctx: &AppContext) -> OnboardingKeybindings { } } +/// Builds the asset for a kitty image whose animation frames changed. Frame 1 of +/// the animation is the image itself, which the asset cache already holds, so +/// only the frames transmitted after it travel with the event. +fn build_animated_image( + asset_cache: &AssetCache, + image_id: u32, + frames: &[(Vec, u32)], +) -> Option { + let mut images: Vec<(Arc, u32)> = Vec::new(); + + if let Some(root) = cached_image(asset_cache, image_id) { + images.push((root, DEFAULT_FRAME_GAP_MS)); + } + + for (data, gap_ms) in frames { + match ImageType::try_from_bytes(data) { + Ok(ImageType::StaticBitmap { image }) => images.push((image, *gap_ms)), + Ok(_) | Err(_) => { + log::warn!("Could not decode an animation frame of kitty image {image_id}"); + } + } + } + + match &images[..] { + [] => None, + // A stopped animation is a single frame. Keeping it static spares the + // renderer a repaint for every frame delay that elapses. + [(image, _)] => Some(ImageType::StaticBitmap { + image: image.clone(), + }), + _ => { + let frames: Vec = images + .iter() + .filter_map(|(image, gap_ms)| animation_frame(image, *gap_ms)) + .collect(); + + Some(ImageType::AnimatedBitmap { + image: Arc::new(AnimatedImage::from(frames)), + }) + } + } +} + +/// The decoded image the asset cache holds for an image id. Once an animation +/// has been built for it, its first frame is that same image. +fn cached_image(asset_cache: &AssetCache, image_id: u32) -> Option> { + let AssetState::Loaded { data } = asset_cache.load_asset::(AssetSource::Raw { + id: image_id.to_string(), + }) else { + return None; + }; + + match &*data { + ImageType::StaticBitmap { image } => Some(image.clone()), + ImageType::AnimatedBitmap { image } => { + image.frames.first().map(|frame| frame.image.clone()) + } + ImageType::Svg { .. } | ImageType::Unrecognized => None, + } +} + +/// Copies a decoded image into an `image` crate frame, which is what +/// [`AnimatedImage`] is built from. +fn animation_frame(image: &StaticImage, gap_ms: u32) -> Option { + let buffer = + image::RgbaImage::from_raw(image.width(), image.height(), image.rgba_bytes().to_vec())?; + + Some(image::Frame::from_parts( + buffer, + 0, + 0, + image::Delay::from_numer_denom_ms(gap_ms, 1), + )) +} + /// Builds the context-menu label for forking an AI conversation from a given query. fn fork_label_for_query(query: &str) -> String { if query.is_empty() { diff --git a/crates/warpui/src/platform/mac/rendering/metal/renderer.rs b/crates/warpui/src/platform/mac/rendering/metal/renderer.rs index c81056517aa..d3495c9deed 100644 --- a/crates/warpui/src/platform/mac/rendering/metal/renderer.rs +++ b/crates/warpui/src/platform/mac/rendering/metal/renderer.rs @@ -331,6 +331,7 @@ impl<'a> Frame<'a> { let is_icon; let icon_color; let ui_corner_radius; + let source_uv; if let Some(to_render) = image { opacity = to_render.opacity; @@ -339,6 +340,7 @@ impl<'a> Frame<'a> { is_icon = false; icon_color = ColorF::new(0.0, 0.0, 0.0, opacity).into(); ui_corner_radius = to_render.corner_radius; + source_uv = to_render.source_uv; } else { let to_render = icon.unwrap(); opacity = to_render.opacity; @@ -347,6 +349,7 @@ impl<'a> Frame<'a> { is_icon = true; icon_color = to_render.color.to_f32().into(); ui_corner_radius = CornerRadius::default(); + source_uv = crate::scene::full_source_uv(); } let mut per_rect_uniforms = Vec::new(); @@ -382,6 +385,8 @@ impl<'a> Frame<'a> { 0_f32, 0., vec2f(0.0, 0.0).into(), + source_uv.origin().into(), + source_uv.size().into(), )); let per_rect_uniforms_buffer = new_metal_buffer( self.ctx.device, @@ -541,6 +546,8 @@ impl<'a> Frame<'a> { padding, dash_length, gap_lengths.into(), + Vector2F::zero().into(), + vec2f(1.0, 1.0).into(), )); } @@ -575,6 +582,8 @@ impl<'a> Frame<'a> { 0_f32, dash_length, gap_lengths.into(), + Vector2F::zero().into(), + vec2f(1.0, 1.0).into(), )); } let per_rect_uniforms_buffer = new_metal_buffer( @@ -845,6 +854,8 @@ mod shader { drop_shadow_padding_factor: f32, dash_length: f32, gap_lengths: Vector2F, + uv_origin: Vector2F, + uv_size: Vector2F, ) -> Self { Self { origin: origin.0, @@ -873,6 +884,8 @@ mod shader { drop_shadow_padding_factor, dash_length, gap_lengths: gap_lengths.0, + uv_origin: uv_origin.0, + uv_size: uv_size.0, } } } diff --git a/crates/warpui/src/platform/mac/rendering/metal/shaders/shader_types.h b/crates/warpui/src/platform/mac/rendering/metal/shaders/shader_types.h index 4f549d7c1d5..a01b5e02bd5 100644 --- a/crates/warpui/src/platform/mac/rendering/metal/shaders/shader_types.h +++ b/crates/warpui/src/platform/mac/rendering/metal/shaders/shader_types.h @@ -34,6 +34,8 @@ typedef struct { float drop_shadow_padding_factor; float dash_length; vector_float2 gap_lengths; + vector_float2 uv_origin; + vector_float2 uv_size; } PerRectUniforms; typedef struct { diff --git a/crates/warpui/src/platform/mac/rendering/metal/shaders/shaders.metal b/crates/warpui/src/platform/mac/rendering/metal/shaders/shaders.metal index 8b3eef7ac57..ffcd3de32fa 100644 --- a/crates/warpui/src/platform/mac/rendering/metal/shaders/shaders.metal +++ b/crates/warpui/src/platform/mac/rendering/metal/shaders/shaders.metal @@ -102,7 +102,10 @@ rect_vertex_shader( out.border_end = rect->border_end * rect->size + rect->origin; out.border_start_color = rect->border_start_color; out.border_end_color = rect->border_end_color; - out.texture_coordinate = vertices[vertex_id]; + // The vertices span the unit square, so they double as the texture + // coordinates of the full asset; map them onto the requested source + // sub-rectangle of the texture. + out.texture_coordinate = vertices[vertex_id] * rect->uv_size + rect->uv_origin; out.is_icon = rect->is_icon; out.icon_color = rect->icon_color; out.drop_shadow_offsets = rect->drop_shadow_offsets; diff --git a/crates/warpui/src/rendering/wgpu/renderer/image.rs b/crates/warpui/src/rendering/wgpu/renderer/image.rs index b850dc93d1e..9d13ed06f2c 100644 --- a/crates/warpui/src/rendering/wgpu/renderer/image.rs +++ b/crates/warpui/src/rendering/wgpu/renderer/image.rs @@ -152,6 +152,7 @@ impl Pipeline { opacity: (image.opacity * 255.) as u8, }, corner_radius, + image.source_uv, )); let (texture_id, _) = self.texture_cache @@ -166,6 +167,7 @@ impl Pipeline { icon.bounds * scale_factor, ColorModifier::Icon { color: icon.color }, crate::rendering::CornerRadius::default(), + crate::scene::full_source_uv(), )); let (texture_id, _) = self .texture_cache @@ -329,20 +331,23 @@ mod shaders { color: ColorF, is_icon: u32, corner_radius: Vector4F, + uv_bounds: Vector4F, } impl ImageInstanceData { - const ATTRIBS: [wgpu::VertexAttribute; 4] = wgpu::vertex_attr_array![ + const ATTRIBS: [wgpu::VertexAttribute; 5] = wgpu::vertex_attr_array![ 1 => Float32x4, // Bounds 2 => Float32x4, // Color 3 => Uint32, // Boolean, image or icon 4 => Float32x4, // Corner radius + 5 => Float32x4, // Source sub-rectangle of the texture: xy = origin, zw = size ]; pub(super) fn new( bounds: RectF, color_modifier: ColorModifier, corner_radius: CornerRadius, + source_uv: RectF, ) -> Self { Self { bounds: bounds.into(), @@ -354,6 +359,7 @@ mod shaders { corner_radius.bottom_left, corner_radius.bottom_right, ), + uv_bounds: source_uv.into(), } } diff --git a/crates/warpui/src/rendering/wgpu/shaders/image_shader.wgsl b/crates/warpui/src/rendering/wgpu/shaders/image_shader.wgsl index 5adb9571e22..f0af600ce54 100644 --- a/crates/warpui/src/rendering/wgpu/shaders/image_shader.wgsl +++ b/crates/warpui/src/rendering/wgpu/shaders/image_shader.wgsl @@ -21,6 +21,9 @@ struct ImageVertexShaderInput { @location(3) is_icon: u32, // Corner radius in the order top_left, top_right, bottom_left, bottom_right. @location(4) corner_radius: vec4, + // The sub-rectangle of the texture to sample, in unit UV space. xy is the + // origin and zw is the size. + @location(5) uv_bounds: vec4, } struct ImageVertexShaderOutput { @@ -52,7 +55,10 @@ fn vs_main( out.rect_corner = clipped_size / 2.0; out.rect_center = clipped_origin + out.rect_corner; - out.texture_coordinate = image.vertex_position; + // The vertex positions span the unit square, so they double as the texture + // coordinates of the full asset; map them onto the requested source + // sub-rectangle of the texture. + out.texture_coordinate = image.vertex_position * image.uv_bounds.zw + image.uv_bounds.xy; out.color = image.color; out.is_icon = image.is_icon; out.corner_radius = image.corner_radius; diff --git a/crates/warpui_core/src/assets/asset_cache.rs b/crates/warpui_core/src/assets/asset_cache.rs index a58e21c53f2..42ff49cdef4 100644 --- a/crates/warpui_core/src/assets/asset_cache.rs +++ b/crates/warpui_core/src/assets/asset_cache.rs @@ -342,6 +342,37 @@ impl AssetCache { assets[&key].to_external_type(source) } + /// Inserts an asset that the caller already built, for the cases where the + /// asset cannot be produced from a single buffer of bytes. + pub fn insert_asset(&self, id: String, asset: T, ctx: &mut ModelContext) { + let mut assets = self.inner.borrow_mut(); + let source = AssetSource::Raw { id }; + let key = AssetHandle { + source: source.clone(), + asset_type: TypeId::of::(), + }; + let timestamp = instant::now() as u64; + let size_in_bytes = asset.size_in_bytes(); + + assets.insert( + key, + AssetStateInternal::Loaded { + data: Rc::new(asset) as Rc, + timestamp, + size_in_bytes, + }, + ); + + ImageCache::as_ref(ctx).evict_image(&source); + + drop(assets); + let image_ids = self.evict_raw_assets_if_needed(ctx); + + if !image_ids.is_empty() { + ctx.emit(AssetCacheEvent::ImagesEvicted { image_ids }); + } + } + pub fn insert_raw_asset_bytes( &self, id: String, diff --git a/crates/warpui_core/src/scene.rs b/crates/warpui_core/src/scene.rs index 49430d8c6ed..bfd5c36de07 100644 --- a/crates/warpui_core/src/scene.rs +++ b/crates/warpui_core/src/scene.rs @@ -109,10 +109,18 @@ pub struct Rect { pub struct Image { pub bounds: RectF, pub asset: Arc, + /// The sub-rectangle of `asset` to sample, in unit UV space. Covering the + /// whole asset means an origin of (0, 0) and a size of (1, 1). + pub source_uv: RectF, pub opacity: f32, pub corner_radius: CornerRadius, } +/// The `source_uv` of an [`Image`] that samples its asset in its entirety. +pub fn full_source_uv() -> RectF { + RectF::new(vec2f(0., 0.), vec2f(1., 1.)) +} + #[derive(Clone)] pub struct Icon { pub bounds: RectF, @@ -611,6 +619,19 @@ impl Scene { asset: Arc, opacity: f32, corner_radius: CornerRadius, + ) { + self.draw_image_with_source(rect, asset, full_source_uv(), opacity, corner_radius); + } + + /// Like [`Scene::draw_image`], but samples only `source_uv` of the asset + /// (in unit UV space) instead of the whole thing. + pub fn draw_image_with_source( + &mut self, + rect: RectF, + asset: Arc, + source_uv: RectF, + opacity: f32, + corner_radius: CornerRadius, ) { #[cfg(debug_assertions)] let location = self.panic_location.take(); @@ -622,6 +643,7 @@ impl Scene { layer.images.push(Image { bounds: rect, asset, + source_uv, opacity, corner_radius, }); diff --git a/crates/warpui_core/src/scene_test.rs b/crates/warpui_core/src/scene_test.rs index 1d8449c58b3..69fc25aee9c 100644 --- a/crates/warpui_core/src/scene_test.rs +++ b/crates/warpui_core/src/scene_test.rs @@ -69,3 +69,19 @@ fn test_click_through_layer_does_not_cover_lower_layers() { assert!(!scene.is_covered(Point::new(10., 10., ZIndex::new(0)))); } + +#[test] +fn test_image_source_uv() { + let mut scene = Scene::new(1., rendering::Config::default()); + let asset = crate::image_cache::test_utils::make_static_image(4, 4); + let bounds = RectF::new(vec2f(0., 0.), vec2f(10., 10.)); + let full_uv = RectF::new(vec2f(0., 0.), vec2f(1., 1.)); + let source_uv = RectF::new(vec2f(0.25, 0.5), vec2f(0.25, 0.5)); + + scene.draw_image(bounds, asset.clone(), 1., CornerRadius::default()); + scene.draw_image_with_source(bounds, asset, source_uv, 1., CornerRadius::default()); + + let images = &scene.layers.first().images; + assert_eq!(images[0].source_uv, full_uv); + assert_eq!(images[1].source_uv, source_uv); +}