From 538d7c5005ddbe5f39a670d193be32e6db220f8d Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Thu, 3 Sep 2026 12:43:40 +0100 Subject: [PATCH 1/7] Repoint to fearless_simd branch with #[simd] macro --- Cargo.lock | 3 +-- Cargo.toml | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 382b4b52b..3c5191218 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1085,8 +1085,7 @@ dependencies = [ [[package]] name = "fearless_simd" version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" +source = "git+https://github.com/Shnatsel/fearless_simd.git?branch=simd-macro#e953125cb9dc6ecb9bdd088831b0d02d21e602d2" dependencies = [ "libm", ] diff --git a/Cargo.toml b/Cargo.toml index c17b9cca4..795b6260f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -176,3 +176,6 @@ codegen-units = 1 inherits = "release" debug = true strip = "none" + +[patch.crates-io] +fearless_simd = {git = "https://github.com/Shnatsel/fearless_simd.git", branch = "simd-macro"} From 2636fde9cd2fa43d57a1847f1319ffb00fdb13fe Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Thu, 3 Sep 2026 14:16:36 +0100 Subject: [PATCH 2/7] convert all #[inline(always)] and vectorize() manual machinery to using the #[simd] proc macro --- Cargo.lock | 90 +++-- Cargo.toml | 2 + sparse_strips/vello_cpu/Cargo.toml | 1 + .../src/fine/common/gradient/linear.rs | 2 +- .../vello_cpu/src/fine/common/gradient/mod.rs | 73 ++-- .../src/fine/common/gradient/radial.rs | 51 ++- .../src/fine/common/gradient/sweep.rs | 21 +- .../vello_cpu/src/fine/common/image.rs | 209 ++++++------ .../src/fine/common/rounded_blurred_rect.rs | 110 +++--- .../vello_cpu/src/fine/highp/blend.rs | 315 ++++++++++-------- .../vello_cpu/src/fine/highp/compose.rs | 99 +++--- sparse_strips/vello_cpu/src/fine/highp/mod.rs | 202 +++++------ .../vello_cpu/src/fine/lowp/blend.rs | 119 +++---- .../vello_cpu/src/fine/lowp/compose.rs | 79 ++--- .../vello_cpu/src/fine/lowp/gradient.rs | 31 +- .../vello_cpu/src/fine/lowp/image.rs | 111 +++--- sparse_strips/vello_cpu/src/fine/lowp/mod.rs | 315 +++++++----------- sparse_strips/vello_cpu/src/fine/mod.rs | 110 +++--- sparse_strips/vello_cpu/src/util.rs | 32 +- 19 files changed, 934 insertions(+), 1038 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c5191218..242680c35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -229,7 +229,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -406,7 +406,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -566,7 +566,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -853,7 +853,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.117", "unicode-xid", ] @@ -897,7 +897,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1090,6 +1090,16 @@ dependencies = [ "libm", ] +[[package]] +name = "fearless_simd_macros" +version = "0.1.0" +source = "git+https://github.com/Shnatsel/fearless_simd.git?branch=simd-macro#e953125cb9dc6ecb9bdd088831b0d02d21e602d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + [[package]] name = "file-id" version = "0.2.3" @@ -1217,7 +1227,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1897,7 +1907,7 @@ checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1943,7 +1953,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.117", ] [[package]] @@ -1971,7 +1981,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2337,7 +2347,7 @@ dependencies = [ "quote", "regex-automata", "regex-syntax", - "syn", + "syn 2.0.117", ] [[package]] @@ -2374,7 +2384,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2597,7 +2607,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3099,7 +3109,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3236,7 +3246,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "version_check", ] @@ -3257,7 +3267,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4488a4a36b9a4ba6b9334a32a39971f77c1436ec82c38707bce707699cc3bbcb" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3615,7 +3625,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.117", ] [[package]] @@ -3676,7 +3686,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3687,7 +3697,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3997,6 +4007,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4011,7 +4032,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4064,7 +4085,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4075,7 +4096,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4194,7 +4215,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4275,7 +4296,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4497,6 +4518,7 @@ dependencies = [ "bytemuck", "crossbeam-channel", "fearless_simd", + "fearless_simd_macros", "glifo", "hashbrown 0.17.1", "ordered-channel", @@ -4523,7 +4545,7 @@ version = "0.0.0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4705,7 +4727,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4787,7 +4809,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -4830,7 +4852,7 @@ checksum = "67008cdde4769831958536b0f11b3bdd0380bde882be17fff9c2f34bb4549abd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5062,7 +5084,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5362,7 +5384,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5373,7 +5395,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5871,7 +5893,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -5892,7 +5914,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5912,7 +5934,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -5948,7 +5970,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 795b6260f..68ba5c6f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,6 +119,7 @@ thread_local = "1.1.9" crossbeam-channel = "0.5.15" ordered-channel = { version = "1.2.0", features = ["crossbeam-channel"] } fearless_simd = { version = "0.7.0", default-features = false } +fearless_simd_macros = "0.1.0" # Unlike the other crates, please do not update these before a release # unless absolutely necessary. @@ -179,3 +180,4 @@ strip = "none" [patch.crates-io] fearless_simd = {git = "https://github.com/Shnatsel/fearless_simd.git", branch = "simd-macro"} +fearless_simd_macros = {git = "https://github.com/Shnatsel/fearless_simd.git", branch = "simd-macro"} diff --git a/sparse_strips/vello_cpu/Cargo.toml b/sparse_strips/vello_cpu/Cargo.toml index e12597297..3636e46d3 100644 --- a/sparse_strips/vello_cpu/Cargo.toml +++ b/sparse_strips/vello_cpu/Cargo.toml @@ -20,6 +20,7 @@ bytemuck = { workspace = true } glifo = { workspace = true, default-features = false, optional = true } vello_common = { workspace = true } hashbrown = { workspace = true } +fearless_simd_macros = { workspace = true } png = { workspace = true, optional = true } crossbeam-channel = { workspace = true, optional = true } ordered-channel = { workspace = true, optional = true, features = ["crossbeam-channel"] } diff --git a/sparse_strips/vello_cpu/src/fine/common/gradient/linear.rs b/sparse_strips/vello_cpu/src/fine/common/gradient/linear.rs index c79d14b72..f2e16467a 100644 --- a/sparse_strips/vello_cpu/src/fine/common/gradient/linear.rs +++ b/sparse_strips/vello_cpu/src/fine/common/gradient/linear.rs @@ -22,7 +22,7 @@ impl SimdLinearKind { impl SimdGradientKind for SimdLinearKind { #[inline(always)] - fn cur_pos(&self, x_pos: f32x8, _: f32x8) -> f32x8 { + fn cur_pos(&self, _simd: S, x_pos: f32x8, _: f32x8) -> f32x8 { x_pos } } diff --git a/sparse_strips/vello_cpu/src/fine/common/gradient/mod.rs b/sparse_strips/vello_cpu/src/fine/common/gradient/mod.rs index 422aa6da1..cd5a949e4 100644 --- a/sparse_strips/vello_cpu/src/fine/common/gradient/mod.rs +++ b/sparse_strips/vello_cpu/src/fine/common/gradient/mod.rs @@ -5,6 +5,7 @@ use crate::fine::{NumericVec, PosExt}; use crate::kurbo::Point; use crate::peniko; use core::slice::ChunksExact; +use fearless_simd_macros::simd; use vello_common::encode::{EncodedGradient, GradientLut}; use vello_common::fearless_simd::*; @@ -14,6 +15,7 @@ pub(crate) mod sweep; const GRADIENT_INVALID_POS: u32 = u32::MAX; +#[simd] pub(crate) fn calculate_t_vals>( simd: S, kind: U, @@ -22,23 +24,18 @@ pub(crate) fn calculate_t_vals>( start_x: f64, start_y: f64, ) { - simd.vectorize( - #[inline(always)] - || { - let mut cur_pos = gradient.transform * Point::new(start_x, start_y); - let x_advances = (gradient.x_advance.x as f32, gradient.x_advance.y as f32); - let y_advances = (gradient.y_advance.x as f32, gradient.y_advance.y as f32); - - for buf_part in buf.chunks_exact_mut(8) { - let x_pos = f32x8::splat_pos(simd, cur_pos.x as f32, x_advances.0, y_advances.0); - let y_pos = f32x8::splat_pos(simd, cur_pos.y as f32, x_advances.1, y_advances.1); - let pos = kind.cur_pos(x_pos, y_pos); - pos.store_slice(buf_part); - - cur_pos += 2.0 * gradient.x_advance; - } - }, - ); + let mut cur_pos = gradient.transform * Point::new(start_x, start_y); + let x_advances = (gradient.x_advance.x as f32, gradient.x_advance.y as f32); + let y_advances = (gradient.y_advance.x as f32, gradient.y_advance.y as f32); + + for buf_part in buf.chunks_exact_mut(8) { + let x_pos = f32x8::splat_pos(simd, cur_pos.x as f32, x_advances.0, y_advances.0); + let y_pos = f32x8::splat_pos(simd, cur_pos.y as f32, x_advances.1, y_advances.1); + let pos = kind.cur_pos(simd, x_pos, y_pos); + pos.store_slice(buf_part); + + cur_pos += 2.0 * gradient.x_advance; + } } #[derive(Debug, Clone)] @@ -52,23 +49,19 @@ pub(crate) struct GradientPainter<'a, S: Simd> { } impl<'a, S: Simd> GradientPainter<'a, S> { + #[simd] pub(crate) fn new(simd: S, gradient: &'a EncodedGradient, t_vals: &'a [f32]) -> Self { - simd.vectorize( - #[inline(always)] - || { - let lut = gradient.f32_lut(simd); - let scale_factor: f32x8 = f32x8::splat(simd, lut.scale_factor()); - - Self { - gradient, - scale_factor, - lut, - t_vals: t_vals.chunks_exact(8), - has_undefined: gradient.has_undefined, - simd, - } - }, - ) + let lut = gradient.f32_lut(simd); + let scale_factor: f32x8 = f32x8::splat(simd, lut.scale_factor()); + + Self { + gradient, + scale_factor, + lut, + t_vals: t_vals.chunks_exact(8), + has_undefined: gradient.has_undefined, + simd, + } } } @@ -79,7 +72,7 @@ impl Iterator for GradientPainter<'_, S> { fn next(&mut self) -> Option { let extend = self.gradient.extend; let pos = f32x8::from_slice(self.simd, self.t_vals.next()?); - let t_vals = apply_extend(pos, extend); + let t_vals = apply_extend(self.simd, pos, extend); let valid = pos.simd_eq(pos); let indices = (t_vals * self.scale_factor).to_int::>(); // In case we had any NaN's, set the index to an explicit invalid sentinel. There @@ -220,15 +213,15 @@ impl crate::fine::Painter for GradientPainter<'_, S> { } } -#[inline(always)] +#[simd] fn invalid_f32_mask(simd: S, indices: u32x4) -> mask32x16 { let indices = indices.zip_low(indices).combine(indices.zip_high(indices)); let indices = indices.zip_low(indices).combine(indices.zip_high(indices)); indices.simd_eq(u32x16::splat(simd, GRADIENT_INVALID_POS)) } -#[inline(always)] -pub(crate) fn apply_extend(val: f32x8, extend: peniko::Extend) -> f32x8 { +#[simd] +pub(crate) fn apply_extend(_simd: S, val: f32x8, extend: peniko::Extend) -> f32x8 { match extend { peniko::Extend::Pad => val.max(0.0).min(1.0), peniko::Extend::Repeat => (val - val.floor()).fract(), @@ -240,6 +233,10 @@ pub(crate) fn apply_extend(val: f32x8, extend: peniko::Extend) -> f3 } } +/// The hot inner operation of [`calculate_t_vals`]. +/// +/// Implementations are force-inlined into that function's `#[simd]` region. A nested SIMD +/// boundary here adds an out-of-line call for every vector of gradient positions. pub(crate) trait SimdGradientKind { - fn cur_pos(&self, x_pos: f32x8, y_pos: f32x8) -> f32x8; + fn cur_pos(&self, simd: S, x_pos: f32x8, y_pos: f32x8) -> f32x8; } diff --git a/sparse_strips/vello_cpu/src/fine/common/gradient/radial.rs b/sparse_strips/vello_cpu/src/fine/common/gradient/radial.rs index 0fce5d8a9..28b9289b8 100644 --- a/sparse_strips/vello_cpu/src/fine/common/gradient/radial.rs +++ b/sparse_strips/vello_cpu/src/fine/common/gradient/radial.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use crate::fine::common::gradient::SimdGradientKind; +use fearless_simd_macros::simd; use vello_common::encode::{FocalData, RadialKind}; use vello_common::fearless_simd::{Simd, SimdBase, SimdFloat, f32x8}; @@ -25,40 +26,34 @@ pub(crate) struct SimdRadialKind { } impl SimdRadialKind { + #[simd] pub(crate) fn new(simd: S, kind: &RadialKind) -> Self { - simd.vectorize( - #[inline(always)] - || { - let inner = match kind { - RadialKind::Radial { bias, scale } => SimdRadialKindInner::Radial { - bias: f32x8::splat(simd, *bias), - scale: f32x8::splat(simd, *scale), - }, - RadialKind::Strip { scaled_r0_squared } => SimdRadialKindInner::Strip { - scaled_r0_squared: f32x8::splat(simd, *scaled_r0_squared), - }, - RadialKind::Focal { - focal_data, - fp0, - fp1, - } => SimdRadialKindInner::Focal { - fp0: f32x8::splat(simd, *fp0), - fp1: f32x8::splat(simd, *fp1), - focal_data: *focal_data, - }, - }; - - Self { inner } + let inner = match kind { + RadialKind::Radial { bias, scale } => SimdRadialKindInner::Radial { + bias: f32x8::splat(simd, *bias), + scale: f32x8::splat(simd, *scale), + }, + RadialKind::Strip { scaled_r0_squared } => SimdRadialKindInner::Strip { + scaled_r0_squared: f32x8::splat(simd, *scaled_r0_squared), }, - ) + RadialKind::Focal { + focal_data, + fp0, + fp1, + } => SimdRadialKindInner::Focal { + fp0: f32x8::splat(simd, *fp0), + fp1: f32x8::splat(simd, *fp1), + focal_data: *focal_data, + }, + }; + + Self { inner } } } impl SimdGradientKind for SimdRadialKind { - #[inline(always)] - fn cur_pos(&self, x_pos: f32x8, y_pos: f32x8) -> f32x8 { - let simd = x_pos.simd; - + #[simd] + fn cur_pos(&self, simd: S, x_pos: f32x8, y_pos: f32x8) -> f32x8 { match &self.inner { SimdRadialKindInner::Radial { bias, scale } => { let radius = x_pos.mul_add(x_pos, y_pos * y_pos).sqrt(); diff --git a/sparse_strips/vello_cpu/src/fine/common/gradient/sweep.rs b/sparse_strips/vello_cpu/src/fine/common/gradient/sweep.rs index 7b38029e5..b885ed58f 100644 --- a/sparse_strips/vello_cpu/src/fine/common/gradient/sweep.rs +++ b/sparse_strips/vello_cpu/src/fine/common/gradient/sweep.rs @@ -3,6 +3,7 @@ use crate::fine::common::gradient::SimdGradientKind; use core::f32::consts::PI; +use fearless_simd_macros::simd; use vello_common::encode::SweepKind; use vello_common::fearless_simd::{Simd, SimdBase, SimdFloat, f32x8}; @@ -14,28 +15,26 @@ pub(crate) struct SimdSweepKind { } impl SimdSweepKind { + #[simd] pub(crate) fn new(simd: S, kind: &SweepKind) -> Self { - simd.vectorize( - #[inline(always)] - || Self { - start_angle: f32x8::splat(simd, kind.start_angle), - inv_angle_delta: f32x8::splat(simd, kind.inv_angle_delta), - simd, - }, - ) + Self { + start_angle: f32x8::splat(simd, kind.start_angle), + inv_angle_delta: f32x8::splat(simd, kind.inv_angle_delta), + simd, + } } } impl SimdGradientKind for SimdSweepKind { - #[inline(always)] - fn cur_pos(&self, x_pos: f32x8, y_pos: f32x8) -> f32x8 { + #[simd] + fn cur_pos(&self, _simd: S, x_pos: f32x8, y_pos: f32x8) -> f32x8 { let angle = x_y_to_unit_angle(self.simd, x_pos, y_pos) * f32x8::splat(self.simd, 2.0 * PI); (angle - self.start_angle) * self.inv_angle_delta } } -#[inline(always)] +#[simd] fn x_y_to_unit_angle(simd: S, x: f32x8, y: f32x8) -> f32x8 { let c0 = f32x8::splat(simd, 0.0); let c1 = f32x8::splat(simd, 1.0); diff --git a/sparse_strips/vello_cpu/src/fine/common/image.rs b/sparse_strips/vello_cpu/src/fine/common/image.rs index f1a06dfcd..701a91b04 100644 --- a/sparse_strips/vello_cpu/src/fine/common/image.rs +++ b/sparse_strips/vello_cpu/src/fine/common/image.rs @@ -4,6 +4,7 @@ use crate::fine::macros::{f32x16_painter, u8x16_painter}; use crate::fine::{PosExt, Splat4thExt, u8_to_f32}; use crate::kurbo::Point; +use fearless_simd_macros::simd; use vello_common::encode::EncodedImage; use vello_common::fearless_simd::{Bytes, Simd, SimdBase, SimdFloat, f32x4, f32x16, u8x16, u32x4}; use vello_common::pixmap::Pixmap; @@ -20,6 +21,7 @@ pub(crate) struct PlainNNImagePainter<'a, S: Simd> { } impl<'a, S: Simd> PlainNNImagePainter<'a, S> { + #[simd] pub(crate) fn new( simd: S, image: &'a EncodedImage, @@ -29,38 +31,33 @@ impl<'a, S: Simd> PlainNNImagePainter<'a, S> { ) -> Self { let data = ImagePainterData::new(simd, image, pixmap, start_x, start_y); - simd.vectorize( - #[inline(always)] - || { - let y_positions = extend( - simd, - f32x4::splat_pos( - simd, - data.cur_pos.y as f32, - data.x_advances.1, - data.y_advances.1, - ), - image.sampler.y_extend, - data.height, - data.height_inv, - ); - - let cur_x_pos = f32x4::splat_pos( - simd, - data.cur_pos.x as f32, - data.x_advances.0, - data.y_advances.0, - ); - - Self { - data, - advance: image.x_advance.x as f32, - y_positions, - cur_x_pos, - simd, - } - }, - ) + let y_positions = extend( + simd, + f32x4::splat_pos( + simd, + data.cur_pos.y as f32, + data.x_advances.1, + data.y_advances.1, + ), + image.sampler.y_extend, + data.height, + data.height_inv, + ); + + let cur_x_pos = f32x4::splat_pos( + simd, + data.cur_pos.x as f32, + data.x_advances.0, + data.y_advances.0, + ); + + Self { + data, + advance: image.x_advance.x as f32, + y_positions, + cur_x_pos, + simd, + } } } @@ -207,13 +204,13 @@ impl Iterator for FilteredImagePainter<'_, S, QUALIT // center of the location we are sampling, and sample those points // using a cubic filter to weight each location's contribution. - let x_fract = fract_floor(x_positions + 0.5); - let y_fract = fract_floor(y_positions + 0.5); + let x_fract = fract_floor(self.simd, x_positions + 0.5); + let y_fract = fract_floor(self.simd, y_positions + 0.5); let mut interpolated_color = f32x16::splat(self.simd, 0.0); let sample = |x_pos: f32x4, y_pos: f32x4| { - u8_to_f32(sample(self.simd, &self.data, x_pos, y_pos)) + u8_to_f32(self.simd, sample(self.simd, &self.data, x_pos, y_pos)) }; macro_rules! extend_x { @@ -345,8 +342,8 @@ f32x16_painter!(FilteredImagePainter<'_, S, 2>); /// /// Unlike `f32::fract()`, this always returns a value in [0, 1), /// even for negative inputs. -#[inline(always)] -pub(crate) fn fract_floor(val: f32x4) -> f32x4 { +#[simd] +pub(crate) fn fract_floor(_simd: S, val: f32x4) -> f32x4 { val - val.floor() } @@ -366,6 +363,7 @@ pub(crate) struct ImagePainterData<'a, S: Simd> { } impl<'a, S: Simd> ImagePainterData<'a, S> { + #[simd] pub(crate) fn new( simd: S, image: &'a EncodedImage, @@ -373,40 +371,35 @@ impl<'a, S: Simd> ImagePainterData<'a, S> { start_x: f64, start_y: f64, ) -> Self { - simd.vectorize( - #[inline(always)] - || { - let width = pixmap.width() as f32; - let height = pixmap.height() as f32; - let start_pos = image.transform * Point::new(start_x, start_y); - - let width_inv = f32x4::splat(simd, 1.0 / width); - let height_inv = f32x4::splat(simd, 1.0 / height); - let width = f32x4::splat(simd, width); - let width_u32 = u32x4::splat(simd, pixmap.width() as u32); - let height = f32x4::splat(simd, height); - - let x_advances = (image.x_advance.x as f32, image.x_advance.y as f32); - let y_advances = (image.y_advance.x as f32, image.y_advance.y as f32); - - Self { - cur_pos: start_pos, - pixmap, - x_advances, - y_advances, - image, - width, - height, - width_u32, - width_inv, - height_inv, - } - }, - ) + let width = pixmap.width() as f32; + let height = pixmap.height() as f32; + let start_pos = image.transform * Point::new(start_x, start_y); + + let width_inv = f32x4::splat(simd, 1.0 / width); + let height_inv = f32x4::splat(simd, 1.0 / height); + let width = f32x4::splat(simd, width); + let width_u32 = u32x4::splat(simd, pixmap.width() as u32); + let height = f32x4::splat(simd, height); + + let x_advances = (image.x_advance.x as f32, image.x_advance.y as f32); + let y_advances = (image.y_advance.x as f32, image.y_advance.y as f32); + + Self { + cur_pos: start_pos, + pixmap, + x_advances, + y_advances, + image, + width, + height, + width_u32, + width_inv, + height_inv, + } } } -#[inline(always)] +#[simd] pub(crate) fn sample( simd: S, data: &ImagePainterData<'_, S>, @@ -427,7 +420,7 @@ pub(crate) fn sample( .to_bytes() } -#[inline(always)] +#[simd] pub(crate) fn extend( simd: S, val: f32x4, @@ -469,50 +462,50 @@ pub(crate) fn extend( } /// Calculate the weights for a single fractional value. +#[simd] fn weights(simd: S, fract: f32x4) -> [f32x4; 4] { - simd.vectorize( - #[inline(always)] - || { - let s = fract.simd; - const MF: [[f32; 4]; 4] = mf_resampler(); - - [ - single_weight( - fract, - f32x4::splat(s, MF[0][0]), - f32x4::splat(s, MF[0][1]), - f32x4::splat(s, MF[0][2]), - f32x4::splat(s, MF[0][3]), - ), - single_weight( - fract, - f32x4::splat(s, MF[1][0]), - f32x4::splat(s, MF[1][1]), - f32x4::splat(s, MF[1][2]), - f32x4::splat(s, MF[1][3]), - ), - single_weight( - fract, - f32x4::splat(s, MF[2][0]), - f32x4::splat(s, MF[2][1]), - f32x4::splat(s, MF[2][2]), - f32x4::splat(s, MF[2][3]), - ), - single_weight( - fract, - f32x4::splat(s, MF[3][0]), - f32x4::splat(s, MF[3][1]), - f32x4::splat(s, MF[3][2]), - f32x4::splat(s, MF[3][3]), - ), - ] - }, - ) + const MF: [[f32; 4]; 4] = mf_resampler(); + + [ + single_weight( + simd, + fract, + f32x4::splat(simd, MF[0][0]), + f32x4::splat(simd, MF[0][1]), + f32x4::splat(simd, MF[0][2]), + f32x4::splat(simd, MF[0][3]), + ), + single_weight( + simd, + fract, + f32x4::splat(simd, MF[1][0]), + f32x4::splat(simd, MF[1][1]), + f32x4::splat(simd, MF[1][2]), + f32x4::splat(simd, MF[1][3]), + ), + single_weight( + simd, + fract, + f32x4::splat(simd, MF[2][0]), + f32x4::splat(simd, MF[2][1]), + f32x4::splat(simd, MF[2][2]), + f32x4::splat(simd, MF[2][3]), + ), + single_weight( + simd, + fract, + f32x4::splat(simd, MF[3][0]), + f32x4::splat(simd, MF[3][1]), + f32x4::splat(simd, MF[3][2]), + f32x4::splat(simd, MF[3][3]), + ), + ] } /// Calculate a weight based on the fractional value t and the cubic coefficients. -#[inline(always)] +#[simd] fn single_weight( + _simd: S, t: f32x4, a: f32x4, b: f32x4, diff --git a/sparse_strips/vello_cpu/src/fine/common/rounded_blurred_rect.rs b/sparse_strips/vello_cpu/src/fine/common/rounded_blurred_rect.rs index 5a66904ce..1c16e4135 100644 --- a/sparse_strips/vello_cpu/src/fine/common/rounded_blurred_rect.rs +++ b/sparse_strips/vello_cpu/src/fine/common/rounded_blurred_rect.rs @@ -7,6 +7,7 @@ use crate::fine::{NumericVec, PosExt, ShaderResultF32}; use crate::kurbo::{Point, Vec2}; +use fearless_simd_macros::simd; use vello_common::encode::EncodedBlurredRoundedRectangle; use vello_common::fearless_simd::{Simd, SimdBase, SimdFloat, f32x8, u8x16}; @@ -24,40 +25,31 @@ pub(crate) struct BlurredRoundedRectFiller { } impl BlurredRoundedRectFiller { + #[simd] pub(crate) fn new( simd: S, rect: &EncodedBlurredRoundedRectangle, start_x: f64, start_y: f64, ) -> Self { - simd.vectorize( - #[inline(always)] - || { - let start_pos = rect.transform * Point::new(start_x, start_y); - let color_components = rect.color.as_premul_f32().components; - let r = f32x8::splat(simd, color_components[0]); - let g = f32x8::splat(simd, color_components[1]); - let b = f32x8::splat(simd, color_components[2]); - let a = f32x8::splat(simd, color_components[3]); - let simd_rect = SimdRoundedBlurredRect::new(rect, simd); - let alpha_calculator = AlphaCalculator::new( - start_pos, - rect.x_advance, - rect.y_advance, - simd_rect, - simd, - ); + let start_pos = rect.transform * Point::new(start_x, start_y); + let color_components = rect.color.as_premul_f32().components; + let r = f32x8::splat(simd, color_components[0]); + let g = f32x8::splat(simd, color_components[1]); + let b = f32x8::splat(simd, color_components[2]); + let a = f32x8::splat(simd, color_components[3]); + let simd_rect = SimdRoundedBlurredRect::new(simd, rect); + let alpha_calculator = + AlphaCalculator::new(start_pos, rect.x_advance, rect.y_advance, simd_rect, simd); - Self { - alpha_calculator, - r, - g, - b, - a, - invert: rect.invert, - } - }, - ) + Self { + alpha_calculator, + r, + g, + b, + a, + invert: rect.invert, + } } } @@ -108,7 +100,7 @@ impl crate::fine::Painter for BlurredRoundedRectFiller { #[inline(always)] || { for chunk in buf.chunks_exact_mut(32) { - let [c1, c2] = self.next().unwrap().get(); + let [c1, c2] = self.next().unwrap().get(self.a.simd); c1[0] .simd .store_four_interleaved_f32x4(c1, (&mut chunk[..16]).try_into().unwrap()); @@ -206,39 +198,35 @@ struct SimdRoundedBlurredRect { } impl SimdRoundedBlurredRect { - fn new(encoded: &EncodedBlurredRoundedRectangle, s: S) -> Self { - s.vectorize( - #[inline(always)] - || { - let h = f32x8::splat(s, encoded.h); - let w = f32x8::splat(s, encoded.w); - let width = f32x8::splat(s, encoded.width); - let height = f32x8::splat(s, encoded.height); - let r1 = f32x8::splat(s, encoded.r1); - let exponent = encoded.exponent; - let recip_exponent = encoded.recip_exponent; - let scale = f32x8::splat(s, encoded.scale); - let min_edge = f32x8::splat(s, encoded.min_edge); - let std_dev_inv = f32x8::splat(s, encoded.std_dev_inv); - let v0 = f32x8::splat(s, 0.0); - let v1 = f32x8::splat(s, 0.5); + #[simd] + fn new(s: S, encoded: &EncodedBlurredRoundedRectangle) -> Self { + let h = f32x8::splat(s, encoded.h); + let w = f32x8::splat(s, encoded.w); + let width = f32x8::splat(s, encoded.width); + let height = f32x8::splat(s, encoded.height); + let r1 = f32x8::splat(s, encoded.r1); + let exponent = encoded.exponent; + let recip_exponent = encoded.recip_exponent; + let scale = f32x8::splat(s, encoded.scale); + let min_edge = f32x8::splat(s, encoded.min_edge); + let std_dev_inv = f32x8::splat(s, encoded.std_dev_inv); + let v0 = f32x8::splat(s, 0.0); + let v1 = f32x8::splat(s, 0.5); - Self { - exponent, - recip_exponent, - scale, - std_dev_inv, - min_edge, - w, - v0, - v1, - h, - width, - height, - r1, - } - }, - ) + Self { + exponent, + recip_exponent, + scale, + std_dev_inv, + min_edge, + w, + v0, + v1, + h, + width, + height, + r1, + } } } @@ -251,7 +239,7 @@ trait FloatExt { } impl FloatExt for f32x8 { - #[inline(always)] + #[simd] fn compute_erf7(simd: S, x: Self) -> Self { // Clamp `x`, because for large `x` the terms here become `inf`, causing the result to be 0 or // `NaN`. This clamping doesn't lose any information, because `erf(±10) ≈ 1` well within `f64` diff --git a/sparse_strips/vello_cpu/src/fine/highp/blend.rs b/sparse_strips/vello_cpu/src/fine/highp/blend.rs index 16c712afa..cd388227d 100644 --- a/sparse_strips/vello_cpu/src/fine/highp/blend.rs +++ b/sparse_strips/vello_cpu/src/fine/highp/blend.rs @@ -3,6 +3,7 @@ use crate::peniko::{BlendMode, Mix}; use crate::util::Premultiply; +use fearless_simd_macros::simd; use vello_common::fearless_simd::*; #[derive(Copy, Clone)] @@ -13,50 +14,47 @@ struct Channels { } impl Channels { - #[inline(always)] - fn unpremultiply(mut self, a: f32x4) -> Self { - self.r = self.r.unpremultiply(a); - self.g = self.g.unpremultiply(a); - self.b = self.b.unpremultiply(a); + #[simd] + fn unpremultiply(mut self, simd: S, a: f32x4) -> Self { + self.r = self.r.unpremultiply(simd, a); + self.g = self.g.unpremultiply(simd, a); + self.b = self.b.unpremultiply(simd, a); self } } -pub(crate) fn mix(src_c: f32x16, bg: f32x16, blend_mode: BlendMode) -> f32x16 { - src_c.simd.vectorize( - #[inline(always)] - || mix_inner(src_c, bg, blend_mode), - ) -} - -#[inline(always)] -fn mix_inner(src_c: f32x16, bg: f32x16, blend_mode: BlendMode) -> f32x16 { +#[simd] +pub(crate) fn mix( + simd: S, + src_c: f32x16, + bg: f32x16, + blend_mode: BlendMode, +) -> f32x16 { if matches!(blend_mode.mix, Mix::Normal) { return src_c; } // See https://www.w3.org/TR/compositing-1/#blending - let simd = src_c.simd; let (bg_channels, bg_a) = split(simd, bg); let (src_channels, src_a) = split(simd, src_c); - let unpremultiplied_bg = bg_channels.unpremultiply(bg_a); - let unpremultiplied_src = src_channels.unpremultiply(src_a); + let unpremultiplied_bg = bg_channels.unpremultiply(simd, bg_a); + let unpremultiplied_src = src_channels.unpremultiply(simd, src_a); let mut res_bg = unpremultiplied_bg; - let mix_src = blend_mode.mix(unpremultiplied_src, unpremultiplied_bg); + let mix_src = blend_mode.mix(simd, unpremultiplied_src, unpremultiplied_bg); - res_bg.r = apply_alpha(bg_a, src_a, unpremultiplied_src.r, mix_src.r); - res_bg.g = apply_alpha(bg_a, src_a, unpremultiplied_src.g, mix_src.g); - res_bg.b = apply_alpha(bg_a, src_a, unpremultiplied_src.b, mix_src.b); + res_bg.r = apply_alpha(simd, bg_a, src_a, unpremultiplied_src.r, mix_src.r); + res_bg.g = apply_alpha(simd, bg_a, src_a, unpremultiplied_src.g, mix_src.g); + res_bg.b = apply_alpha(simd, bg_a, src_a, unpremultiplied_src.b, mix_src.b); let mut storage = [0.0; 16]; simd.store_four_interleaved_f32x4([res_bg.r, res_bg.g, res_bg.b, src_a], &mut storage); f32x16::from_slice(simd, &storage) } -#[inline(always)] +#[simd] fn split(simd: S, input: f32x16) -> (Channels, f32x4) { let mut storage = [0.0; 16]; input.store_slice(&mut storage); @@ -65,8 +63,9 @@ fn split(simd: S, input: f32x16) -> (Channels, f32x4) { (Channels { r, g, b }, a) } -#[inline(always)] +#[simd] fn apply_alpha( + simd: S, bg_a: f32x4, src_a: f32x4, unpremultiplied_src_channel: f32x4, @@ -75,61 +74,61 @@ fn apply_alpha( let p1 = (1.0 - bg_a) * unpremultiplied_src_channel; let p2 = bg_a * mix_src_channel; - (p1 + p2).premultiply(src_a) + (p1 + p2).premultiply(simd, src_a) } trait MixExt { - fn mix(&self, src: Channels, bg: Channels) -> Channels; + fn mix(&self, simd: S, src: Channels, bg: Channels) -> Channels; } impl MixExt for BlendMode { - #[inline(always)] - fn mix(&self, src: Channels, bg: Channels) -> Channels { + #[simd] + fn mix(&self, simd: S, src: Channels, bg: Channels) -> Channels { match self.mix { Mix::Normal => src, - Mix::Multiply => Multiply::mix(src, bg), - Mix::Screen => Screen::mix(src, bg), - Mix::Overlay => Overlay::mix(src, bg), - Mix::Darken => Darken::mix(src, bg), - Mix::Lighten => Lighten::mix(src, bg), - Mix::ColorDodge => ColorDodge::mix(src, bg), - Mix::ColorBurn => ColorBurn::mix(src, bg), - Mix::HardLight => HardLight::mix(src, bg), - Mix::SoftLight => SoftLight::mix(src, bg), - Mix::Difference => Difference::mix(src, bg), - Mix::Exclusion => Exclusion::mix(src, bg), - Mix::Luminosity => Luminosity::mix(src, bg), - Mix::Color => Color::mix(src, bg), - Mix::Hue => Hue::mix(src, bg), - Mix::Saturation => Saturation::mix(src, bg), + Mix::Multiply => Multiply::mix(simd, src, bg), + Mix::Screen => Screen::mix(simd, src, bg), + Mix::Overlay => Overlay::mix(simd, src, bg), + Mix::Darken => Darken::mix(simd, src, bg), + Mix::Lighten => Lighten::mix(simd, src, bg), + Mix::ColorDodge => ColorDodge::mix(simd, src, bg), + Mix::ColorBurn => ColorBurn::mix(simd, src, bg), + Mix::HardLight => HardLight::mix(simd, src, bg), + Mix::SoftLight => SoftLight::mix(simd, src, bg), + Mix::Difference => Difference::mix(simd, src, bg), + Mix::Exclusion => Exclusion::mix(simd, src, bg), + Mix::Luminosity => Luminosity::mix(simd, src, bg), + Mix::Color => Color::mix(simd, src, bg), + Mix::Hue => Hue::mix(simd, src, bg), + Mix::Saturation => Saturation::mix(simd, src, bg), } } } impl Multiply { - #[inline(always)] - fn single(src: f32x4, bg: f32x4) -> f32x4 { + #[simd] + fn single(simd: S, src: f32x4, bg: f32x4) -> f32x4 { src * bg } } impl Screen { - #[inline(always)] - fn single(src: f32x4, bg: f32x4) -> f32x4 { + #[simd] + fn single(simd: S, src: f32x4, bg: f32x4) -> f32x4 { bg + src - src * bg } } impl HardLight { - #[inline(always)] - fn single(src: f32x4, bg: f32x4) -> f32x4 { - let two = f32x4::splat(src.simd, 2.0); + #[simd] + fn single(simd: S, src: f32x4, bg: f32x4) -> f32x4 { + let two = f32x4::splat(simd, 2.0); - let mask = src.simd.simd_le_f32x4(src, f32x4::splat(src.simd, 0.5)); - let opt1 = Multiply::single(bg, src * two); - let opt2 = Screen::single(bg, two * src - 1.0); + let mask = simd.simd_le_f32x4(src, f32x4::splat(simd, 0.5)); + let opt1 = Multiply::single(simd, bg, src * two); + let opt2 = Screen::single(simd, bg, two * src - 1.0); - src.simd.select_f32x4(mask, opt1, opt2) + simd.select_f32x4(mask, opt1, opt2) } } @@ -138,11 +137,11 @@ macro_rules! separable_mix { pub(crate) struct $name; impl $name { - #[inline(always)] - fn mix(mut src: Channels, bg: Channels) -> Channels { - src.r = $calc(src.r, bg.r); - src.g = $calc(src.g, bg.g); - src.b = $calc(src.b, bg.b); + #[simd] + fn mix(simd: S, mut src: Channels, bg: Channels) -> Channels { + src.r = ($calc)(simd, src.r, bg.r); + src.g = ($calc)(simd, src.g, bg.g); + src.b = ($calc)(simd, src.b, bg.b); src } @@ -150,71 +149,70 @@ macro_rules! separable_mix { }; } -separable_mix!(Multiply, |cs: f32x4, cb: f32x4| Multiply::single( - cs, cb -)); -separable_mix!(Screen, |cs: f32x4, cb: f32x4| Screen::single(cs, cb)); -separable_mix!(Overlay, |cs: f32x4, cb: f32x4| HardLight::single( - cb, cs -)); -separable_mix!(Darken, |cs: f32x4, cb: f32x4| cs.min(cb)); -separable_mix!(Lighten, |cs: f32x4, cb: f32x4| cs.max(cb)); -separable_mix!(Difference, |cs: f32x4, cb: f32x4| { - cs.simd - .select_f32x4(cs.simd.simd_le_f32x4(cs, cb), cb - cs, cs - cb) +separable_mix!(Multiply, |simd: S, cs: f32x4, cb: f32x4| { + Multiply::single(simd, cs, cb) +}); +separable_mix!(Screen, |simd: S, cs: f32x4, cb: f32x4| { + Screen::single(simd, cs, cb) +}); +separable_mix!(Overlay, |simd: S, cs: f32x4, cb: f32x4| { + HardLight::single(simd, cb, cs) }); -separable_mix!(HardLight, |cs: f32x4, cb: f32x4| HardLight::single( - cs, cb -)); -separable_mix!(Exclusion, |cs: f32x4, cb: f32x4| { +separable_mix!(Darken, |_: S, cs: f32x4, cb: f32x4| cs.min(cb)); +separable_mix!(Lighten, |_: S, cs: f32x4, cb: f32x4| cs.max(cb)); +separable_mix!(Difference, |simd: S, cs: f32x4, cb: f32x4| { + simd.select_f32x4(simd.simd_le_f32x4(cs, cb), cb - cs, cs - cb) +}); +separable_mix!(HardLight, |simd: S, cs: f32x4, cb: f32x4| { + HardLight::single(simd, cs, cb) +}); +separable_mix!(Exclusion, |_: S, cs: f32x4, cb: f32x4| { (cs + cb) - 2.0 * (cs * cb) }); -separable_mix!(SoftLight, |cs: f32x4, cb: f32x4| { - let mask_1 = cs.simd.simd_le_f32x4(cb, f32x4::splat(cs.simd, 0.25)); +separable_mix!(SoftLight, |simd: S, cs: f32x4, cb: f32x4| { + let mask_1 = simd.simd_le_f32x4(cb, f32x4::splat(simd, 0.25)); - let d = cs - .simd - .select_f32x4(mask_1, ((16.0 * cb - 12.0) * cb + 4.0) * cb, cb.sqrt()); + let d = simd.select_f32x4(mask_1, ((16.0 * cb - 12.0) * cb + 4.0) * cb, cb.sqrt()); - let mask_2 = cs.simd.simd_le_f32x4(cs, f32x4::splat(cs.simd, 0.5)); + let mask_2 = simd.simd_le_f32x4(cs, f32x4::splat(simd, 0.5)); - cs.simd.select_f32x4( + simd.select_f32x4( mask_2, cb - (1.0 - 2.0 * cs) * cb * (1.0 - cb), cb + (2.0 * cs - 1.0) * (d - cb), ) }); -separable_mix!(ColorDodge, |cs: f32x4, cb: f32x4| { - let mask_1 = cb.simd.simd_eq_f32x4(cb, f32x4::splat(cb.simd, 0.0)); - let mask_2 = cs.simd.simd_eq_f32x4(cs, f32x4::splat(cs.simd, 1.0)); +separable_mix!(ColorDodge, |simd: S, cs: f32x4, cb: f32x4| { + let mask_1 = simd.simd_eq_f32x4(cb, f32x4::splat(simd, 0.0)); + let mask_2 = simd.simd_eq_f32x4(cs, f32x4::splat(simd, 1.0)); - cs.simd.select_f32x4( + simd.select_f32x4( // if cb == 0 mask_1, - f32x4::splat(cs.simd, 0.0), + f32x4::splat(simd, 0.0), // else if cs == 1 - cs.simd.select_f32x4( + simd.select_f32x4( mask_2, - f32x4::splat(cs.simd, 1.0), + f32x4::splat(simd, 1.0), // else - f32x4::splat(cs.simd, 1.0).min(cb / (1.0 - cs)), + f32x4::splat(simd, 1.0).min(cb / (1.0 - cs)), ), ) }); -separable_mix!(ColorBurn, |cs: f32x4, cb: f32x4| { - let mask_1 = cb.simd.simd_eq_f32x4(cb, f32x4::splat(cb.simd, 1.0)); - let mask_2 = cs.simd.simd_eq_f32x4(cs, f32x4::splat(cs.simd, 0.0)); +separable_mix!(ColorBurn, |simd: S, cs: f32x4, cb: f32x4| { + let mask_1 = simd.simd_eq_f32x4(cb, f32x4::splat(simd, 1.0)); + let mask_2 = simd.simd_eq_f32x4(cs, f32x4::splat(simd, 0.0)); - cs.simd.select_f32x4( + simd.select_f32x4( // if cb == 1 mask_1, - f32x4::splat(cs.simd, 1.0), + f32x4::splat(simd, 1.0), // else if cs == 0 - cs.simd.select_f32x4( + simd.select_f32x4( mask_2, - f32x4::splat(cs.simd, 0.0), + f32x4::splat(simd, 0.0), // else - 1.0 - f32x4::splat(cs.simd, 1.0).min((1.0 - cb) / cs), + 1.0 - f32x4::splat(simd, 1.0).min((1.0 - cb) / cs), ), ) }); @@ -224,55 +222,93 @@ macro_rules! non_separable_mix { pub(crate) struct $name; impl $name { - #[inline(always)] - fn mix(mut src: Channels, mut bg: Channels) -> Channels { - $calc(&mut src, &mut bg) + #[simd] + fn mix(simd: S, mut src: Channels, mut bg: Channels) -> Channels { + ($calc)(simd, &mut src, &mut bg) } } }; } -non_separable_mix!(Hue, |cs: &mut Channels, cb: &mut Channels| { - set_sat(&mut cs.r, &mut cs.g, &mut cs.b, sat(cb.r, cb.g, cb.b)); - set_lum(&mut cs.r, &mut cs.g, &mut cs.b, lum(cb.r, cb.g, cb.b)); +non_separable_mix!(Hue, |simd: S, + cs: &mut Channels, + cb: &mut Channels| { + set_sat( + simd, + &mut cs.r, + &mut cs.g, + &mut cs.b, + sat(simd, cb.r, cb.g, cb.b), + ); + set_lum( + simd, + &mut cs.r, + &mut cs.g, + &mut cs.b, + lum(simd, cb.r, cb.g, cb.b), + ); *cs }); -non_separable_mix!(Saturation, |cs: &mut Channels, cb: &mut Channels| { - let lum = lum(cb.r, cb.g, cb.b); - set_sat(&mut cb.r, &mut cb.g, &mut cb.b, sat(cs.r, cs.g, cs.b)); - set_lum(&mut cb.r, &mut cb.g, &mut cb.b, lum); - - *cb -}); +non_separable_mix!( + Saturation, + |simd: S, cs: &mut Channels, cb: &mut Channels| { + let lum = lum(simd, cb.r, cb.g, cb.b); + set_sat( + simd, + &mut cb.r, + &mut cb.g, + &mut cb.b, + sat(simd, cs.r, cs.g, cs.b), + ); + set_lum(simd, &mut cb.r, &mut cb.g, &mut cb.b, lum); -non_separable_mix!(Color, |cs: &mut Channels, cb: &mut Channels| { - set_lum(&mut cs.r, &mut cs.g, &mut cs.b, lum(cb.r, cb.g, cb.b)); + *cb + } +); + +non_separable_mix!(Color, |simd: S, + cs: &mut Channels, + cb: &mut Channels| { + set_lum( + simd, + &mut cs.r, + &mut cs.g, + &mut cs.b, + lum(simd, cb.r, cb.g, cb.b), + ); *cs }); -non_separable_mix!(Luminosity, |cs: &mut Channels, cb: &mut Channels| { - set_lum(&mut cb.r, &mut cb.g, &mut cb.b, lum(cs.r, cs.g, cs.b)); +non_separable_mix!( + Luminosity, + |simd: S, cs: &mut Channels, cb: &mut Channels| { + set_lum( + simd, + &mut cb.r, + &mut cb.g, + &mut cb.b, + lum(simd, cs.r, cs.g, cs.b), + ); - *cb -}); + *cb + } +); -#[inline(always)] -fn lum(r: f32x4, g: f32x4, b: f32x4) -> f32x4 { +#[simd] +fn lum(simd: S, r: f32x4, g: f32x4, b: f32x4) -> f32x4 { 0.3 * r + 0.59 * g + 0.11 * b } -#[inline(always)] -fn sat(r: f32x4, g: f32x4, b: f32x4) -> f32x4 { +#[simd] +fn sat(simd: S, r: f32x4, g: f32x4, b: f32x4) -> f32x4 { r.max(g).max(b) - r.min(g).min(b) } -#[inline(always)] -fn clip_color(r: &mut f32x4, g: &mut f32x4, b: &mut f32x4) { - let simd = r.simd; - - let l = lum(*r, *g, *b); +#[simd] +fn clip_color(simd: S, r: &mut f32x4, g: &mut f32x4, b: &mut f32x4) { + let l = lum(simd, *r, *g, *b); let n = r.min(g.min(*b)); let x = r.max(g.max(*b)); @@ -291,32 +327,37 @@ fn clip_color(r: &mut f32x4, g: &mut f32x4, b: &mut f32x4) { } } -#[inline(always)] -fn set_lum(r: &mut f32x4, g: &mut f32x4, b: &mut f32x4, l: f32x4) { - let d = l - lum(*r, *g, *b); +#[simd] +fn set_lum(simd: S, r: &mut f32x4, g: &mut f32x4, b: &mut f32x4, l: f32x4) { + let d = l - lum(simd, *r, *g, *b); *r += d; *g += d; *b += d; - clip_color(r, g, b); + clip_color(simd, r, g, b); } // Adapted from tiny-skia -#[inline(always)] -fn set_sat(r: &mut f32x4, g: &mut f32x4, b: &mut f32x4, s: f32x4) { +#[simd] +fn set_sat(simd: S, r: &mut f32x4, g: &mut f32x4, b: &mut f32x4, s: f32x4) { let mn = r.min(g.min(*b)); let mx = r.max(g.max(*b)); let sat = mx - mn; // Map min channel to 0, max channel to s, and scale the middle proportionally. - *r = scale_sat_channel(*r, mn, sat, s); - *g = scale_sat_channel(*g, mn, sat, s); - *b = scale_sat_channel(*b, mn, sat, s); + *r = scale_sat_channel(simd, *r, mn, sat, s); + *g = scale_sat_channel(simd, *g, mn, sat, s); + *b = scale_sat_channel(simd, *b, mn, sat, s); } -#[inline(always)] -fn scale_sat_channel(c: f32x4, mn: f32x4, sat: f32x4, s: f32x4) -> f32x4 { - let simd = c.simd; +#[simd] +fn scale_sat_channel( + simd: S, + c: f32x4, + mn: f32x4, + sat: f32x4, + s: f32x4, +) -> f32x4 { simd.select_f32x4( simd.simd_eq_f32x4(sat, f32x4::splat(simd, 0.0)), f32x4::splat(simd, 0.0), diff --git a/sparse_strips/vello_cpu/src/fine/highp/compose.rs b/sparse_strips/vello_cpu/src/fine/highp/compose.rs index 3e3b68905..892fd175a 100644 --- a/sparse_strips/vello_cpu/src/fine/highp/compose.rs +++ b/sparse_strips/vello_cpu/src/fine/highp/compose.rs @@ -3,6 +3,7 @@ use crate::fine::Splat4thExt; use crate::peniko::{BlendMode, Compose}; +use fearless_simd_macros::simd; use vello_common::fearless_simd::*; pub(crate) trait ComposeExt { @@ -16,6 +17,7 @@ pub(crate) trait ComposeExt { } impl ComposeExt for BlendMode { + #[simd] fn compose( &self, simd: S, @@ -23,62 +25,49 @@ impl ComposeExt for BlendMode { bg_c: f32x16, alpha_mask: Option>, ) -> f32x16 { - simd.vectorize( - #[inline(always)] - || compose_inner(*self, simd, src_c, bg_c, alpha_mask), - ) - } -} + // There some non-obvious subtleties worth highlighting here. + // We support two kinds of blending (in this case, we focus on compositing specifically): + // - Isolated blending, where layers as a whole are blended together with their backdrop. + // If we are currently performing this kind of blending, `alpha_mask` will always be + // `None`. After all, there is no concrete shape opacity associated with a layer. Instead, + // we are just compositing the RGBA values at _all_ positions of the source layer with the + // backdrop layer. For example, if the backdrop contains a green rectangle and source + // layer is just empty, if we perform blending with `Compose::Clear`, then _everything_ + // will be cleared, because we are compositing the whole source layer with the whole + // backdrop, and not just the parts of the source layer that have actually be drawn on. + // - Non-isolated blending, where a single path is blended with the backdrop. In this case, + // `alpha_mask` _might_ be `Some` and contain the alpha values of the strips we are + // currently compositing. Remember that strips always have a fixed height of 4, because + // of this, the strips might cover areas that aren't actually covered by the path (and + // just have an alpha value of 0, or a value between 0-254 for anti-aliased parts). + // Because of this, for non-isolated blending, we need to lerp the result with the + // backdrop using `alpha_mask`. -#[inline(always)] -fn compose_inner( - blend_mode: BlendMode, - simd: S, - src_c: f32x16, - bg_c: f32x16, - alpha_mask: Option>, -) -> f32x16 { - // There some non-obvious subtleties worth highlighting here. - // We support two kinds of blending (in this case, we focus on compositing specifically): - // - Isolated blending, where layers as a whole are blended together with their backdrop. - // If we are currently performing this kind of blending, `alpha_mask` will always be `None`. - // After all, there is no concrete shape opacity associated with a layer. Instead, we are - // just compositing the RGBA values at _all_ positions of the source layer with the backdrop - // layer. For example, if the backdrop contains a green rectangle and source layer is just - // empty, if we perform blending with `Compose::Clear`, then _everything_ will be cleared, - // because we are compositing the whole source layer with the whole backdrop, and not - // just the parts of the source layer that have actually be drawn on. - // - Non-isolated blending, where a single path is blended with the backdrop. In this case, - // `alpha_mask` _might_ be `Some` and contain the alpha values of the strips we are currently - // compositing. Remember that strips always have a fixed height of 4, because of this, the - // strips might cover areas that aren't actually covered by the path (and just have an alpha - // value of 0, or a value between 0-254 for anti-aliased parts). Because of this, for - // non-isolated blending, we need to lerp the result with the backdrop using `alpha_mask`. + let mut res = match self.compose { + Compose::SrcOver => SrcOver::compose(simd, src_c, bg_c), + Compose::Clear => Clear::compose(simd, src_c, bg_c), + Compose::Copy => Copy::compose(simd, src_c, bg_c), + Compose::DestOver => DestOver::compose(simd, src_c, bg_c), + Compose::Dest => Dest::compose(simd, src_c, bg_c), + Compose::SrcIn => SrcIn::compose(simd, src_c, bg_c), + Compose::DestIn => DestIn::compose(simd, src_c, bg_c), + Compose::SrcOut => SrcOut::compose(simd, src_c, bg_c), + Compose::DestOut => DestOut::compose(simd, src_c, bg_c), + Compose::SrcAtop => SrcAtop::compose(simd, src_c, bg_c), + Compose::DestAtop => DestAtop::compose(simd, src_c, bg_c), + Compose::Xor => Xor::compose(simd, src_c, bg_c), + Compose::Plus => Plus::compose(simd, src_c, bg_c), + // Have not been able to find a formula for this, so just fallback to Plus. + Compose::PlusLighter => Plus::compose(simd, src_c, bg_c), + }; - let mut res = match blend_mode.compose { - Compose::SrcOver => SrcOver::compose(simd, src_c, bg_c), - Compose::Clear => Clear::compose(simd, src_c, bg_c), - Compose::Copy => Copy::compose(simd, src_c, bg_c), - Compose::DestOver => DestOver::compose(simd, src_c, bg_c), - Compose::Dest => Dest::compose(simd, src_c, bg_c), - Compose::SrcIn => SrcIn::compose(simd, src_c, bg_c), - Compose::DestIn => DestIn::compose(simd, src_c, bg_c), - Compose::SrcOut => SrcOut::compose(simd, src_c, bg_c), - Compose::DestOut => DestOut::compose(simd, src_c, bg_c), - Compose::SrcAtop => SrcAtop::compose(simd, src_c, bg_c), - Compose::DestAtop => DestAtop::compose(simd, src_c, bg_c), - Compose::Xor => Xor::compose(simd, src_c, bg_c), - Compose::Plus => Plus::compose(simd, src_c, bg_c), - // Have not been able to find a formula for this, so just fallback to Plus. - Compose::PlusLighter => Plus::compose(simd, src_c, bg_c), - }; + if let Some(alpha_mask) = alpha_mask { + let alpha_mask_inv = 1.0 - alpha_mask; + res = alpha_mask * res + alpha_mask_inv * bg_c; + } - if let Some(alpha_mask) = alpha_mask { - let alpha_mask_inv = 1.0 - alpha_mask; - res = alpha_mask * res + alpha_mask_inv * bg_c; + res } - - res } macro_rules! compose { @@ -86,13 +75,13 @@ macro_rules! compose { struct $name; impl $name { - #[inline(always)] + #[simd] fn compose(simd: S, src_c: f32x16, bg_c: f32x16) -> f32x16 { let al_b = bg_c.splat_4th(); let al_s = src_c.splat_4th(); - let fa = $fa(simd, al_s, al_b); - let fb = $fb(simd, al_s, al_b); + let fa = ($fa)(simd, al_s, al_b); + let fb = ($fb)(simd, al_s, al_b); if $sat { (src_c * fa + fb * bg_c) diff --git a/sparse_strips/vello_cpu/src/fine/highp/mod.rs b/sparse_strips/vello_cpu/src/fine/highp/mod.rs index f4386f523..baa3d39c1 100644 --- a/sparse_strips/vello_cpu/src/fine/highp/mod.rs +++ b/sparse_strips/vello_cpu/src/fine/highp/mod.rs @@ -19,6 +19,7 @@ use crate::fine::FineKernel; use crate::fine::{COLOR_COMPONENTS, Painter, Splat4thExt}; use crate::peniko::BlendMode; use crate::region::Region; +use fearless_simd_macros::simd; use vello_common::fearless_simd::*; use vello_common::filter_effects::Filter; use vello_common::kurbo::Affine; @@ -65,38 +66,30 @@ impl FineKernel for F32Kernel { /// /// Efficiently broadcasts a single RGBA color across all pixels in the destination. #[inline(never)] + #[simd] fn copy_solid(simd: S, dest: &mut [Self::Numeric], src: [Self::Numeric; 4]) { - simd.vectorize( - #[inline(always)] - || { - let color = f32x16::block_splat(src.simd_into(simd)); + let color = f32x16::block_splat(src.simd_into(simd)); - for el in dest.chunks_exact_mut(16) { - color.store_slice(el); - } - }, - ); + for el in dest.chunks_exact_mut(16) { + color.store_slice(el); + } } /// Applies per-pixel mask values to a buffer by multiplying each component. /// /// Used for anti-aliasing and clipping effects. Each pixel is multiplied by /// its corresponding mask value (already normalized to [0.0, 1.0]). + #[simd] fn apply_mask( simd: S, dest: &mut [Self::Numeric], mut src: impl Iterator, ) { - simd.vectorize( - #[inline(always)] - || { - for el in dest.chunks_exact_mut(16) { - let loaded = f32x16::from_slice(simd, el); - let mulled = loaded * src.next().unwrap(); - mulled.store_slice(el); - } - }, - ); + for el in dest.chunks_exact_mut(16) { + let loaded = f32x16::from_slice(simd, el); + let mulled = loaded * src.next().unwrap(); + mulled.store_slice(el); + } } /// Applies a painter's output to the destination buffer. @@ -107,34 +100,29 @@ impl FineKernel for F32Kernel { painter.paint_f32(dest); } + #[simd] fn apply_tint(simd: S, dest: &mut [Self::Numeric], tint: &Tint) { let premul = tint.color.premultiply(); let [r, g, b, a] = premul.components; - - simd.vectorize( - #[inline(always)] - || { - let tint_v = f32x16::block_splat(f32x4::from_slice(simd, &[r, g, b, a])); - - match tint.mode { - TintMode::AlphaMask => { - for chunk in dest.chunks_exact_mut(16) { - let pixel = f32x16::from_slice(simd, chunk); - let alphas = pixel.splat_4th(); - let tinted = tint_v * alphas; - tinted.store_slice(chunk); - } - } - TintMode::Multiply => { - for chunk in dest.chunks_exact_mut(16) { - let pixel = f32x16::from_slice(simd, chunk); - let tinted = pixel * tint_v; - tinted.store_slice(chunk); - } - } + let tint_v = f32x16::block_splat(f32x4::from_slice(simd, &[r, g, b, a])); + + match tint.mode { + TintMode::AlphaMask => { + for chunk in dest.chunks_exact_mut(16) { + let pixel = f32x16::from_slice(simd, chunk); + let alphas = pixel.splat_4th(); + let tinted = tint_v * alphas; + tinted.store_slice(chunk); + } + } + TintMode::Multiply => { + for chunk in dest.chunks_exact_mut(16) { + let pixel = f32x16::from_slice(simd, chunk); + let tinted = pixel * tint_v; + tinted.store_slice(chunk); } - }, - ); + } + } } /// Composites a solid color onto a buffer using alpha blending. @@ -310,75 +298,61 @@ mod fill { use crate::fine::highp::compose::ComposeExt; use crate::peniko::BlendMode; + use fearless_simd_macros::simd; use vello_common::fearless_simd::*; - // IMPORTANT: The inlining attributes (#[inline(always)], #[inline(never)]) in this - // module have been carefully tuned through benchmarking. Changing them can cause - // significant performance regressions. + // IMPORTANT: The SIMD boundaries in this module are performance-sensitive and should be + // changed only with benchmarking. /// Composites a solid color onto a buffer using alpha blending. /// /// Uses the "over" operator: `result = src + bg * (1 - src_alpha)` - #[inline(always)] + #[simd] pub(super) fn alpha_composite_solid(s: S, dest: &mut [f32], src: [f32; 4]) { - s.vectorize( - #[inline(always)] - || { - let one_minus_alpha = 1.0 - f32x16::block_splat(f32x4::splat(s, src[3])); - let src_c = f32x16::block_splat(f32x4::simd_from(s, src)); - - for next_dest in dest.chunks_exact_mut(16) { - alpha_composite_inner(s, next_dest, src_c, one_minus_alpha); - } - }, - ); + let one_minus_alpha = 1.0 - f32x16::block_splat(f32x4::splat(s, src[3])); + let src_c = f32x16::block_splat(f32x4::simd_from(s, src)); + + for next_dest in dest.chunks_exact_mut(16) { + alpha_composite_inner(s, next_dest, src_c, one_minus_alpha); + } } /// Composites a buffer of colors onto another buffer using alpha blending. /// /// Each source pixel is composited individually based on its alpha channel. - #[inline(always)] + #[simd] pub(super) fn alpha_composite_arbitrary>>( simd: S, dest: &mut [f32], src: T, ) { - simd.vectorize( - #[inline(always)] - || { - for (next_dest, next_src) in dest.chunks_exact_mut(16).zip(src) { - let one_minus_alpha = 1.0 - next_src.splat_4th(); - alpha_composite_inner(simd, next_dest, next_src, one_minus_alpha); - } - }, - ); + for (next_dest, next_src) in dest.chunks_exact_mut(16).zip(src) { + let one_minus_alpha = 1.0 - next_src.splat_4th(); + alpha_composite_inner(simd, next_dest, next_src, one_minus_alpha); + } } /// Applies blend mode compositing to a buffer without per-pixel masks. + #[simd] pub(super) fn blend>>( simd: S, dest: &mut [f32], src: T, blend_mode: BlendMode, ) { - simd.vectorize( - #[inline(always)] - || { - for (next_dest, next_src) in dest.chunks_exact_mut(16).zip(src) { - let bg_v = f32x16::from_slice(simd, next_dest); - let src_c = blend::mix(next_src, bg_v, blend_mode); - let res = blend_mode.compose(simd, src_c, bg_v, None); - res.store_slice(next_dest); - } - }, - ); + for (next_dest, next_src) in dest.chunks_exact_mut(16).zip(src) { + let bg_v = f32x16::from_slice(simd, next_dest); + let src_c = blend::mix(simd, next_src, bg_v, blend_mode); + let res = blend_mode.compose(simd, src_c, bg_v, None); + res.store_slice(next_dest); + } } /// Performs the core alpha compositing calculation. /// /// Formula: `result = src + bg * (1 - src_alpha)` /// This implements the Porter-Duff "source over" operator using FMA for efficiency. - #[inline(always)] + #[simd] fn alpha_composite_inner( s: S, dest: &mut [f32], @@ -401,57 +375,48 @@ mod alpha_fill { use crate::fine::highp::compose::ComposeExt; use crate::fine::highp::{blend, extract_masks}; use crate::peniko::BlendMode; + use fearless_simd_macros::simd; use vello_common::fearless_simd::*; /// Composites a solid color with per-pixel alpha masks. /// /// Combines source alpha with mask values: `effective_alpha = src_alpha * mask` - #[inline(always)] + #[simd] pub(super) fn alpha_composite_solid( s: S, dest: &mut [f32], src: [f32; 4], alphas: impl Iterator, ) { - s.vectorize( - #[inline(always)] - || { - let src_a = f32x16::splat(s, src[3]); - let src_c = f32x16::block_splat(src.simd_into(s)); - let one = f32x16::splat(s, 1.0); - - for (next_dest, next_mask) in dest.chunks_exact_mut(16).zip(alphas) { - alpha_composite_inner(s, next_dest, &next_mask, src_c, src_a, one); - } - }, - ); + let src_a = f32x16::splat(s, src[3]); + let src_c = f32x16::block_splat(src.simd_into(s)); + let one = f32x16::splat(s, 1.0); + + for (next_dest, next_mask) in dest.chunks_exact_mut(16).zip(alphas) { + alpha_composite_inner(s, next_dest, &next_mask, src_c, src_a, one); + } } /// Composites a buffer of colors with per-pixel alpha masks. /// /// Each pixel's source alpha is modulated by its corresponding mask value. + #[simd] pub(super) fn alpha_composite_arbitrary>>( simd: S, dest: &mut [f32], src: T, alphas: impl Iterator, ) { - simd.vectorize( - #[inline(always)] - || { - let one = f32x16::splat(simd, 1.0); - - for ((next_dest, next_mask), next_src) in - dest.chunks_exact_mut(16).zip(alphas).zip(src) - { - let src_a = next_src.splat_4th(); - alpha_composite_inner(simd, next_dest, &next_mask, next_src, src_a, one); - } - }, - ); + let one = f32x16::splat(simd, 1.0); + + for ((next_dest, next_mask), next_src) in dest.chunks_exact_mut(16).zip(alphas).zip(src) { + let src_a = next_src.splat_4th(); + alpha_composite_inner(simd, next_dest, &next_mask, next_src, src_a, one); + } } /// Applies blend mode compositing with per-pixel alpha masks. + #[simd] pub(super) fn blend>>( simd: S, dest: &mut [f32], @@ -459,20 +424,13 @@ mod alpha_fill { alphas: impl Iterator, blend_mode: BlendMode, ) { - simd.vectorize( - #[inline(always)] - || { - for ((next_dest, next_mask), next_src) in - dest.chunks_exact_mut(16).zip(alphas).zip(src) - { - let masks = extract_masks(simd, &next_mask); - let bg = f32x16::from_slice(simd, next_dest); - let src_c = blend::mix(next_src, bg, blend_mode); - let res = blend_mode.compose(simd, src_c, bg, Some(masks)); - res.store_slice(next_dest); - } - }, - ); + for ((next_dest, next_mask), next_src) in dest.chunks_exact_mut(16).zip(alphas).zip(src) { + let masks = extract_masks(simd, &next_mask); + let bg = f32x16::from_slice(simd, next_dest); + let src_c = blend::mix(simd, next_src, bg, blend_mode); + let res = blend_mode.compose(simd, src_c, bg, Some(masks)); + res.store_slice(next_dest); + } } /// Performs alpha compositing with mask modulation. @@ -480,7 +438,7 @@ mod alpha_fill { /// Formula: `result = src * mask + bg * (1 - src_alpha * mask)` /// The mask value modulates both the source contribution and the inverse alpha. /// Uses FMA instructions for optimal performance. - #[inline(always)] + #[simd] fn alpha_composite_inner( s: S, dest: &mut [f32], @@ -506,7 +464,7 @@ mod alpha_fill { /// /// Input: [m0, m1, m2, m3] (as u8, 0-255) /// Output: [m0/255, m0/255, m0/255, m0/255, m1/255, ..., m3/255] (as f32, 16 elements) -#[inline(always)] +#[simd] fn extract_masks(simd: S, masks: &[u8; 4]) -> f32x16 { let mut base_mask = [ masks[0] as f32, diff --git a/sparse_strips/vello_cpu/src/fine/lowp/blend.rs b/sparse_strips/vello_cpu/src/fine/lowp/blend.rs index 336f84d97..dfd3a852a 100644 --- a/sparse_strips/vello_cpu/src/fine/lowp/blend.rs +++ b/sparse_strips/vello_cpu/src/fine/lowp/blend.rs @@ -3,56 +3,57 @@ use crate::fine::{Splat4thExt, highp, u8_to_f32}; use crate::peniko::{BlendMode, Mix}; +use fearless_simd_macros::simd; use vello_common::fearless_simd::*; use vello_common::util::{Div255Ext, f32_to_u8, normalized_mul_u8, saturating_narrow, widen}; -pub(crate) fn mix(src_c: u8x32, bg_c: u8x32, blend_mode: BlendMode) -> u8x32 { - src_c.simd.vectorize( - #[inline(always)] - || mix_inner(src_c, bg_c, blend_mode), - ) -} - -#[inline(always)] -fn mix_inner(src_c: u8x32, bg_c: u8x32, blend_mode: BlendMode) -> u8x32 { - if let Some(res) = try_u8_mix(blend_mode, src_c, bg_c) { +#[simd] +pub(crate) fn mix( + simd: S, + src_c: u8x32, + bg_c: u8x32, + blend_mode: BlendMode, +) -> u8x32 { + if let Some(res) = try_u8_mix(simd, blend_mode, src_c, bg_c) { return res; } // Fallback for blend modes that aren't supported in u8. - let (mut src_1, mut src_2) = to_f32(src_c); - let (bg_1, bg_2) = to_f32(bg_c); + let (mut src_1, mut src_2) = to_f32(simd, src_c); + let (bg_1, bg_2) = to_f32(simd, bg_c); - src_1 = highp::blend::mix(src_1, bg_1, blend_mode); - src_2 = highp::blend::mix(src_2, bg_2, blend_mode); + src_1 = highp::blend::mix(simd, src_1, bg_1, blend_mode); + src_2 = highp::blend::mix(simd, src_2, bg_2, blend_mode); - to_u8(src_1, src_2) + to_u8(simd, src_1, src_2) } -#[inline(always)] -fn to_f32(val: u8x32) -> (f32x16, f32x16) { - let simd = val.simd; +#[simd] +fn to_f32(simd: S, val: u8x32) -> (f32x16, f32x16) { let (a, b) = simd.split_u8x32(val); - let mut a = u8_to_f32(a); - let mut b = u8_to_f32(b); + let mut a = u8_to_f32(simd, a); + let mut b = u8_to_f32(simd, b); a *= f32x16::splat(simd, 1.0 / 255.0); b *= f32x16::splat(simd, 1.0 / 255.0); (a, b) } -#[inline(always)] -fn to_u8(val1: f32x16, val2: f32x16) -> u8x32 { - let val1 = - f32_to_u8(f32x16::splat(val1.simd, 255.0).mul_add(val1, f32x16::splat(val1.simd, 0.5))); - let val2 = - f32_to_u8(f32x16::splat(val2.simd, 255.0).mul_add(val2, f32x16::splat(val2.simd, 0.5))); +#[simd] +fn to_u8(simd: S, val1: f32x16, val2: f32x16) -> u8x32 { + let val1 = f32_to_u8(f32x16::splat(simd, 255.0).mul_add(val1, f32x16::splat(simd, 0.5))); + let val2 = f32_to_u8(f32x16::splat(simd, 255.0).mul_add(val2, f32x16::splat(simd, 0.5))); - val1.simd.combine_u8x16(val1, val2) + simd.combine_u8x16(val1, val2) } -#[inline(always)] -fn try_u8_mix(blend_mode: BlendMode, src_c: u8x32, bg_c: u8x32) -> Option> { +#[simd] +fn try_u8_mix( + simd: S, + blend_mode: BlendMode, + src_c: u8x32, + bg_c: u8x32, +) -> Option> { // We implement the u8 fast path for blend modes that // 1) are separable. // 2) don't have too many divisions, since integer normalization is @@ -61,14 +62,14 @@ fn try_u8_mix(blend_mode: BlendMode, src_c: u8x32, bg_c: u8x32) - // some more blend modes are worth doing in integer space. Some(match blend_mode.mix { Mix::Normal => src_c, - Mix::Multiply => Multiply::mix(src_c, bg_c), - Mix::Screen => Screen::mix(src_c, bg_c), - Mix::Overlay => Overlay::mix(src_c, bg_c), - Mix::Darken => Darken::mix(src_c, bg_c), - Mix::Lighten => Lighten::mix(src_c, bg_c), - Mix::HardLight => HardLight::mix(src_c, bg_c), - Mix::Difference => Difference::mix(src_c, bg_c), - Mix::Exclusion => Exclusion::mix(src_c, bg_c), + Mix::Multiply => Multiply::mix(simd, src_c, bg_c), + Mix::Screen => Screen::mix(simd, src_c, bg_c), + Mix::Overlay => Overlay::mix(simd, src_c, bg_c), + Mix::Darken => Darken::mix(simd, src_c, bg_c), + Mix::Lighten => Lighten::mix(simd, src_c, bg_c), + Mix::HardLight => HardLight::mix(simd, src_c, bg_c), + Mix::Difference => Difference::mix(simd, src_c, bg_c), + Mix::Exclusion => Exclusion::mix(simd, src_c, bg_c), Mix::ColorDodge | Mix::ColorBurn | Mix::SoftLight @@ -84,10 +85,9 @@ macro_rules! u8_mix { struct $name; impl $name { - #[inline(always)] - fn mix(src_c: u8x32, bg_c: u8x32) -> u8x32 { - let simd = src_c.simd; - let res = $calc(src_c, bg_c); + #[simd] + fn mix(simd: S, src_c: u8x32, bg_c: u8x32) -> u8x32 { + let res = ($calc)(simd, src_c, bg_c); with_src_alpha(simd, res, src_c) } @@ -108,7 +108,7 @@ macro_rules! u8_mix { // B(Cb, Cs) = Cb * Cs // M = S * (1 - Ab) + As * Ab * Cb * Cs // = S * (1 - Ab) + S * D -u8_mix!(Multiply, |src_c: u8x32, bg_c: u8x32| { +u8_mix!(Multiply, |_: S, src_c: u8x32, bg_c: u8x32| { let one_minus_bg_a = 255 - bg_c.splat_4th(); let p1 = normalized_mul_u8(src_c, one_minus_bg_a); let p2 = normalized_mul_u8(src_c, bg_c); @@ -120,7 +120,7 @@ u8_mix!(Multiply, |src_c: u8x32, bg_c: u8x32| { // B(Cb, Cs) = Cb + Cs - Cb * Cs // M = S * (1 - Ab) + As * D + S * Ab - S * D // = S + As * D - S * D -u8_mix!(Screen, |src_c: u8x32, bg_c: u8x32| { +u8_mix!(Screen, |_: S, src_c: u8x32, bg_c: u8x32| { let p1 = normalized_mul_u8(src_c.splat_4th(), bg_c); let p2 = normalized_mul_u8(src_c, bg_c); let res = widen(src_c) + p1 - p2; @@ -129,14 +129,14 @@ u8_mix!(Screen, |src_c: u8x32, bg_c: u8x32| { }); // Overlay is hard-light with source and backdrop swapped. -u8_mix!(Overlay, |src_c: u8x32, bg_c: u8x32| { - hard_light_inner(src_c, bg_c, bg_c) +u8_mix!(Overlay, |simd: S, src_c: u8x32, bg_c: u8x32| { + hard_light_inner(simd, src_c, bg_c, bg_c) }); // Darken: // B(Cb, Cs) = min(Cb, Cs) // M = S * (1 - Ab) + min(S * Ab, D * As) -u8_mix!(Darken, |src_c: u8x32, bg_c: u8x32| { +u8_mix!(Darken, |_: S, src_c: u8x32, bg_c: u8x32| { let src_a = src_c.splat_4th(); let bg_a = bg_c.splat_4th(); let p1 = normalized_mul_u8(src_c, 255 - bg_a); @@ -148,7 +148,7 @@ u8_mix!(Darken, |src_c: u8x32, bg_c: u8x32| { // Lighten: // B(Cb, Cs) = max(Cb, Cs) // M = S * (1 - Ab) + max(S * Ab, D * As) -u8_mix!(Lighten, |src_c: u8x32, bg_c: u8x32| { +u8_mix!(Lighten, |_: S, src_c: u8x32, bg_c: u8x32| { let src_a = src_c.splat_4th(); let bg_a = bg_c.splat_4th(); let p1 = normalized_mul_u8(src_c, 255 - bg_a); @@ -160,14 +160,14 @@ u8_mix!(Lighten, |src_c: u8x32, bg_c: u8x32| { // Hard-light: // if Cs <= 0.5: B(Cb, Cs) = 2 * Cb * Cs // otherwise: B(Cb, Cs) = 1 - 2 * (1 - Cb) * (1 - Cs) -u8_mix!(HardLight, |src_c: u8x32, bg_c: u8x32| { - hard_light_inner(src_c, bg_c, src_c) +u8_mix!(HardLight, |simd: S, src_c: u8x32, bg_c: u8x32| { + hard_light_inner(simd, src_c, bg_c, src_c) }); // Difference: // B(Cb, Cs) = abs(Cb - Cs) // M = S * (1 - Ab) + abs(S * Ab - D * As) -u8_mix!(Difference, |src_c: u8x32, bg_c: u8x32| { +u8_mix!(Difference, |_: S, src_c: u8x32, bg_c: u8x32| { let src_a = src_c.splat_4th(); let bg_a = bg_c.splat_4th(); let p1 = normalized_mul_u8(src_c, 255 - bg_a); @@ -182,8 +182,7 @@ u8_mix!(Difference, |src_c: u8x32, bg_c: u8x32| { // B(Cb, Cs) = Cb + Cs - 2 * Cb * Cs // M = S * (1 - Ab) + As * D + S * Ab - 2 * S * D // = S + As * D - 2 * S * D -u8_mix!(Exclusion, |src_c: u8x32, bg_c: u8x32| { - let simd = src_c.simd; +u8_mix!(Exclusion, |simd: S, src_c: u8x32, bg_c: u8x32| { let p1 = normalized_mul_u8(src_c.splat_4th(), bg_c); let p2 = normalized_mul_u8(src_c, bg_c); let res = widen(src_c) + p1; @@ -193,9 +192,13 @@ u8_mix!(Exclusion, |src_c: u8x32, bg_c: u8x32| { saturating_narrow(res) }); -#[inline(always)] -fn hard_light_inner(src_c: u8x32, bg_c: u8x32, condition: u8x32) -> u8x32 { - let simd = src_c.simd; +#[simd] +fn hard_light_inner( + simd: S, + src_c: u8x32, + bg_c: u8x32, + condition: u8x32, +) -> u8x32 { let src = widen(src_c); let bg = widen(bg_c); let src_a = widen(src_c.splat_4th()); @@ -222,7 +225,7 @@ fn hard_light_inner(src_c: u8x32, bg_c: u8x32, condition: u8x32(simd: S, rgb: u8x32, src_c: u8x32) -> u8x32 { let alpha_mask = u32x8::splat(simd, u32::from_ne_bytes([0, 0, 0, 255])).to_bytes(); // It can happen that we end up with an R/G/B larger than the alpha value due to @@ -244,7 +247,7 @@ mod tests { let src = u8x32::from_slice(simd, &src.repeat(8)); let bg = u8x32::from_slice(simd, &bg.repeat(8)); let blend_mode = BlendMode::new(blend, Compose::SrcOver); - let res = mix(src, bg, blend_mode); + let res = mix(simd, src, bg, blend_mode); let mut out = [0; 32]; res.store_slice(&mut out); out[..4].try_into().unwrap() @@ -260,7 +263,7 @@ mod tests { let bg = to_f32(bg); let blend_mode = BlendMode::new(mix, Compose::SrcOver); - let res = highp::blend::mix(src, bg, blend_mode); + let res = highp::blend::mix(simd, src, bg, blend_mode); let res = f32_to_u8(f32x16::splat(simd, 255.0).mul_add(res, f32x16::splat(simd, 0.5))); let mut out = [0; 16]; res.store_slice(&mut out); diff --git a/sparse_strips/vello_cpu/src/fine/lowp/compose.rs b/sparse_strips/vello_cpu/src/fine/lowp/compose.rs index facd04821..0487313a4 100644 --- a/sparse_strips/vello_cpu/src/fine/lowp/compose.rs +++ b/sparse_strips/vello_cpu/src/fine/lowp/compose.rs @@ -4,6 +4,7 @@ use crate::fine::Splat4thExt; use crate::peniko::{BlendMode, Compose}; use crate::util::NormalizedMulExt; +use fearless_simd_macros::simd; use vello_common::fearless_simd::*; use vello_common::util::{Div255Ext, narrow, widen}; @@ -18,6 +19,7 @@ pub(crate) trait ComposeExt { } impl ComposeExt for BlendMode { + #[simd] fn compose( &self, simd: S, @@ -25,47 +27,33 @@ impl ComposeExt for BlendMode { bg_c: u8x32, alpha_mask: Option>, ) -> u8x32 { - simd.vectorize( - #[inline(always)] - || compose_inner(*self, simd, src_c, bg_c, alpha_mask), - ) - } -} + let mut res = match self.compose { + Compose::SrcOver => SrcOver::compose(simd, src_c, bg_c), + Compose::Clear => Clear::compose(simd, src_c, bg_c), + Compose::Copy => Copy::compose(simd, src_c, bg_c), + Compose::DestOver => DestOver::compose(simd, src_c, bg_c), + Compose::Dest => Dest::compose(simd, src_c, bg_c), + Compose::SrcIn => SrcIn::compose(simd, src_c, bg_c), + Compose::DestIn => DestIn::compose(simd, src_c, bg_c), + Compose::SrcOut => SrcOut::compose(simd, src_c, bg_c), + Compose::DestOut => DestOut::compose(simd, src_c, bg_c), + Compose::SrcAtop => SrcAtop::compose(simd, src_c, bg_c), + Compose::DestAtop => DestAtop::compose(simd, src_c, bg_c), + Compose::Xor => Xor::compose(simd, src_c, bg_c), + Compose::Plus => Plus::compose(simd, src_c, bg_c), + // Have not been able to find a formula for this, so just fallback to Plus. + Compose::PlusLighter => Plus::compose(simd, src_c, bg_c), + }; -#[inline(always)] -fn compose_inner( - blend_mode: BlendMode, - simd: S, - src_c: u8x32, - bg_c: u8x32, - alpha_mask: Option>, -) -> u8x32 { - let mut res = match blend_mode.compose { - Compose::SrcOver => SrcOver::compose(simd, src_c, bg_c), - Compose::Clear => Clear::compose(simd, src_c, bg_c), - Compose::Copy => Copy::compose(simd, src_c, bg_c), - Compose::DestOver => DestOver::compose(simd, src_c, bg_c), - Compose::Dest => Dest::compose(simd, src_c, bg_c), - Compose::SrcIn => SrcIn::compose(simd, src_c, bg_c), - Compose::DestIn => DestIn::compose(simd, src_c, bg_c), - Compose::SrcOut => SrcOut::compose(simd, src_c, bg_c), - Compose::DestOut => DestOut::compose(simd, src_c, bg_c), - Compose::SrcAtop => SrcAtop::compose(simd, src_c, bg_c), - Compose::DestAtop => DestAtop::compose(simd, src_c, bg_c), - Compose::Xor => Xor::compose(simd, src_c, bg_c), - Compose::Plus => Plus::compose(simd, src_c, bg_c), - // Have not been able to find a formula for this, so just fallback to Plus. - Compose::PlusLighter => Plus::compose(simd, src_c, bg_c), - }; + if let Some(alpha_mask) = alpha_mask { + let alpha_mask_inv = 255 - alpha_mask; + let p1 = widen(alpha_mask) * widen(res); + let p2 = widen(alpha_mask_inv) * widen(bg_c); + res = narrow((p1 + p2).div_255()); + } - if let Some(alpha_mask) = alpha_mask { - let alpha_mask_inv = 255 - alpha_mask; - let p1 = widen(alpha_mask) * widen(res); - let p2 = widen(alpha_mask_inv) * widen(bg_c); - res = narrow((p1 + p2).div_255()); + res } - - res } macro_rules! compose { @@ -73,22 +61,23 @@ macro_rules! compose { struct $name; impl $name { - #[inline(always)] + #[simd] fn compose(simd: S, src_c: u8x32, bg_c: u8x32) -> u8x32 { let al_b = bg_c.splat_4th(); let al_s = src_c.splat_4th(); - let fa = $fa(simd, al_s, al_b); - let fb = $fb(simd, al_s, al_b); + let fa = ($fa)(simd, al_s, al_b); + let fb = ($fb)(simd, al_s, al_b); if $sat { narrow( - (widen(src_c.normalized_mul(fa)) + widen(fb.normalized_mul(bg_c))) - .min(u16x32::splat(simd, 255)) - .max(u16x32::splat(simd, 0)), + (widen(src_c.normalized_mul(simd, fa)) + + widen(fb.normalized_mul(simd, bg_c))) + .min(u16x32::splat(simd, 255)) + .max(u16x32::splat(simd, 0)), ) } else { - src_c.normalized_mul(fa) + fb.normalized_mul(bg_c) + src_c.normalized_mul(simd, fa) + fb.normalized_mul(simd, bg_c) } } } diff --git a/sparse_strips/vello_cpu/src/fine/lowp/gradient.rs b/sparse_strips/vello_cpu/src/fine/lowp/gradient.rs index 5a612d434..883feeadd 100644 --- a/sparse_strips/vello_cpu/src/fine/lowp/gradient.rs +++ b/sparse_strips/vello_cpu/src/fine/lowp/gradient.rs @@ -3,6 +3,7 @@ use crate::peniko; use core::slice::ChunksExact; +use fearless_simd_macros::simd; use vello_common::encode::EncodedGradient; use vello_common::fearless_simd::*; @@ -19,22 +20,18 @@ pub(crate) struct GradientPainter<'a, S: Simd> { } impl<'a, S: Simd> GradientPainter<'a, S> { + #[simd] pub(crate) fn new(simd: S, gradient: &'a EncodedGradient, t_vals: &'a [f32]) -> Self { - simd.vectorize( - #[inline(always)] - || { - let lut = gradient.u8_lut(simd); - let scale_factor = f32x16::splat(simd, lut.scale_factor()); + let lut = gradient.u8_lut(simd); + let scale_factor = f32x16::splat(simd, lut.scale_factor()); - Self { - gradient, - scale_factor, - lut: lut.lut(), - t_vals: t_vals.chunks_exact(16), - simd, - } - }, - ) + Self { + gradient, + scale_factor, + lut: lut.lut(), + t_vals: t_vals.chunks_exact(16), + simd, + } } } @@ -45,7 +42,7 @@ impl Iterator for GradientPainter<'_, S> { fn next(&mut self) -> Option { let extend = self.gradient.extend; let pos = f32x16::from_slice(self.simd, self.t_vals.next()?); - let t_vals = apply_extend(pos, extend); + let t_vals = apply_extend(self.simd, pos, extend); let indices = (t_vals * self.scale_factor).to_int::>(); let mut vals = [0_u8; 64]; @@ -75,8 +72,8 @@ impl crate::fine::Painter for GradientPainter<'_, S> { } // TODO: Maybe delete this method and use `apply_extend` from highp by splitting into two f32x8. -#[inline(always)] -pub(crate) fn apply_extend(val: f32x16, extend: peniko::Extend) -> f32x16 { +#[simd] +pub(crate) fn apply_extend(simd: S, val: f32x16, extend: peniko::Extend) -> f32x16 { match extend { peniko::Extend::Pad => val.max(0.0).min(1.0), peniko::Extend::Repeat => (val - val.floor()).fract(), diff --git a/sparse_strips/vello_cpu/src/fine/lowp/image.rs b/sparse_strips/vello_cpu/src/fine/lowp/image.rs index 96984dd7e..d86943ea3 100644 --- a/sparse_strips/vello_cpu/src/fine/lowp/image.rs +++ b/sparse_strips/vello_cpu/src/fine/lowp/image.rs @@ -4,6 +4,7 @@ use crate::fine::PosExt; use crate::fine::common::image::{ImagePainterData, extend, fract_floor, sample}; use crate::fine::macros::u8x16_painter; +use fearless_simd_macros::simd; use vello_common::encode::EncodedImage; use vello_common::fearless_simd::{f32x4, prelude::*, u8x16, u16x16}; use vello_common::pixmap::Pixmap; @@ -72,11 +73,11 @@ impl Iterator for BilinearImagePainter<'_, S> { let fx = f32_to_u8(element_wise_splat( self.simd, - fract_floor(x_positions + 0.5).mul_add(255.0, 0.5), + fract_floor(self.simd, x_positions + 0.5).mul_add(255.0, 0.5), )); let fy = f32_to_u8(element_wise_splat( self.simd, - fract_floor(y_positions + 0.5).mul_add(255.0, 0.5), + fract_floor(self.simd, y_positions + 0.5).mul_add(255.0, 0.5), )); let fx = widen(fx); @@ -129,6 +130,7 @@ pub(crate) struct PlainBilinearImagePainter<'a, S: Simd> { } impl<'a, S: Simd> PlainBilinearImagePainter<'a, S> { + #[simd] pub(crate) fn new( simd: S, image: &'a EncodedImage, @@ -138,60 +140,55 @@ impl<'a, S: Simd> PlainBilinearImagePainter<'a, S> { ) -> Self { let data = ImagePainterData::new(simd, image, pixmap, start_x, start_y); - simd.vectorize( - #[inline(always)] - || { - // For axis-aligned images, y doesn't change across the strip - let y_positions = f32x4::splat_pos( - simd, - data.cur_pos.y as f32, - data.x_advances.1, - data.y_advances.1, - ); - - // Pre-compute y extend positions - let y_pos1 = extend( - simd, - y_positions - 0.5, - image.sampler.y_extend, - data.height, - data.height_inv, - ); - let y_pos2 = extend( - simd, - y_positions + 0.5, - image.sampler.y_extend, - data.height, - data.height_inv, - ); - - // Pre-compute y interpolation weights - let fy = f32_to_u8(element_wise_splat( - simd, - fract_floor(y_positions + 0.5).mul_add(255.0, 0.5), - )); - let fy = widen(fy); - let fy_inv = u16x16::splat(simd, 255) - fy; - - let cur_x_pos = f32x4::splat_pos( - simd, - data.cur_pos.x as f32, - data.x_advances.0, - data.y_advances.0, - ); - - Self { - data, - y_pos1, - y_pos2, - fy, - fy_inv, - cur_x_pos, - advance: image.x_advance.x as f32, - simd, - } - }, - ) + // For axis-aligned images, y doesn't change across the strip + let y_positions = f32x4::splat_pos( + simd, + data.cur_pos.y as f32, + data.x_advances.1, + data.y_advances.1, + ); + + // Pre-compute y extend positions + let y_pos1 = extend( + simd, + y_positions - 0.5, + image.sampler.y_extend, + data.height, + data.height_inv, + ); + let y_pos2 = extend( + simd, + y_positions + 0.5, + image.sampler.y_extend, + data.height, + data.height_inv, + ); + + // Pre-compute y interpolation weights + let fy = f32_to_u8(element_wise_splat( + simd, + fract_floor(simd, y_positions + 0.5).mul_add(255.0, 0.5), + )); + let fy = widen(fy); + let fy_inv = u16x16::splat(simd, 255) - fy; + + let cur_x_pos = f32x4::splat_pos( + simd, + data.cur_pos.x as f32, + data.x_advances.0, + data.y_advances.0, + ); + + Self { + data, + y_pos1, + y_pos2, + fy, + fy_inv, + cur_x_pos, + advance: image.x_advance.x as f32, + simd, + } } } @@ -222,7 +219,7 @@ impl Iterator for PlainBilinearImagePainter<'_, S> { // Compute x interpolation weights let fx = f32_to_u8(element_wise_splat( self.simd, - fract_floor(x_plus_half).mul_add(255.0, 0.5), + fract_floor(self.simd, x_plus_half).mul_add(255.0, 0.5), )); let fx = widen(fx); let fx_inv = u16x16::splat(self.simd, 255) - fx; diff --git a/sparse_strips/vello_cpu/src/fine/lowp/mod.rs b/sparse_strips/vello_cpu/src/fine/lowp/mod.rs index 5717e22f2..36a2dba90 100644 --- a/sparse_strips/vello_cpu/src/fine/lowp/mod.rs +++ b/sparse_strips/vello_cpu/src/fine/lowp/mod.rs @@ -23,6 +23,7 @@ use crate::util::NormalizedMulExt; use crate::util::scalar::div_255; use bytemuck::{cast_slice, cast_slice_mut}; use core::iter; +use fearless_simd_macros::simd; use vello_common::encode::{EncodedGradient, EncodedImage}; use vello_common::fearless_simd::*; use vello_common::filter_effects::Filter; @@ -67,33 +68,28 @@ impl FineKernel for U8Kernel { /// Fills a buffer with a solid color using SIMD operations. /// /// Efficiently broadcasts a single RGBA color across all pixels in the destination. + #[simd] fn copy_solid(simd: S, dest: &mut [Self::Numeric], src: [Self::Numeric; 4]) { - simd.vectorize( - #[inline(always)] - || { - let target: &mut [u32] = cast_slice_mut(dest); - target.fill(u32::from_ne_bytes(src)); - }, - ); + let target: &mut [u32] = cast_slice_mut(dest); + target.fill(u32::from_ne_bytes(src)); } /// Creates a painter for rendering gradients in u8 precision. /// /// Returns a painter that evaluates the gradient at each pixel position. + #[simd] fn gradient_painter<'a>( simd: S, gradient: &'a EncodedGradient, t_vals: &'a [f32], ) -> impl Painter + 'a { - simd.vectorize( - #[inline(always)] - || gradient::GradientPainter::new(simd, gradient, t_vals), - ) + gradient::GradientPainter::new(simd, gradient, t_vals) } /// Creates a painter for rendering images with bilinear filtering in u8 precision. /// /// Returns a painter that samples the image with bilinear interpolation. + #[simd] fn medium_quality_image_painter<'a>( simd: S, image: &'a EncodedImage, @@ -101,16 +97,14 @@ impl FineKernel for U8Kernel { start_x: f64, start_y: f64, ) -> impl Painter + 'a { - simd.vectorize( - #[inline(always)] - || BilinearImagePainter::new(simd, image, pixmap, start_x, start_y), - ) + BilinearImagePainter::new(simd, image, pixmap, start_x, start_y) } /// Creates a painter for rendering axis-aligned images with bilinear filtering in u8 precision. /// /// Returns a painter that samples the image with bilinear interpolation. /// This is an optimized version for images without skew transformation. + #[simd] fn plain_medium_quality_image_painter<'a>( simd: S, image: &'a EncodedImage, @@ -118,31 +112,24 @@ impl FineKernel for U8Kernel { start_x: f64, start_y: f64, ) -> impl Painter + 'a { - simd.vectorize( - #[inline(always)] - || PlainBilinearImagePainter::new(simd, image, pixmap, start_x, start_y), - ) + PlainBilinearImagePainter::new(simd, image, pixmap, start_x, start_y) } /// Applies per-pixel mask values to a buffer by multiplying each component. /// /// Used for anti-aliasing and clipping effects. Each pixel is multiplied by /// its corresponding mask value and normalized. + #[simd] fn apply_mask( simd: S, dest: &mut [Self::Numeric], mut src: impl Iterator, ) { - simd.vectorize( - #[inline(always)] - || { - for el in dest.chunks_exact_mut(16) { - let loaded = u8x16::from_slice(simd, el); - let mulled = narrow(normalized_mul_u8(loaded, src.next().unwrap())); - mulled.store_slice(el); - } - }, - ); + for el in dest.chunks_exact_mut(16) { + let loaded = u8x16::from_slice(simd, el); + let mulled = narrow(normalized_mul_u8(loaded, src.next().unwrap())); + mulled.store_slice(el); + } } /// Applies a painter's output to the destination buffer. @@ -153,37 +140,32 @@ impl FineKernel for U8Kernel { painter.paint_u8(dest); } - #[inline(always)] + #[simd] fn apply_tint(simd: S, dest: &mut [Self::Numeric], tint: &Tint) { let premul = tint.color.premultiply(); let [r, g, b, a] = premul.components; let to_u8 = |v: f32| (v * 255.0 + 0.5) as u8; let color = u32::from_ne_bytes([to_u8(r), to_u8(g), to_u8(b), to_u8(a)]); - simd.vectorize( - #[inline(always)] - || { - let tint_v = u32x8::block_splat(u32x4::splat(simd, color)).to_bytes(); - - match tint.mode { - TintMode::AlphaMask => { - for chunk in dest.chunks_exact_mut(32) { - let pixel = u8x32::from_slice(simd, chunk); - let alphas = pixel.splat_4th(); - let tinted = tint_v.normalized_mul(alphas); - tinted.store_slice(chunk); - } - } - TintMode::Multiply => { - for chunk in dest.chunks_exact_mut(32) { - let pixel = u8x32::from_slice(simd, chunk); - let tinted = pixel.normalized_mul(tint_v); - tinted.store_slice(chunk); - } - } + let tint_v = u32x8::block_splat(u32x4::splat(simd, color)).to_bytes(); + + match tint.mode { + TintMode::AlphaMask => { + for chunk in dest.chunks_exact_mut(32) { + let pixel = u8x32::from_slice(simd, chunk); + let alphas = pixel.splat_4th(); + let tinted = tint_v.normalized_mul(simd, alphas); + tinted.store_slice(chunk); } - }, - ); + } + TintMode::Multiply => { + for chunk in dest.chunks_exact_mut(32) { + let pixel = u8x32::from_slice(simd, chunk); + let tinted = pixel.normalized_mul(simd, tint_v); + tinted.store_slice(chunk); + } + } + } } /// Composites a solid color onto a buffer using alpha blending. @@ -302,26 +284,18 @@ impl FineKernel for U8Kernel { } } + #[simd] fn pack(simd: S, scratch: &[Self::Numeric], width: usize, region: &mut Region<'_>) { - simd.vectorize( - #[inline(always)] - || { - pack(simd, scratch, width, region); - }, - ); + pack(simd, scratch, width, region); } + #[simd] fn unpack(simd: S, region: &mut Region<'_>, width: usize, scratch: &mut [Self::Numeric]) { - simd.vectorize( - #[inline(always)] - || { - unpack(simd, region, width, scratch); - }, - ); + unpack(simd, region, width, scratch); } } -#[inline(always)] +#[simd] fn pack(simd: S, scratch: &[u8], width: usize, region: &mut Region<'_>) { let block_width = if region.height == Tile::HEIGHT { (width / Tile::WIDTH as usize) * Tile::WIDTH as usize @@ -343,7 +317,7 @@ fn pack(simd: S, scratch: &[u8], width: usize, region: &mut Region<'_>) } } -#[inline(always)] +#[simd] fn pack_block(simd: S, scratch: &[u8], width: usize, region: &mut Region<'_>) { const CHUNK_LENGTH: usize = Tile::WIDTH as usize * TILE_HEIGHT_COMPONENTS; @@ -389,7 +363,7 @@ fn pack_tail(scratch: &[u8], x: usize, width: usize, region: &mut Region<'_>) { } } -#[inline(always)] +#[simd] fn unpack(simd: S, region: &mut Region<'_>, width: usize, scratch: &mut [u8]) { let block_width = if region.height == Tile::HEIGHT { width / Tile::WIDTH as usize * Tile::WIDTH as usize @@ -411,7 +385,7 @@ fn unpack(simd: S, region: &mut Region<'_>, width: usize, scratch: &mut } } -#[inline(always)] +#[simd] fn unpack_block(simd: S, region: &mut Region<'_>, width: usize, scratch: &mut [u8]) { let scratch: &mut [f32] = cast_slice_mut(&mut scratch[..width * TILE_HEIGHT_COMPONENTS]); const CHUNK_LENGTH: usize = 16; @@ -466,84 +440,73 @@ mod fill { use crate::fine::lowp::blend; use crate::fine::lowp::compose::ComposeExt; use crate::peniko::{BlendMode, Mix}; + use fearless_simd_macros::simd; use vello_common::fearless_simd::*; use vello_common::util::{narrow, normalized_mul_u8}; /// Applies blend mode compositing to a buffer without per-pixel masks. + #[simd] pub(super) fn blend>>( simd: S, dest: &mut [u8], src: T, blend_mode: BlendMode, ) { - simd.vectorize( - #[inline(always)] - || { - let default_mix = matches!(blend_mode.mix, Mix::Normal); - for (next_dest, next_src) in dest.chunks_exact_mut(32).zip(src) { - let bg_v = u8x32::from_slice(simd, next_dest); - let src_v = if default_mix { - next_src - } else { - blend::mix(next_src, bg_v, blend_mode) - }; - let res = blend_mode.compose(simd, src_v, bg_v, None); - res.store_slice(next_dest); - } - }, - ); + let default_mix = matches!(blend_mode.mix, Mix::Normal); + for (next_dest, next_src) in dest.chunks_exact_mut(32).zip(src) { + let bg_v = u8x32::from_slice(simd, next_dest); + let src_v = if default_mix { + next_src + } else { + blend::mix(simd, next_src, bg_v, blend_mode) + }; + let res = blend_mode.compose(simd, src_v, bg_v, None); + res.store_slice(next_dest); + } } /// Composites a solid color onto a buffer using alpha blending. /// /// Uses the "over" operator: `result = src + bg * (1 - src_alpha)` + #[simd] pub(super) fn alpha_composite_solid(s: S, dest: &mut [u8], src: [u8; 4]) { - s.vectorize( - #[inline(always)] - || { - let one_minus_alpha = 255 - u8x32::splat(s, src[3]); - let src_c = u32x8::splat(s, u32::from_ne_bytes(src)).to_bytes(); - - for next_dest in dest.chunks_exact_mut(64) { - // We process in batches of 64 because loading/storing is much faster this way (at least on NEON), - // but since we widen to u16, we can only work with 256 bits, so we split it up. - let bg_v = u8x64::from_slice(s, next_dest); - let (bg_1, bg_2) = s.split_u8x64(bg_v); - let res_1 = alpha_composite_inner(s, bg_1, src_c, one_minus_alpha); - let res_2 = alpha_composite_inner(s, bg_2, src_c, one_minus_alpha); - let combined = s.combine_u8x32(res_1, res_2); - combined.store_slice(next_dest); - } - }, - ); + let one_minus_alpha = 255 - u8x32::splat(s, src[3]); + let src_c = u32x8::splat(s, u32::from_ne_bytes(src)).to_bytes(); + + for next_dest in dest.chunks_exact_mut(64) { + // We process in batches of 64 because loading/storing is much faster this way (at least on NEON), + // but since we widen to u16, we can only work with 256 bits, so we split it up. + let bg_v = u8x64::from_slice(s, next_dest); + let (bg_1, bg_2) = s.split_u8x64(bg_v); + let res_1 = alpha_composite_inner(s, bg_1, src_c, one_minus_alpha); + let res_2 = alpha_composite_inner(s, bg_2, src_c, one_minus_alpha); + let combined = s.combine_u8x32(res_1, res_2); + combined.store_slice(next_dest); + } } /// Composites a buffer of colors onto another buffer using alpha blending. /// /// Each source pixel is composited individually based on its alpha channel. + #[simd] pub(super) fn alpha_composite>>( simd: S, dest: &mut [u8], src: T, ) { - simd.vectorize( - #[inline(always)] - || { - for (next_dest, next_src) in dest.chunks_exact_mut(32).zip(src) { - let one_minus_alpha = 255 - next_src.splat_4th(); - let bg_v = u8x32::from_slice(simd, next_dest); - let res = alpha_composite_inner(simd, bg_v, next_src, one_minus_alpha); - res.store_slice(next_dest); - } - }, - ); + for (next_dest, next_src) in dest.chunks_exact_mut(32).zip(src) { + let one_minus_alpha = 255 - next_src.splat_4th(); + let bg_v = u8x32::from_slice(simd, next_dest); + let res = alpha_composite_inner(simd, bg_v, next_src, one_minus_alpha); + res.store_slice(next_dest); + } } /// Performs the core alpha compositing calculation. /// /// Formula: `result = src + bg * (1 - src_alpha)` /// This implements the Porter-Duff "source over" operator. - #[inline(always)] + #[simd] fn alpha_composite_inner( _s: S, bg: u8x32, @@ -564,10 +527,12 @@ mod alpha_fill { use crate::fine::lowp::compose::ComposeExt; use crate::fine::lowp::{blend, extract_masks}; use crate::peniko::{BlendMode, Mix}; + use fearless_simd_macros::simd; use vello_common::fearless_simd::*; use vello_common::util::{Div255Ext, narrow, normalized_mul_u8, widen}; /// Applies blend mode compositing with per-pixel alpha masks. + #[simd] pub(super) fn blend>>( simd: S, dest: &mut [u8], @@ -575,83 +540,64 @@ mod alpha_fill { blend_mode: BlendMode, alphas: impl Iterator, ) { - simd.vectorize( - #[inline(always)] - || { - let default_mix = matches!(blend_mode.mix, Mix::Normal); - - for ((next_bg, next_mask), next_src) in - dest.chunks_exact_mut(32).zip(alphas).zip(src) - { - let bg_v = u8x32::from_slice(simd, next_bg); - let src_c = if default_mix { - next_src - } else { - blend::mix(next_src, bg_v, blend_mode) - }; - let masks = extract_masks(simd, &next_mask); - let res = blend_mode.compose(simd, src_c, bg_v, Some(masks)); - - res.store_slice(next_bg); - } - }, - ); + let default_mix = matches!(blend_mode.mix, Mix::Normal); + + for ((next_bg, next_mask), next_src) in dest.chunks_exact_mut(32).zip(alphas).zip(src) { + let bg_v = u8x32::from_slice(simd, next_bg); + let src_c = if default_mix { + next_src + } else { + blend::mix(simd, next_src, bg_v, blend_mode) + }; + let masks = extract_masks(simd, &next_mask); + let res = blend_mode.compose(simd, src_c, bg_v, Some(masks)); + + res.store_slice(next_bg); + } } /// Composites a solid color with per-pixel alpha masks. /// /// Combines source alpha with mask values: `effective_alpha = src_alpha * mask / 255` - #[inline(always)] + #[simd] pub(super) fn alpha_composite_solid( s: S, dest: &mut [u8], src: [u8; 4], alphas: impl Iterator, ) { - s.vectorize( - #[inline(always)] - || { - let src_a = u8x32::splat(s, src[3]); - let src_c = u32x8::splat(s, u32::from_ne_bytes(src)).to_bytes(); - let one = u8x32::splat(s, 255); - - for (next_bg, next_mask) in dest.chunks_exact_mut(32).zip(alphas) { - alpha_composite_inner(s, next_bg, &next_mask, src_c, src_a, one); - } - }, - ); + let src_a = u8x32::splat(s, src[3]); + let src_c = u32x8::splat(s, u32::from_ne_bytes(src)).to_bytes(); + let one = u8x32::splat(s, 255); + + for (next_bg, next_mask) in dest.chunks_exact_mut(32).zip(alphas) { + alpha_composite_inner(s, next_bg, &next_mask, src_c, src_a, one); + } } /// Composites a buffer of colors with per-pixel alpha masks. /// /// Each pixel's source alpha is modulated by its corresponding mask value. - #[inline(always)] + #[simd] pub(super) fn alpha_composite>>( simd: S, dest: &mut [u8], src: T, alphas: impl Iterator, ) { - simd.vectorize( - #[inline(always)] - || { - let one = u8x32::splat(simd, 255); - - for ((next_dest, next_mask), next_src) in - dest.chunks_exact_mut(32).zip(alphas).zip(src) - { - let src_a = next_src.splat_4th(); - alpha_composite_inner(simd, next_dest, &next_mask, next_src, src_a, one); - } - }, - ); + let one = u8x32::splat(simd, 255); + + for ((next_dest, next_mask), next_src) in dest.chunks_exact_mut(32).zip(alphas).zip(src) { + let src_a = next_src.splat_4th(); + alpha_composite_inner(simd, next_dest, &next_mask, next_src, src_a, one); + } } /// Performs alpha compositing with mask modulation. /// /// Formula: `result = src * mask + bg * (1 - src_alpha * mask)` /// The mask value modulates both the source contribution and the inverse alpha. - #[inline(always)] + #[simd] fn alpha_composite_inner( s: S, dest: &mut [u8], @@ -660,24 +606,19 @@ mod alpha_fill { src_a: u8x32, one: u8x32, ) { - s.vectorize( - #[inline(always)] - || { - let bg_v = u8x32::from_slice(s, dest); - - let mask_v = extract_masks(s, masks); - let inv_src_a_mask_a = one - narrow(normalized_mul_u8(src_a, mask_v)); - - let bg = widen(bg_v); - let inv = widen(inv_src_a_mask_a); - let src = widen(src_c); - let mask = widen(mask_v); - let result = (bg * inv + src * mask).div_255(); - let res = narrow(result); - - res.store_slice(dest); - }, - ); + let bg_v = u8x32::from_slice(s, dest); + + let mask_v = extract_masks(s, masks); + let inv_src_a_mask_a = one - narrow(normalized_mul_u8(src_a, mask_v)); + + let bg = widen(bg_v); + let inv = widen(inv_src_a_mask_a); + let src = widen(src_c); + let mask = widen(mask_v); + let result = (bg * inv + src * mask).div_255(); + let res = narrow(result); + + res.store_slice(dest); } } @@ -686,7 +627,7 @@ mod alpha_fill { /// /// Input: [m0, m1, m2, m3, m4, m5, m6, m7] /// Output: [m0, m0, m0, m0, m1, m1, m1, m1, ..., m7, m7, m7, m7] -#[inline(always)] +#[simd] fn extract_masks(simd: S, masks: &[u8; 8]) -> u8x32 { let m1 = u32x4::splat(simd, u32::from_ne_bytes(masks[0..4].try_into().unwrap())).to_bytes(); let m2 = u32x4::splat(simd, u32::from_ne_bytes(masks[4..8].try_into().unwrap())).to_bytes(); @@ -740,10 +681,10 @@ mod tests { test_pack_unpack_roundtrip( width, |region, scratch| { - simd.vectorize(|| pack_block(simd, scratch, usize::from(width), region)); + pack_block(simd, scratch, usize::from(width), region); }, |region, scratch| { - simd.vectorize(|| unpack_block(simd, region, usize::from(width), scratch)); + unpack_block(simd, region, usize::from(width), scratch); }, ); }); @@ -756,10 +697,10 @@ mod tests { test_pack_unpack_roundtrip( width, |region, scratch| { - simd.vectorize(|| pack(simd, scratch, usize::from(width), region)); + pack(simd, scratch, usize::from(width), region); }, |region, scratch| { - simd.vectorize(|| unpack(simd, region, usize::from(width), scratch)); + unpack(simd, region, usize::from(width), scratch); }, ); }); diff --git a/sparse_strips/vello_cpu/src/fine/mod.rs b/sparse_strips/vello_cpu/src/fine/mod.rs index 923f5c7d0..2bdeef558 100644 --- a/sparse_strips/vello_cpu/src/fine/mod.rs +++ b/sparse_strips/vello_cpu/src/fine/mod.rs @@ -28,6 +28,7 @@ use alloc::vec; use alloc::vec::Vec; use core::fmt::Debug; use core::iter; +use fearless_simd_macros::simd; use vello_common::encode::{ EncodedBlurredRoundedRectangle, EncodedGradient, EncodedImage, EncodedKind, EncodedPaint, }; @@ -102,15 +103,15 @@ impl NumericVec for f32x16 { val } - #[inline(always)] + #[simd] fn from_u8(simd: S, val: u8x16) -> Self { - let converted = u8_to_f32(val); + let converted = u8_to_f32(simd, val); converted * Self::splat(simd, 1.0 / 255.0) } } impl NumericVec for u8x16 { - #[inline(always)] + #[simd] fn from_f32(simd: S, val: f32x16) -> Self { let v1 = f32x16::splat(simd, 255.0); let v2 = f32x16::splat(simd, 0.5); @@ -129,9 +130,8 @@ impl NumericVec for u8x16 { /// /// This function efficiently converts 16 u8 values to their f32 equivalents using SIMD operations, /// preserving the values without normalization (i.e., 255 becomes 255.0, not 1.0). -#[inline(always)] -pub(crate) fn u8_to_f32(val: u8x16) -> f32x16 { - let simd = val.simd; +#[simd] +pub(crate) fn u8_to_f32(simd: S, val: u8x16) -> f32x16 { let zeroes = u8x16::splat(simd, 0); #[cfg(target_endian = "little")] @@ -187,12 +187,12 @@ pub trait CompositeType: Copy + Clone + Send + Sync { impl CompositeType for f32x16 { const LENGTH: usize = 16; - #[inline(always)] + #[simd] fn from_slice(simd: S, slice: &[f32]) -> Self { >::from_slice(simd, slice) } - #[inline(always)] + #[simd] fn from_color(simd: S, color: [f32; 4]) -> Self { Self::block_splat(f32x4::from_slice(simd, &color[..])) } @@ -201,12 +201,12 @@ impl CompositeType for f32x16 { impl CompositeType for u8x32 { const LENGTH: usize = 32; - #[inline(always)] + #[simd] fn from_slice(simd: S, slice: &[u8]) -> Self { >::from_slice(simd, slice) } - #[inline(always)] + #[simd] fn from_color(simd: S, color: [u8; 4]) -> Self { u32x8::block_splat(u32x4::splat(simd, u32::from_ne_bytes(color))).to_bytes() } @@ -271,15 +271,13 @@ pub trait FineKernel: Send + Sync + 'static { /// /// Returns a painter that can render linear, radial, or sweep gradients based on /// pre-computed t values (gradient interpolation parameters). + #[simd] fn gradient_painter<'a>( simd: S, gradient: &'a EncodedGradient, t_vals: &'a [f32], ) -> impl Painter + 'a { - simd.vectorize( - #[inline(always)] - || GradientPainter::new(simd, gradient, t_vals), - ) + GradientPainter::new(simd, gradient, t_vals) } /// Create a painter for rendering gradients with undefined region support. @@ -289,20 +287,19 @@ pub trait FineKernel: Send + Sync + 'static { /// /// This is intentionally a duplicate of the default [`FineKernel::gradient_painter`] /// implementation--the `U8Kernel` overrides that method, but not this one. + #[simd] fn gradient_painter_with_undefined<'a>( simd: S, gradient: &'a EncodedGradient, t_vals: &'a [f32], ) -> impl Painter + 'a { - simd.vectorize( - #[inline(always)] - || GradientPainter::new(simd, gradient, t_vals), - ) + GradientPainter::new(simd, gradient, t_vals) } /// Create a painter for rendering axis-aligned nearest-neighbor images. /// /// Optimized painter for images with `Low` quality and no skewing component in their /// transform. This is the fastest image rendering path. + #[simd] fn plain_nn_image_painter<'a>( simd: S, image: &'a EncodedImage, @@ -310,16 +307,14 @@ pub trait FineKernel: Send + Sync + 'static { start_x: f64, start_y: f64, ) -> impl Painter + 'a { - simd.vectorize( - #[inline(always)] - || PlainNNImagePainter::new(simd, image, pixmap, start_x, start_y), - ) + PlainNNImagePainter::new(simd, image, pixmap, start_x, start_y) } /// Create a painter for rendering nearest-neighbor images with transforms. /// /// Similar to `plain_nn_image_painter`, but supports arbitrary affine transforms /// including skewing and rotation. + #[simd] fn nn_image_painter<'a>( simd: S, image: &'a EncodedImage, @@ -327,15 +322,13 @@ pub trait FineKernel: Send + Sync + 'static { start_x: f64, start_y: f64, ) -> impl Painter + 'a { - simd.vectorize( - #[inline(always)] - || NNImagePainter::new(simd, image, pixmap, start_x, start_y), - ) + NNImagePainter::new(simd, image, pixmap, start_x, start_y) } /// Create a painter for rendering images with `Medium` quality filtering. /// /// Uses bilinear filtering for smoother appearance than nearest-neighbor. + #[simd] fn medium_quality_image_painter<'a>( simd: S, image: &'a EncodedImage, @@ -343,15 +336,13 @@ pub trait FineKernel: Send + Sync + 'static { start_x: f64, start_y: f64, ) -> impl Painter + 'a { - simd.vectorize( - #[inline(always)] - || FilteredImagePainter::::new(simd, image, pixmap, start_x, start_y), - ) + FilteredImagePainter::::new(simd, image, pixmap, start_x, start_y) } /// Create a painter for rendering axis-aligned images with `Medium` quality filtering. /// /// Optimized painter for images with bilinear filtering and no skewing component. + #[simd] fn plain_medium_quality_image_painter<'a>( simd: S, image: &'a EncodedImage, @@ -359,15 +350,13 @@ pub trait FineKernel: Send + Sync + 'static { start_x: f64, start_y: f64, ) -> impl Painter + 'a { - simd.vectorize( - #[inline(always)] - || FilteredImagePainter::::new(simd, image, pixmap, start_x, start_y), - ) + FilteredImagePainter::::new(simd, image, pixmap, start_x, start_y) } /// Create a painter for rendering images with `High` quality filtering. /// /// Uses high-quality filtering for the best visual appearance. + #[simd] fn high_quality_image_painter<'a>( simd: S, image: &'a EncodedImage, @@ -375,26 +364,21 @@ pub trait FineKernel: Send + Sync + 'static { start_x: f64, start_y: f64, ) -> impl Painter + 'a { - simd.vectorize( - #[inline(always)] - || FilteredImagePainter::::new(simd, image, pixmap, start_x, start_y), - ) + FilteredImagePainter::::new(simd, image, pixmap, start_x, start_y) } /// Create a painter for rendering blurred rounded rectangles. /// /// Efficiently renders rounded rectangles with gaussian blur applied, /// computing the blur analytically rather than as a post-process. + #[simd] fn blurred_rounded_rectangle_painter( simd: S, rect: &EncodedBlurredRoundedRectangle, start_x: f64, start_y: f64, ) -> impl Painter { - simd.vectorize( - #[inline(always)] - || BlurredRoundedRectFiller::new(simd, rect, start_x, start_y), - ) + BlurredRoundedRectFiller::new(simd, rect, start_x, start_y) } /// Apply a mask to the destination buffer. /// @@ -709,26 +693,25 @@ impl> Fine { } } - fn opacity(&mut self, span: Span, opacity: f32) { + #[simd] + fn opacity(&mut self, simd: S, span: Span, opacity: f32) { let target = self.blend_buffers.last_mut().unwrap(); let target = &mut target[Self::scratch_range(span)]; T::apply_mask( - self.simd, + simd, target, - iter::repeat(T::NumericVec::from_f32( - self.simd, - f32x16::splat(self.simd, opacity), - )), + iter::repeat(T::NumericVec::from_f32(simd, f32x16::splat(simd, opacity))), ); } - fn mask(&mut self, row_y: u16, span: Span, mask: &Mask) { + #[simd] + fn mask(&mut self, simd: S, row_y: u16, span: Span, mask: &Mask) { let x = span.pixel_x(); let width = span.pixel_width(); let target = self.blend_buffers.last_mut().unwrap(); let target = &mut target[Self::scratch_range(span)]; - let y = u32::from(row_y) + u32x4::from_slice(self.simd, &[0, 1, 2, 3]); + let y = u32::from(row_y) + u32x4::from_slice(simd, &[0, 1, 2, 3]); let iter = (x..x.saturating_add(width)).map(|x| { let x_in_range = x < mask.width(); @@ -748,15 +731,15 @@ impl> Fine { let s4 = sample!(3); let samples = u8x16::from_slice( - self.simd, + simd, &[ s1, s1, s1, s1, s2, s2, s2, s2, s3, s3, s3, s3, s4, s4, s4, s4, ], ); - T::NumericVec::from_u8(self.simd, samples) + T::NumericVec::from_u8(simd, samples) }); - T::apply_mask(self.simd, target, iter); + T::apply_mask(simd, target, iter); } fn layer_fill( @@ -766,11 +749,12 @@ impl> Fine { attrs: &LayerFillAttrs, alphas: Option<&[u8]>, ) { + let simd = self.simd; if attrs.opacity != 1.0 { - self.opacity(span, attrs.opacity); + self.opacity(simd, span, attrs.opacity); } if let Some(mask) = attrs.mask.as_ref() { - self.mask(row_y, span, mask); + self.mask(simd, row_y, span, mask); } let x = span.pixel_x(); @@ -1139,7 +1123,7 @@ pub trait PosExt { } impl PosExt for f32x4 { - #[inline(always)] + #[simd] fn splat_pos(simd: S, pos: f32, _: f32, y_advance: f32) -> Self { let columns: [f32; Tile::HEIGHT as usize] = [0.0, 1.0, 2.0, 3.0]; let column_mask: Self = columns.simd_into(simd); @@ -1149,7 +1133,7 @@ impl PosExt for f32x4 { } impl PosExt for f32x8 { - #[inline(always)] + #[simd] fn splat_pos(simd: S, pos: f32, x_advance: f32, y_advance: f32) -> Self { simd.combine_f32x4( f32x4::splat_pos(simd, pos, x_advance, y_advance), @@ -1177,12 +1161,12 @@ impl ShaderResultF32 { /// Convert from planar format to interleaved RGBA format. /// /// Returns two sets of four f32x4 vectors containing 8 pixels (4 RGBA components each), - #[inline(always)] - pub(crate) fn get(&self) -> [[f32x4; 4]; 2] { - let (r_1, r_2) = self.r.simd.split_f32x8(self.r); - let (g_1, g_2) = self.g.simd.split_f32x8(self.g); - let (b_1, b_2) = self.b.simd.split_f32x8(self.b); - let (a_1, a_2) = self.a.simd.split_f32x8(self.a); + #[simd] + pub(crate) fn get(&self, simd: S) -> [[f32x4; 4]; 2] { + let (r_1, r_2) = simd.split_f32x8(self.r); + let (g_1, g_2) = simd.split_f32x8(self.g); + let (b_1, b_2) = simd.split_f32x8(self.b); + let (a_1, a_2) = simd.split_f32x8(self.a); [[r_1, g_1, b_1, a_1], [r_2, g_2, b_2, a_2]] } diff --git a/sparse_strips/vello_cpu/src/util.rs b/sparse_strips/vello_cpu/src/util.rs index aa11c5c16..f49cf8ecd 100644 --- a/sparse_strips/vello_cpu/src/util.rs +++ b/sparse_strips/vello_cpu/src/util.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use crate::peniko::ImageQuality; +use fearless_simd_macros::simd; use vello_common::encode::EncodedImage; use vello_common::fearless_simd::{f32x4, prelude::*, u8x32}; use vello_common::math::FloatExt; @@ -64,13 +65,13 @@ pub(crate) mod scalar { } } -pub(crate) trait NormalizedMulExt { - fn normalized_mul(self, other: Self) -> Self; +pub(crate) trait NormalizedMulExt { + fn normalized_mul(self, simd: S, other: Self) -> Self; } -impl NormalizedMulExt for u8x32 { - #[inline(always)] - fn normalized_mul(self, other: Self) -> Self { +impl NormalizedMulExt for u8x32 { + #[simd] + fn normalized_mul(self, simd: S, other: Self) -> Self { narrow(normalized_mul_u8(self, other)) } } @@ -90,24 +91,23 @@ impl EncodedImageExt for EncodedImage { } } -pub(crate) trait Premultiply { - fn premultiply(self, alphas: Self) -> Self; - fn unpremultiply(self, alphas: Self) -> Self; +pub(crate) trait Premultiply { + fn premultiply(self, simd: S, alphas: Self) -> Self; + fn unpremultiply(self, simd: S, alphas: Self) -> Self; } -impl Premultiply for f32x4 { - #[inline(always)] - fn premultiply(self, alphas: Self) -> Self { +impl Premultiply for f32x4 { + #[simd] + fn premultiply(self, simd: S, alphas: Self) -> Self { self * alphas } - #[inline(always)] - fn unpremultiply(self, alphas: Self) -> Self { - let zero = Self::splat(alphas.simd, 0.0); + #[simd] + fn unpremultiply(self, simd: S, alphas: Self) -> Self { + let zero = Self::splat(simd, 0.0); let divided = self / alphas; - self.simd - .select_f32x4(self.simd.simd_eq_f32x4(alphas, zero), zero, divided) + simd.select_f32x4(simd.simd_eq_f32x4(alphas, zero), zero, divided) } } From de2e4c1a1a8e569154f902a87e5d2ea83d3529e8 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Thu, 3 Sep 2026 14:17:12 +0100 Subject: [PATCH 3/7] Restore #[inline(always)] in the three places where it was needed for performance --- sparse_strips/vello_cpu/src/fine/common/gradient/radial.rs | 2 +- sparse_strips/vello_cpu/src/fine/common/gradient/sweep.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sparse_strips/vello_cpu/src/fine/common/gradient/radial.rs b/sparse_strips/vello_cpu/src/fine/common/gradient/radial.rs index 28b9289b8..518bb694c 100644 --- a/sparse_strips/vello_cpu/src/fine/common/gradient/radial.rs +++ b/sparse_strips/vello_cpu/src/fine/common/gradient/radial.rs @@ -52,7 +52,7 @@ impl SimdRadialKind { } impl SimdGradientKind for SimdRadialKind { - #[simd] + #[inline(always)] fn cur_pos(&self, simd: S, x_pos: f32x8, y_pos: f32x8) -> f32x8 { match &self.inner { SimdRadialKindInner::Radial { bias, scale } => { diff --git a/sparse_strips/vello_cpu/src/fine/common/gradient/sweep.rs b/sparse_strips/vello_cpu/src/fine/common/gradient/sweep.rs index b885ed58f..456ebb0b4 100644 --- a/sparse_strips/vello_cpu/src/fine/common/gradient/sweep.rs +++ b/sparse_strips/vello_cpu/src/fine/common/gradient/sweep.rs @@ -26,7 +26,7 @@ impl SimdSweepKind { } impl SimdGradientKind for SimdSweepKind { - #[simd] + #[inline(always)] fn cur_pos(&self, _simd: S, x_pos: f32x8, y_pos: f32x8) -> f32x8 { let angle = x_y_to_unit_angle(self.simd, x_pos, y_pos) * f32x8::splat(self.simd, 2.0 * PI); @@ -34,7 +34,7 @@ impl SimdGradientKind for SimdSweepKind { } } -#[simd] +#[inline(always)] fn x_y_to_unit_angle(simd: S, x: f32x8, y: f32x8) -> f32x8 { let c0 = f32x8::splat(simd, 0.0); let c1 = f32x8::splat(simd, 1.0); From 608fb05913370d36fa2fcf445bf744db7da4443e Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Fri, 4 Sep 2026 09:15:19 +0100 Subject: [PATCH 4/7] Upgrade to the latest git state --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 242680c35..1b14577de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1085,7 +1085,7 @@ dependencies = [ [[package]] name = "fearless_simd" version = "0.7.0" -source = "git+https://github.com/Shnatsel/fearless_simd.git?branch=simd-macro#e953125cb9dc6ecb9bdd088831b0d02d21e602d2" +source = "git+https://github.com/Shnatsel/fearless_simd.git?branch=simd-macro#9ad80a4051d492431c3b4c080175e85a19dae842" dependencies = [ "libm", ] @@ -1093,7 +1093,7 @@ dependencies = [ [[package]] name = "fearless_simd_macros" version = "0.1.0" -source = "git+https://github.com/Shnatsel/fearless_simd.git?branch=simd-macro#e953125cb9dc6ecb9bdd088831b0d02d21e602d2" +source = "git+https://github.com/Shnatsel/fearless_simd.git?branch=simd-macro#9ad80a4051d492431c3b4c080175e85a19dae842" dependencies = [ "proc-macro2", "quote", From 1d345282dd6674f697960c35dbfacb29618a82e9 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Fri, 4 Sep 2026 10:07:05 +0100 Subject: [PATCH 5/7] Fix pixmap/unpremultiply/opaque benchmark regression --- sparse_strips/vello_common/src/pixmap.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/sparse_strips/vello_common/src/pixmap.rs b/sparse_strips/vello_common/src/pixmap.rs index 8067a29d5..05b8f9032 100644 --- a/sparse_strips/vello_common/src/pixmap.rs +++ b/sparse_strips/vello_common/src/pixmap.rs @@ -399,6 +399,7 @@ impl Pixmap { /// If both, premultiplied and unpremultiplied RGBA are acceptable formats, it is /// recommended to choose [`ImageAlphaType::AlphaPremultiplied`], as the data can /// be returned as is without any additional post-processing. + #[inline] pub fn take_rgba8(mut self, alpha_type: ImageAlphaType) -> Vec { if self.may_have_transparency && alpha_type == ImageAlphaType::Alpha { unpremultiply_rgba8(bytemuck::cast_slice_mut(&mut self.buf)); From 1ea0d04de8f01760f9ef35f2fa38c045469dbb1b Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sat, 19 Sep 2026 14:04:57 +0100 Subject: [PATCH 6/7] Upgrade to the latest upstream state --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- vello_common/src/flatten_simd.rs | 4 ++-- vello_cpu/src/fine/common/gradient/mod.rs | 2 +- vello_cpu/src/fine/common/gradient/radial.rs | 2 +- vello_cpu/src/fine/common/gradient/sweep.rs | 2 +- vello_cpu/src/fine/common/image.rs | 4 +++- vello_cpu/src/fine/common/rounded_blurred_rect.rs | 2 +- vello_cpu/src/fine/highp/blend.rs | 2 +- vello_cpu/src/fine/highp/compose.rs | 2 +- vello_cpu/src/fine/highp/mod.rs | 6 +++--- vello_cpu/src/fine/lowp/blend.rs | 2 +- vello_cpu/src/fine/lowp/compose.rs | 2 +- vello_cpu/src/fine/lowp/gradient.rs | 2 +- vello_cpu/src/fine/lowp/image.rs | 2 +- vello_cpu/src/fine/lowp/mod.rs | 6 +++--- vello_cpu/src/fine/mod.rs | 4 ++-- vello_cpu/src/util.rs | 2 +- 18 files changed, 30 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 61b3bbd4d..117c558da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1084,8 +1084,8 @@ dependencies = [ [[package]] name = "fearless_simd" -version = "0.7.0" -source = "git+https://github.com/Shnatsel/fearless_simd.git?branch=simd-macro#9ad80a4051d492431c3b4c080175e85a19dae842" +version = "1.0.0-rc.1" +source = "git+https://github.com/linebender/fearless_simd.git?branch=main#02a63208de2d500ec039d6ca4cb1b444ac469d45" dependencies = [ "libm", ] @@ -1093,7 +1093,7 @@ dependencies = [ [[package]] name = "fearless_simd_macros" version = "0.1.0" -source = "git+https://github.com/Shnatsel/fearless_simd.git?branch=simd-macro#9ad80a4051d492431c3b4c080175e85a19dae842" +source = "git+https://github.com/linebender/fearless_simd.git?branch=main#02a63208de2d500ec039d6ca4cb1b444ac469d45" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index c7c081be9..011572250 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,7 +119,7 @@ rayon = { version = "1.12.0" } thread_local = "1.1.9" crossbeam-channel = "0.5.15" ordered-channel = { version = "1.2.0", features = ["crossbeam-channel"] } -fearless_simd = { version = "0.7.0", default-features = false } +fearless_simd = { version = "1.0.0-rc.1", default-features = false } fearless_simd_macros = "0.1.0" # Unlike the other crates, please do not update these before a release @@ -180,5 +180,5 @@ debug = true strip = "none" [patch.crates-io] -fearless_simd = {git = "https://github.com/Shnatsel/fearless_simd.git", branch = "simd-macro"} -fearless_simd_macros = {git = "https://github.com/Shnatsel/fearless_simd.git", branch = "simd-macro"} +fearless_simd = { git = "https://github.com/linebender/fearless_simd.git", branch = "main" } +fearless_simd_macros = { git = "https://github.com/linebender/fearless_simd.git", branch = "main" } diff --git a/vello_common/src/flatten_simd.rs b/vello_common/src/flatten_simd.rs index 0ab71eb93..2243039bb 100644 --- a/vello_common/src/flatten_simd.rs +++ b/vello_common/src/flatten_simd.rs @@ -369,12 +369,12 @@ fn is_finite_simd(x: f32x4) -> mask32x4 { /// SIMD version of [`approx_parabola_integral`]. #[inline(always)] fn approx_parabola_integral_simd>(x: F) -> F { - let simd = x.witness(); + let simd = x.token(); const D: f32 = 0.67; const D_POWI_4: f32 = 0.201_511_2; - let temp = F::splat(x.witness(), 0.25) + let temp = F::splat(simd, 0.25) .mul_add(x * x, F::splat(simd, D_POWI_4)) .sqrt() .sqrt(); diff --git a/vello_cpu/src/fine/common/gradient/mod.rs b/vello_cpu/src/fine/common/gradient/mod.rs index cd5a949e4..e690badac 100644 --- a/vello_cpu/src/fine/common/gradient/mod.rs +++ b/vello_cpu/src/fine/common/gradient/mod.rs @@ -7,7 +7,7 @@ use crate::peniko; use core::slice::ChunksExact; use fearless_simd_macros::simd; use vello_common::encode::{EncodedGradient, GradientLut}; -use vello_common::fearless_simd::*; +use vello_common::fearless_simd::{self, *}; pub(crate) mod linear; pub(crate) mod radial; diff --git a/vello_cpu/src/fine/common/gradient/radial.rs b/vello_cpu/src/fine/common/gradient/radial.rs index 518bb694c..2d7b77bcc 100644 --- a/vello_cpu/src/fine/common/gradient/radial.rs +++ b/vello_cpu/src/fine/common/gradient/radial.rs @@ -4,7 +4,7 @@ use crate::fine::common::gradient::SimdGradientKind; use fearless_simd_macros::simd; use vello_common::encode::{FocalData, RadialKind}; -use vello_common::fearless_simd::{Simd, SimdBase, SimdFloat, f32x8}; +use vello_common::fearless_simd::{self, Simd, SimdBase, SimdFloat, f32x8}; pub(crate) enum SimdRadialKindInner { Radial { diff --git a/vello_cpu/src/fine/common/gradient/sweep.rs b/vello_cpu/src/fine/common/gradient/sweep.rs index 456ebb0b4..8625960aa 100644 --- a/vello_cpu/src/fine/common/gradient/sweep.rs +++ b/vello_cpu/src/fine/common/gradient/sweep.rs @@ -5,7 +5,7 @@ use crate::fine::common::gradient::SimdGradientKind; use core::f32::consts::PI; use fearless_simd_macros::simd; use vello_common::encode::SweepKind; -use vello_common::fearless_simd::{Simd, SimdBase, SimdFloat, f32x8}; +use vello_common::fearless_simd::{self, Simd, SimdBase, SimdFloat, f32x8}; #[derive(Debug)] pub(crate) struct SimdSweepKind { diff --git a/vello_cpu/src/fine/common/image.rs b/vello_cpu/src/fine/common/image.rs index 701a91b04..530a481a8 100644 --- a/vello_cpu/src/fine/common/image.rs +++ b/vello_cpu/src/fine/common/image.rs @@ -6,7 +6,9 @@ use crate::fine::{PosExt, Splat4thExt, u8_to_f32}; use crate::kurbo::Point; use fearless_simd_macros::simd; use vello_common::encode::EncodedImage; -use vello_common::fearless_simd::{Bytes, Simd, SimdBase, SimdFloat, f32x4, f32x16, u8x16, u32x4}; +use vello_common::fearless_simd::{ + self, Bytes, Simd, SimdBase, SimdFloat, f32x4, f32x16, u8x16, u32x4, +}; use vello_common::pixmap::Pixmap; use vello_common::simd::element_wise_splat; diff --git a/vello_cpu/src/fine/common/rounded_blurred_rect.rs b/vello_cpu/src/fine/common/rounded_blurred_rect.rs index 1c16e4135..a1f99689d 100644 --- a/vello_cpu/src/fine/common/rounded_blurred_rect.rs +++ b/vello_cpu/src/fine/common/rounded_blurred_rect.rs @@ -9,7 +9,7 @@ use crate::fine::{NumericVec, PosExt, ShaderResultF32}; use crate::kurbo::{Point, Vec2}; use fearless_simd_macros::simd; use vello_common::encode::EncodedBlurredRoundedRectangle; -use vello_common::fearless_simd::{Simd, SimdBase, SimdFloat, f32x8, u8x16}; +use vello_common::fearless_simd::{self, Simd, SimdBase, SimdFloat, f32x8, u8x16}; #[cfg(not(feature = "std"))] use vello_common::kurbo::common::FloatFuncs as _; diff --git a/vello_cpu/src/fine/highp/blend.rs b/vello_cpu/src/fine/highp/blend.rs index cd388227d..ea31524ad 100644 --- a/vello_cpu/src/fine/highp/blend.rs +++ b/vello_cpu/src/fine/highp/blend.rs @@ -4,7 +4,7 @@ use crate::peniko::{BlendMode, Mix}; use crate::util::Premultiply; use fearless_simd_macros::simd; -use vello_common::fearless_simd::*; +use vello_common::fearless_simd::{self, *}; #[derive(Copy, Clone)] struct Channels { diff --git a/vello_cpu/src/fine/highp/compose.rs b/vello_cpu/src/fine/highp/compose.rs index 892fd175a..7fe90d33c 100644 --- a/vello_cpu/src/fine/highp/compose.rs +++ b/vello_cpu/src/fine/highp/compose.rs @@ -4,7 +4,7 @@ use crate::fine::Splat4thExt; use crate::peniko::{BlendMode, Compose}; use fearless_simd_macros::simd; -use vello_common::fearless_simd::*; +use vello_common::fearless_simd::{self, *}; pub(crate) trait ComposeExt { fn compose( diff --git a/vello_cpu/src/fine/highp/mod.rs b/vello_cpu/src/fine/highp/mod.rs index baa3d39c1..9d6e528b1 100644 --- a/vello_cpu/src/fine/highp/mod.rs +++ b/vello_cpu/src/fine/highp/mod.rs @@ -20,7 +20,7 @@ use crate::fine::{COLOR_COMPONENTS, Painter, Splat4thExt}; use crate::peniko::BlendMode; use crate::region::Region; use fearless_simd_macros::simd; -use vello_common::fearless_simd::*; +use vello_common::fearless_simd::{self, *}; use vello_common::filter_effects::Filter; use vello_common::kurbo::Affine; use vello_common::mask::Mask; @@ -299,7 +299,7 @@ mod fill { use crate::peniko::BlendMode; use fearless_simd_macros::simd; - use vello_common::fearless_simd::*; + use vello_common::fearless_simd::{self, *}; // IMPORTANT: The SIMD boundaries in this module are performance-sensitive and should be // changed only with benchmarking. @@ -376,7 +376,7 @@ mod alpha_fill { use crate::fine::highp::{blend, extract_masks}; use crate::peniko::BlendMode; use fearless_simd_macros::simd; - use vello_common::fearless_simd::*; + use vello_common::fearless_simd::{self, *}; /// Composites a solid color with per-pixel alpha masks. /// diff --git a/vello_cpu/src/fine/lowp/blend.rs b/vello_cpu/src/fine/lowp/blend.rs index dfd3a852a..38601f0b7 100644 --- a/vello_cpu/src/fine/lowp/blend.rs +++ b/vello_cpu/src/fine/lowp/blend.rs @@ -4,7 +4,7 @@ use crate::fine::{Splat4thExt, highp, u8_to_f32}; use crate::peniko::{BlendMode, Mix}; use fearless_simd_macros::simd; -use vello_common::fearless_simd::*; +use vello_common::fearless_simd::{self, *}; use vello_common::util::{Div255Ext, f32_to_u8, normalized_mul_u8, saturating_narrow, widen}; #[simd] diff --git a/vello_cpu/src/fine/lowp/compose.rs b/vello_cpu/src/fine/lowp/compose.rs index 0487313a4..895ddcd85 100644 --- a/vello_cpu/src/fine/lowp/compose.rs +++ b/vello_cpu/src/fine/lowp/compose.rs @@ -5,7 +5,7 @@ use crate::fine::Splat4thExt; use crate::peniko::{BlendMode, Compose}; use crate::util::NormalizedMulExt; use fearless_simd_macros::simd; -use vello_common::fearless_simd::*; +use vello_common::fearless_simd::{self, *}; use vello_common::util::{Div255Ext, narrow, widen}; pub(crate) trait ComposeExt { diff --git a/vello_cpu/src/fine/lowp/gradient.rs b/vello_cpu/src/fine/lowp/gradient.rs index 883feeadd..6b9a95af9 100644 --- a/vello_cpu/src/fine/lowp/gradient.rs +++ b/vello_cpu/src/fine/lowp/gradient.rs @@ -5,7 +5,7 @@ use crate::peniko; use core::slice::ChunksExact; use fearless_simd_macros::simd; use vello_common::encode::EncodedGradient; -use vello_common::fearless_simd::*; +use vello_common::fearless_simd::{self, *}; /// An accelerated gradient painter for u8. /// diff --git a/vello_cpu/src/fine/lowp/image.rs b/vello_cpu/src/fine/lowp/image.rs index d86943ea3..b5da1f979 100644 --- a/vello_cpu/src/fine/lowp/image.rs +++ b/vello_cpu/src/fine/lowp/image.rs @@ -6,7 +6,7 @@ use crate::fine::common::image::{ImagePainterData, extend, fract_floor, sample}; use crate::fine::macros::u8x16_painter; use fearless_simd_macros::simd; use vello_common::encode::EncodedImage; -use vello_common::fearless_simd::{f32x4, prelude::*, u8x16, u16x16}; +use vello_common::fearless_simd::{self, f32x4, prelude::*, u8x16, u16x16}; use vello_common::pixmap::Pixmap; use vello_common::simd::element_wise_splat; use vello_common::util::{Div255Ext, f32_to_u8, narrow, widen}; diff --git a/vello_cpu/src/fine/lowp/mod.rs b/vello_cpu/src/fine/lowp/mod.rs index 36a2dba90..24be8a1da 100644 --- a/vello_cpu/src/fine/lowp/mod.rs +++ b/vello_cpu/src/fine/lowp/mod.rs @@ -25,7 +25,7 @@ use bytemuck::{cast_slice, cast_slice_mut}; use core::iter; use fearless_simd_macros::simd; use vello_common::encode::{EncodedGradient, EncodedImage}; -use vello_common::fearless_simd::*; +use vello_common::fearless_simd::{self, *}; use vello_common::filter_effects::Filter; use vello_common::kurbo::Affine; use vello_common::mask::Mask; @@ -441,7 +441,7 @@ mod fill { use crate::fine::lowp::compose::ComposeExt; use crate::peniko::{BlendMode, Mix}; use fearless_simd_macros::simd; - use vello_common::fearless_simd::*; + use vello_common::fearless_simd::{self, *}; use vello_common::util::{narrow, normalized_mul_u8}; /// Applies blend mode compositing to a buffer without per-pixel masks. @@ -528,7 +528,7 @@ mod alpha_fill { use crate::fine::lowp::{blend, extract_masks}; use crate::peniko::{BlendMode, Mix}; use fearless_simd_macros::simd; - use vello_common::fearless_simd::*; + use vello_common::fearless_simd::{self, *}; use vello_common::util::{Div255Ext, narrow, normalized_mul_u8, widen}; /// Applies blend mode compositing with per-pixel alpha masks. diff --git a/vello_cpu/src/fine/mod.rs b/vello_cpu/src/fine/mod.rs index df2c5c715..bdb6914bd 100644 --- a/vello_cpu/src/fine/mod.rs +++ b/vello_cpu/src/fine/mod.rs @@ -35,8 +35,8 @@ use vello_common::encode::{ EncodedBlurredRoundedRectangle, EncodedGradient, EncodedImage, EncodedKind, EncodedPaint, }; use vello_common::fearless_simd::{ - Bytes, Simd, SimdBase, SimdFloat, SimdInt, SimdInto, f32x4, f32x8, f32x16, u8x16, u8x32, u32x4, - u32x8, + self, Bytes, Simd, SimdBase, SimdFloat, SimdInt, SimdInto, f32x4, f32x8, f32x16, u8x16, u8x32, + u32x4, u32x8, }; use vello_common::filter_effects::Filter; use vello_common::kurbo::Affine; diff --git a/vello_cpu/src/util.rs b/vello_cpu/src/util.rs index f49cf8ecd..d97c63218 100644 --- a/vello_cpu/src/util.rs +++ b/vello_cpu/src/util.rs @@ -4,7 +4,7 @@ use crate::peniko::ImageQuality; use fearless_simd_macros::simd; use vello_common::encode::EncodedImage; -use vello_common::fearless_simd::{f32x4, prelude::*, u8x32}; +use vello_common::fearless_simd::{self, f32x4, prelude::*, u8x32}; use vello_common::math::FloatExt; use vello_common::tile::Tile; use vello_common::util::{narrow, normalized_mul_u8}; From bccff6a98c1359d2c4a24a661667b81b04e365ad Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sat, 19 Sep 2026 14:11:10 +0100 Subject: [PATCH 7/7] Remove now-redundant `_simd: S,` thanks to improvements to the #[simd] macro --- vello_cpu/src/fine/common/gradient/mod.rs | 4 +- vello_cpu/src/fine/common/image.rs | 11 +--- vello_cpu/src/fine/highp/blend.rs | 75 +++++++---------------- vello_cpu/src/fine/lowp/compose.rs | 9 ++- vello_cpu/src/fine/lowp/gradient.rs | 4 +- vello_cpu/src/fine/lowp/image.rs | 8 +-- vello_cpu/src/fine/lowp/mod.rs | 11 ++-- vello_cpu/src/util.rs | 8 +-- 8 files changed, 46 insertions(+), 84 deletions(-) diff --git a/vello_cpu/src/fine/common/gradient/mod.rs b/vello_cpu/src/fine/common/gradient/mod.rs index e690badac..25558b858 100644 --- a/vello_cpu/src/fine/common/gradient/mod.rs +++ b/vello_cpu/src/fine/common/gradient/mod.rs @@ -72,7 +72,7 @@ impl Iterator for GradientPainter<'_, S> { fn next(&mut self) -> Option { let extend = self.gradient.extend; let pos = f32x8::from_slice(self.simd, self.t_vals.next()?); - let t_vals = apply_extend(self.simd, pos, extend); + let t_vals = apply_extend(pos, extend); let valid = pos.simd_eq(pos); let indices = (t_vals * self.scale_factor).to_int::>(); // In case we had any NaN's, set the index to an explicit invalid sentinel. There @@ -221,7 +221,7 @@ fn invalid_f32_mask(simd: S, indices: u32x4) -> mask32x16 { } #[simd] -pub(crate) fn apply_extend(_simd: S, val: f32x8, extend: peniko::Extend) -> f32x8 { +pub(crate) fn apply_extend(val: f32x8, extend: peniko::Extend) -> f32x8 { match extend { peniko::Extend::Pad => val.max(0.0).min(1.0), peniko::Extend::Repeat => (val - val.floor()).fract(), diff --git a/vello_cpu/src/fine/common/image.rs b/vello_cpu/src/fine/common/image.rs index 530a481a8..42d5710f0 100644 --- a/vello_cpu/src/fine/common/image.rs +++ b/vello_cpu/src/fine/common/image.rs @@ -206,8 +206,8 @@ impl Iterator for FilteredImagePainter<'_, S, QUALIT // center of the location we are sampling, and sample those points // using a cubic filter to weight each location's contribution. - let x_fract = fract_floor(self.simd, x_positions + 0.5); - let y_fract = fract_floor(self.simd, y_positions + 0.5); + let x_fract = fract_floor(x_positions + 0.5); + let y_fract = fract_floor(y_positions + 0.5); let mut interpolated_color = f32x16::splat(self.simd, 0.0); @@ -345,7 +345,7 @@ f32x16_painter!(FilteredImagePainter<'_, S, 2>); /// Unlike `f32::fract()`, this always returns a value in [0, 1), /// even for negative inputs. #[simd] -pub(crate) fn fract_floor(_simd: S, val: f32x4) -> f32x4 { +pub(crate) fn fract_floor(val: f32x4) -> f32x4 { val - val.floor() } @@ -470,7 +470,6 @@ fn weights(simd: S, fract: f32x4) -> [f32x4; 4] { [ single_weight( - simd, fract, f32x4::splat(simd, MF[0][0]), f32x4::splat(simd, MF[0][1]), @@ -478,7 +477,6 @@ fn weights(simd: S, fract: f32x4) -> [f32x4; 4] { f32x4::splat(simd, MF[0][3]), ), single_weight( - simd, fract, f32x4::splat(simd, MF[1][0]), f32x4::splat(simd, MF[1][1]), @@ -486,7 +484,6 @@ fn weights(simd: S, fract: f32x4) -> [f32x4; 4] { f32x4::splat(simd, MF[1][3]), ), single_weight( - simd, fract, f32x4::splat(simd, MF[2][0]), f32x4::splat(simd, MF[2][1]), @@ -494,7 +491,6 @@ fn weights(simd: S, fract: f32x4) -> [f32x4; 4] { f32x4::splat(simd, MF[2][3]), ), single_weight( - simd, fract, f32x4::splat(simd, MF[3][0]), f32x4::splat(simd, MF[3][1]), @@ -507,7 +503,6 @@ fn weights(simd: S, fract: f32x4) -> [f32x4; 4] { /// Calculate a weight based on the fractional value t and the cubic coefficients. #[simd] fn single_weight( - _simd: S, t: f32x4, a: f32x4, b: f32x4, diff --git a/vello_cpu/src/fine/highp/blend.rs b/vello_cpu/src/fine/highp/blend.rs index ea31524ad..434a27694 100644 --- a/vello_cpu/src/fine/highp/blend.rs +++ b/vello_cpu/src/fine/highp/blend.rs @@ -45,9 +45,9 @@ pub(crate) fn mix( let mut res_bg = unpremultiplied_bg; let mix_src = blend_mode.mix(simd, unpremultiplied_src, unpremultiplied_bg); - res_bg.r = apply_alpha(simd, bg_a, src_a, unpremultiplied_src.r, mix_src.r); - res_bg.g = apply_alpha(simd, bg_a, src_a, unpremultiplied_src.g, mix_src.g); - res_bg.b = apply_alpha(simd, bg_a, src_a, unpremultiplied_src.b, mix_src.b); + res_bg.r = apply_alpha(bg_a, src_a, unpremultiplied_src.r, mix_src.r); + res_bg.g = apply_alpha(bg_a, src_a, unpremultiplied_src.g, mix_src.g); + res_bg.b = apply_alpha(bg_a, src_a, unpremultiplied_src.b, mix_src.b); let mut storage = [0.0; 16]; simd.store_four_interleaved_f32x4([res_bg.r, res_bg.g, res_bg.b, src_a], &mut storage); @@ -65,7 +65,6 @@ fn split(simd: S, input: f32x16) -> (Channels, f32x4) { #[simd] fn apply_alpha( - simd: S, bg_a: f32x4, src_a: f32x4, unpremultiplied_src_channel: f32x4, @@ -74,7 +73,7 @@ fn apply_alpha( let p1 = (1.0 - bg_a) * unpremultiplied_src_channel; let p2 = bg_a * mix_src_channel; - (p1 + p2).premultiply(simd, src_a) + (p1 + p2).premultiply(src_a) } trait MixExt { @@ -107,14 +106,14 @@ impl MixExt for BlendMode { impl Multiply { #[simd] - fn single(simd: S, src: f32x4, bg: f32x4) -> f32x4 { + fn single(src: f32x4, bg: f32x4) -> f32x4 { src * bg } } impl Screen { #[simd] - fn single(simd: S, src: f32x4, bg: f32x4) -> f32x4 { + fn single(src: f32x4, bg: f32x4) -> f32x4 { bg + src - src * bg } } @@ -125,8 +124,8 @@ impl HardLight { let two = f32x4::splat(simd, 2.0); let mask = simd.simd_le_f32x4(src, f32x4::splat(simd, 0.5)); - let opt1 = Multiply::single(simd, bg, src * two); - let opt2 = Screen::single(simd, bg, two * src - 1.0); + let opt1 = Multiply::single(bg, src * two); + let opt2 = Screen::single(bg, two * src - 1.0); simd.select_f32x4(mask, opt1, opt2) } @@ -149,11 +148,11 @@ macro_rules! separable_mix { }; } -separable_mix!(Multiply, |simd: S, cs: f32x4, cb: f32x4| { - Multiply::single(simd, cs, cb) +separable_mix!(Multiply, |_: S, cs: f32x4, cb: f32x4| { + Multiply::single(cs, cb) }); -separable_mix!(Screen, |simd: S, cs: f32x4, cb: f32x4| { - Screen::single(simd, cs, cb) +separable_mix!(Screen, |_: S, cs: f32x4, cb: f32x4| { + Screen::single(cs, cb) }); separable_mix!(Overlay, |simd: S, cs: f32x4, cb: f32x4| { HardLight::single(simd, cb, cs) @@ -233,20 +232,8 @@ macro_rules! non_separable_mix { non_separable_mix!(Hue, |simd: S, cs: &mut Channels, cb: &mut Channels| { - set_sat( - simd, - &mut cs.r, - &mut cs.g, - &mut cs.b, - sat(simd, cb.r, cb.g, cb.b), - ); - set_lum( - simd, - &mut cs.r, - &mut cs.g, - &mut cs.b, - lum(simd, cb.r, cb.g, cb.b), - ); + set_sat(simd, &mut cs.r, &mut cs.g, &mut cs.b, sat(cb.r, cb.g, cb.b)); + set_lum(simd, &mut cs.r, &mut cs.g, &mut cs.b, lum(cb.r, cb.g, cb.b)); *cs }); @@ -254,14 +241,8 @@ non_separable_mix!(Hue, |simd: S, non_separable_mix!( Saturation, |simd: S, cs: &mut Channels, cb: &mut Channels| { - let lum = lum(simd, cb.r, cb.g, cb.b); - set_sat( - simd, - &mut cb.r, - &mut cb.g, - &mut cb.b, - sat(simd, cs.r, cs.g, cs.b), - ); + let lum = lum(cb.r, cb.g, cb.b); + set_sat(simd, &mut cb.r, &mut cb.g, &mut cb.b, sat(cs.r, cs.g, cs.b)); set_lum(simd, &mut cb.r, &mut cb.g, &mut cb.b, lum); *cb @@ -271,44 +252,32 @@ non_separable_mix!( non_separable_mix!(Color, |simd: S, cs: &mut Channels, cb: &mut Channels| { - set_lum( - simd, - &mut cs.r, - &mut cs.g, - &mut cs.b, - lum(simd, cb.r, cb.g, cb.b), - ); + set_lum(simd, &mut cs.r, &mut cs.g, &mut cs.b, lum(cb.r, cb.g, cb.b)); *cs }); non_separable_mix!( Luminosity, |simd: S, cs: &mut Channels, cb: &mut Channels| { - set_lum( - simd, - &mut cb.r, - &mut cb.g, - &mut cb.b, - lum(simd, cs.r, cs.g, cs.b), - ); + set_lum(simd, &mut cb.r, &mut cb.g, &mut cb.b, lum(cs.r, cs.g, cs.b)); *cb } ); #[simd] -fn lum(simd: S, r: f32x4, g: f32x4, b: f32x4) -> f32x4 { +fn lum(r: f32x4, g: f32x4, b: f32x4) -> f32x4 { 0.3 * r + 0.59 * g + 0.11 * b } #[simd] -fn sat(simd: S, r: f32x4, g: f32x4, b: f32x4) -> f32x4 { +fn sat(r: f32x4, g: f32x4, b: f32x4) -> f32x4 { r.max(g).max(b) - r.min(g).min(b) } #[simd] fn clip_color(simd: S, r: &mut f32x4, g: &mut f32x4, b: &mut f32x4) { - let l = lum(simd, *r, *g, *b); + let l = lum(*r, *g, *b); let n = r.min(g.min(*b)); let x = r.max(g.max(*b)); @@ -329,7 +298,7 @@ fn clip_color(simd: S, r: &mut f32x4, g: &mut f32x4, b: &mut f32x #[simd] fn set_lum(simd: S, r: &mut f32x4, g: &mut f32x4, b: &mut f32x4, l: f32x4) { - let d = l - lum(simd, *r, *g, *b); + let d = l - lum(*r, *g, *b); *r += d; *g += d; *b += d; diff --git a/vello_cpu/src/fine/lowp/compose.rs b/vello_cpu/src/fine/lowp/compose.rs index 895ddcd85..da0538e48 100644 --- a/vello_cpu/src/fine/lowp/compose.rs +++ b/vello_cpu/src/fine/lowp/compose.rs @@ -71,13 +71,12 @@ macro_rules! compose { if $sat { narrow( - (widen(src_c.normalized_mul(simd, fa)) - + widen(fb.normalized_mul(simd, bg_c))) - .min(u16x32::splat(simd, 255)) - .max(u16x32::splat(simd, 0)), + (widen(src_c.normalized_mul(fa)) + widen(fb.normalized_mul(bg_c))) + .min(u16x32::splat(simd, 255)) + .max(u16x32::splat(simd, 0)), ) } else { - src_c.normalized_mul(simd, fa) + fb.normalized_mul(simd, bg_c) + src_c.normalized_mul(fa) + fb.normalized_mul(bg_c) } } } diff --git a/vello_cpu/src/fine/lowp/gradient.rs b/vello_cpu/src/fine/lowp/gradient.rs index 6b9a95af9..336642aba 100644 --- a/vello_cpu/src/fine/lowp/gradient.rs +++ b/vello_cpu/src/fine/lowp/gradient.rs @@ -42,7 +42,7 @@ impl Iterator for GradientPainter<'_, S> { fn next(&mut self) -> Option { let extend = self.gradient.extend; let pos = f32x16::from_slice(self.simd, self.t_vals.next()?); - let t_vals = apply_extend(self.simd, pos, extend); + let t_vals = apply_extend(pos, extend); let indices = (t_vals * self.scale_factor).to_int::>(); let mut vals = [0_u8; 64]; @@ -73,7 +73,7 @@ impl crate::fine::Painter for GradientPainter<'_, S> { // TODO: Maybe delete this method and use `apply_extend` from highp by splitting into two f32x8. #[simd] -pub(crate) fn apply_extend(simd: S, val: f32x16, extend: peniko::Extend) -> f32x16 { +pub(crate) fn apply_extend(val: f32x16, extend: peniko::Extend) -> f32x16 { match extend { peniko::Extend::Pad => val.max(0.0).min(1.0), peniko::Extend::Repeat => (val - val.floor()).fract(), diff --git a/vello_cpu/src/fine/lowp/image.rs b/vello_cpu/src/fine/lowp/image.rs index b5da1f979..1e5e43da9 100644 --- a/vello_cpu/src/fine/lowp/image.rs +++ b/vello_cpu/src/fine/lowp/image.rs @@ -73,11 +73,11 @@ impl Iterator for BilinearImagePainter<'_, S> { let fx = f32_to_u8(element_wise_splat( self.simd, - fract_floor(self.simd, x_positions + 0.5).mul_add(255.0, 0.5), + fract_floor(x_positions + 0.5).mul_add(255.0, 0.5), )); let fy = f32_to_u8(element_wise_splat( self.simd, - fract_floor(self.simd, y_positions + 0.5).mul_add(255.0, 0.5), + fract_floor(y_positions + 0.5).mul_add(255.0, 0.5), )); let fx = widen(fx); @@ -167,7 +167,7 @@ impl<'a, S: Simd> PlainBilinearImagePainter<'a, S> { // Pre-compute y interpolation weights let fy = f32_to_u8(element_wise_splat( simd, - fract_floor(simd, y_positions + 0.5).mul_add(255.0, 0.5), + fract_floor(y_positions + 0.5).mul_add(255.0, 0.5), )); let fy = widen(fy); let fy_inv = u16x16::splat(simd, 255) - fy; @@ -219,7 +219,7 @@ impl Iterator for PlainBilinearImagePainter<'_, S> { // Compute x interpolation weights let fx = f32_to_u8(element_wise_splat( self.simd, - fract_floor(self.simd, x_plus_half).mul_add(255.0, 0.5), + fract_floor(x_plus_half).mul_add(255.0, 0.5), )); let fx = widen(fx); let fx_inv = u16x16::splat(self.simd, 255) - fx; diff --git a/vello_cpu/src/fine/lowp/mod.rs b/vello_cpu/src/fine/lowp/mod.rs index 24be8a1da..26a056dc6 100644 --- a/vello_cpu/src/fine/lowp/mod.rs +++ b/vello_cpu/src/fine/lowp/mod.rs @@ -154,14 +154,14 @@ impl FineKernel for U8Kernel { for chunk in dest.chunks_exact_mut(32) { let pixel = u8x32::from_slice(simd, chunk); let alphas = pixel.splat_4th(); - let tinted = tint_v.normalized_mul(simd, alphas); + let tinted = tint_v.normalized_mul(alphas); tinted.store_slice(chunk); } } TintMode::Multiply => { for chunk in dest.chunks_exact_mut(32) { let pixel = u8x32::from_slice(simd, chunk); - let tinted = pixel.normalized_mul(simd, tint_v); + let tinted = pixel.normalized_mul(tint_v); tinted.store_slice(chunk); } } @@ -478,8 +478,8 @@ mod fill { // but since we widen to u16, we can only work with 256 bits, so we split it up. let bg_v = u8x64::from_slice(s, next_dest); let (bg_1, bg_2) = s.split_u8x64(bg_v); - let res_1 = alpha_composite_inner(s, bg_1, src_c, one_minus_alpha); - let res_2 = alpha_composite_inner(s, bg_2, src_c, one_minus_alpha); + let res_1 = alpha_composite_inner(bg_1, src_c, one_minus_alpha); + let res_2 = alpha_composite_inner(bg_2, src_c, one_minus_alpha); let combined = s.combine_u8x32(res_1, res_2); combined.store_slice(next_dest); } @@ -497,7 +497,7 @@ mod fill { for (next_dest, next_src) in dest.chunks_exact_mut(32).zip(src) { let one_minus_alpha = 255 - next_src.splat_4th(); let bg_v = u8x32::from_slice(simd, next_dest); - let res = alpha_composite_inner(simd, bg_v, next_src, one_minus_alpha); + let res = alpha_composite_inner(bg_v, next_src, one_minus_alpha); res.store_slice(next_dest); } } @@ -508,7 +508,6 @@ mod fill { /// This implements the Porter-Duff "source over" operator. #[simd] fn alpha_composite_inner( - _s: S, bg: u8x32, src: u8x32, one_minus_alpha: u8x32, diff --git a/vello_cpu/src/util.rs b/vello_cpu/src/util.rs index d97c63218..3545f4d9a 100644 --- a/vello_cpu/src/util.rs +++ b/vello_cpu/src/util.rs @@ -66,12 +66,12 @@ pub(crate) mod scalar { } pub(crate) trait NormalizedMulExt { - fn normalized_mul(self, simd: S, other: Self) -> Self; + fn normalized_mul(self, other: Self) -> Self; } impl NormalizedMulExt for u8x32 { #[simd] - fn normalized_mul(self, simd: S, other: Self) -> Self { + fn normalized_mul(self, other: Self) -> Self { narrow(normalized_mul_u8(self, other)) } } @@ -92,13 +92,13 @@ impl EncodedImageExt for EncodedImage { } pub(crate) trait Premultiply { - fn premultiply(self, simd: S, alphas: Self) -> Self; + fn premultiply(self, alphas: Self) -> Self; fn unpremultiply(self, simd: S, alphas: Self) -> Self; } impl Premultiply for f32x4 { #[simd] - fn premultiply(self, simd: S, alphas: Self) -> Self { + fn premultiply(self, alphas: Self) -> Self { self * alphas }