From 41a6ff30214f6616c87831f9eafc0d3bd0b3779f Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 20:23:42 +0100 Subject: [PATCH 1/5] Optimize swizzle_dyn for double the native vector width on SSE4.2, AVX2 and WASM --- crates/core_simd/src/swizzle_dyn.rs | 60 +++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/crates/core_simd/src/swizzle_dyn.rs b/crates/core_simd/src/swizzle_dyn.rs index 95fe086a4b4..30e6d18451e 100644 --- a/crates/core_simd/src/swizzle_dyn.rs +++ b/crates/core_simd/src/swizzle_dyn.rs @@ -41,6 +41,8 @@ impl Simd { 16 => transize(x86::_mm_shuffle_epi8, self, zeroing_idxs(idxs)), #[cfg(target_feature = "simd128")] 16 => transize(wasm::i8x16_swizzle, self, idxs), + #[cfg(target_feature = "simd128")] + 32 => transize(swizzle_dyn_split::<32, 16>, self, idxs), #[cfg(all( target_arch = "arm", target_feature = "v7", @@ -64,9 +66,17 @@ impl Simd { }; transize(swizzler, self, idxs) } + #[cfg(all( + target_feature = "ssse3", + not(target_feature = "avx2"), + not(target_feature = "avx512vbmi") + ))] + 32 => transize(swizzle_dyn_split::<32, 16>, self, idxs), #[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))] 32 => transize(loong64_lasx_swizzle, self, idxs), // Notable absence: avx512bw pshufb shuffle + #[cfg(all(target_feature = "avx2", not(target_feature = "avx512vbmi")))] + 64 => transize(swizzle_dyn_split::<64, 32>, self, idxs), #[cfg(all(target_feature = "avx512vl", target_feature = "avx512vbmi"))] 64 => { // Unlike vpshufb, vpermb doesn't zero out values in the result based on the index high bit @@ -93,6 +103,56 @@ impl Simd { } } +#[cfg(any( + target_feature = "simd128", + all( + target_feature = "ssse3", + not(target_feature = "avx2"), + not(target_feature = "avx512vbmi") + ), + all(target_feature = "avx2", not(target_feature = "avx512vbmi")) +))] +/// Implements an arbitrary shuffle over double the native vector width +/// using 4 native-width shuffles +fn swizzle_dyn_split( + bytes: Simd, + idxs: Simd, +) -> Simd { + let table_low = bytes.extract::<0, HALF>(); + let table_high = bytes.extract::(); + let idxs_low = idxs.extract::<0, HALF>(); + let idxs_high = idxs.extract::(); + let table_high_offset = Simd::::splat(HALF as u8); + + let output_low_from_low = table_low.swizzle_dyn(idxs_low); + let output_low_from_high = table_high.swizzle_dyn(idxs_low - table_high_offset); + let output_low = output_low_from_low | output_low_from_high; + + let output_high_from_low = table_low.swizzle_dyn(idxs_high); + let output_high_from_high = table_high.swizzle_dyn(idxs_high - table_high_offset); + let output_high = output_high_from_low | output_high_from_high; + + // This is simply a concatenation of two native-sized vectors. + // The swizzle does nothing - it maps the elements right back where they already are. + // There doesn't seem to be a more direct way to do this as of this writing. + // TODO: simplify once a plain `concat` is available. + use crate::simd::Swizzle; + struct CombineHalves; + impl Swizzle for CombineHalves { + const INDEX: [usize; N] = const { + let mut index = [0; N]; + let mut i = 0; + while i < N { + index[i] = i; + i += 1; + } + index + }; + } + + CombineHalves::concat_swizzle(output_low, output_high) +} + /// armv7 neon supports swizzling `u8x16` by swizzling two u8x8 blocks /// with a u8x8x2 lookup table. /// From 9659c20ced5fe77824dd5ffd3cdb82175caf318f Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 23:41:26 +0100 Subject: [PATCH 2/5] Use swizzle_dyn_split for LSX and LASX as well --- crates/core_simd/src/swizzle_dyn.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/core_simd/src/swizzle_dyn.rs b/crates/core_simd/src/swizzle_dyn.rs index 30e6d18451e..07d0c1ded14 100644 --- a/crates/core_simd/src/swizzle_dyn.rs +++ b/crates/core_simd/src/swizzle_dyn.rs @@ -52,6 +52,12 @@ impl Simd { 16 => transize(armv7_neon_swizzle_u8x16, self, idxs), #[cfg(all(target_arch = "loongarch64", target_feature = "lsx"))] 16 => transize(loong64_lsx_swizzle, self, idxs), + #[cfg(all( + target_arch = "loongarch64", + target_feature = "lsx", + not(target_feature = "lasx") + ))] + 32 => transize(swizzle_dyn_split::<32, 16>, self, idxs), #[cfg(all(target_feature = "avx2", not(target_feature = "avx512vbmi")))] 32 => transize(avx2_pshufb, self, idxs), #[cfg(all(target_feature = "avx512vl", target_feature = "avx512vbmi"))] @@ -77,6 +83,8 @@ impl Simd { // Notable absence: avx512bw pshufb shuffle #[cfg(all(target_feature = "avx2", not(target_feature = "avx512vbmi")))] 64 => transize(swizzle_dyn_split::<64, 32>, self, idxs), + #[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))] + 64 => transize(swizzle_dyn_split::<64, 32>, self, idxs), #[cfg(all(target_feature = "avx512vl", target_feature = "avx512vbmi"))] 64 => { // Unlike vpshufb, vpermb doesn't zero out values in the result based on the index high bit @@ -110,7 +118,13 @@ impl Simd { not(target_feature = "avx2"), not(target_feature = "avx512vbmi") ), - all(target_feature = "avx2", not(target_feature = "avx512vbmi")) + all(target_feature = "avx2", not(target_feature = "avx512vbmi")), + all( + target_arch = "loongarch64", + target_feature = "lsx", + not(target_feature = "lasx") + ), + all(target_arch = "loongarch64", target_feature = "lasx") ))] /// Implements an arbitrary shuffle over double the native vector width /// using 4 native-width shuffles From f8aa515d15795524963988b04e2d0bc4d6b5bab4 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 22 Jul 2026 08:47:08 +0100 Subject: [PATCH 3/5] #[allow(dead_code)] instead of a humongous cfg --- crates/core_simd/src/swizzle_dyn.rs | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/crates/core_simd/src/swizzle_dyn.rs b/crates/core_simd/src/swizzle_dyn.rs index 07d0c1ded14..22ede383739 100644 --- a/crates/core_simd/src/swizzle_dyn.rs +++ b/crates/core_simd/src/swizzle_dyn.rs @@ -111,21 +111,7 @@ impl Simd { } } -#[cfg(any( - target_feature = "simd128", - all( - target_feature = "ssse3", - not(target_feature = "avx2"), - not(target_feature = "avx512vbmi") - ), - all(target_feature = "avx2", not(target_feature = "avx512vbmi")), - all( - target_arch = "loongarch64", - target_feature = "lsx", - not(target_feature = "lasx") - ), - all(target_arch = "loongarch64", target_feature = "lasx") -))] +#[allow(dead_code, reason = "only used on some targets/features")] /// Implements an arbitrary shuffle over double the native vector width /// using 4 native-width shuffles fn swizzle_dyn_split( From 072095cf12ce56db3400c5fbfcffbd72616ed6ce Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 22 Jul 2026 09:59:02 +0100 Subject: [PATCH 4/5] Add #[allow(unreachable_patterns)], remove all #[cfg(not(...)] and rely on match branch priority for selecting the best implementation instead --- crates/core_simd/src/swizzle_dyn.rs | 55 ++++++++++++++++------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/crates/core_simd/src/swizzle_dyn.rs b/crates/core_simd/src/swizzle_dyn.rs index 22ede383739..36d6bfe1adc 100644 --- a/crates/core_simd/src/swizzle_dyn.rs +++ b/crates/core_simd/src/swizzle_dyn.rs @@ -30,19 +30,20 @@ impl Simd { use core::arch::x86_64 as x86; // SAFETY: Intrinsics covered by cfg unsafe { + #[allow( + unreachable_patterns, + reason = "avoids writing verbose cfg(not), earlier branches take priority" + )] match N { + // Aarch64 #[cfg(all( any(target_arch = "aarch64", target_arch = "arm64ec"), target_feature = "neon", target_endian = "little" ))] 8 | 16 | 24 | 32 | 48 | 64 => aarch64_swizzle(self, idxs), - #[cfg(target_feature = "ssse3")] - 16 => transize(x86::_mm_shuffle_epi8, self, zeroing_idxs(idxs)), - #[cfg(target_feature = "simd128")] - 16 => transize(wasm::i8x16_swizzle, self, idxs), - #[cfg(target_feature = "simd128")] - 32 => transize(swizzle_dyn_split::<32, 16>, self, idxs), + + // 32-bit ARMv7 #[cfg(all( target_arch = "arm", target_feature = "v7", @@ -50,16 +51,26 @@ impl Simd { target_endian = "little" ))] 16 => transize(armv7_neon_swizzle_u8x16, self, idxs), + + // WASM SIMD128 + #[cfg(target_feature = "simd128")] + 16 => transize(wasm::i8x16_swizzle, self, idxs), + #[cfg(target_feature = "simd128")] + 32 => transize(swizzle_dyn_split::<32, 16>, self, idxs), + + // LoongArch64 #[cfg(all(target_arch = "loongarch64", target_feature = "lsx"))] 16 => transize(loong64_lsx_swizzle, self, idxs), - #[cfg(all( - target_arch = "loongarch64", - target_feature = "lsx", - not(target_feature = "lasx") - ))] + #[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))] + 32 => transize(loong64_lasx_swizzle, self, idxs), + #[cfg(all(target_arch = "loongarch64", target_feature = "lsx",))] 32 => transize(swizzle_dyn_split::<32, 16>, self, idxs), - #[cfg(all(target_feature = "avx2", not(target_feature = "avx512vbmi")))] - 32 => transize(avx2_pshufb, self, idxs), + #[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))] + 64 => transize(swizzle_dyn_split::<64, 32>, self, idxs), + + // x86, x86-64 + #[cfg(target_feature = "ssse3")] + 16 => transize(x86::_mm_shuffle_epi8, self, zeroing_idxs(idxs)), #[cfg(all(target_feature = "avx512vl", target_feature = "avx512vbmi"))] 32 => { // Unlike vpshufb, vpermb doesn't zero out values in the result based on the index high bit @@ -72,19 +83,11 @@ impl Simd { }; transize(swizzler, self, idxs) } - #[cfg(all( - target_feature = "ssse3", - not(target_feature = "avx2"), - not(target_feature = "avx512vbmi") - ))] + #[cfg(target_feature = "avx2")] + 32 => transize(avx2_pshufb, self, idxs), + #[cfg(target_feature = "ssse3")] 32 => transize(swizzle_dyn_split::<32, 16>, self, idxs), - #[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))] - 32 => transize(loong64_lasx_swizzle, self, idxs), // Notable absence: avx512bw pshufb shuffle - #[cfg(all(target_feature = "avx2", not(target_feature = "avx512vbmi")))] - 64 => transize(swizzle_dyn_split::<64, 32>, self, idxs), - #[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))] - 64 => transize(swizzle_dyn_split::<64, 32>, self, idxs), #[cfg(all(target_feature = "avx512vl", target_feature = "avx512vbmi"))] 64 => { // Unlike vpshufb, vpermb doesn't zero out values in the result based on the index high bit @@ -97,6 +100,10 @@ impl Simd { }; transize(swizzler, self, idxs) } + #[cfg(target_feature = "avx2")] + 64 => transize(swizzle_dyn_split::<64, 32>, self, idxs), + + // scalar fallback _ => { let mut array = [0; N]; for (i, k) in idxs.to_array().into_iter().enumerate() { From 4ca1d7cc4c654218c225c1f8018865028b2a7d32 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 22 Jul 2026 10:13:18 +0100 Subject: [PATCH 5/5] remove stray comma Co-authored-by: Jacob Lifshay --- crates/core_simd/src/swizzle_dyn.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core_simd/src/swizzle_dyn.rs b/crates/core_simd/src/swizzle_dyn.rs index 36d6bfe1adc..4894abbf441 100644 --- a/crates/core_simd/src/swizzle_dyn.rs +++ b/crates/core_simd/src/swizzle_dyn.rs @@ -63,7 +63,7 @@ impl Simd { 16 => transize(loong64_lsx_swizzle, self, idxs), #[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))] 32 => transize(loong64_lasx_swizzle, self, idxs), - #[cfg(all(target_arch = "loongarch64", target_feature = "lsx",))] + #[cfg(all(target_arch = "loongarch64", target_feature = "lsx"))] 32 => transize(swizzle_dyn_split::<32, 16>, self, idxs), #[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))] 64 => transize(swizzle_dyn_split::<64, 32>, self, idxs),