From b24a2f2c99bcbe1d0c391fb25afd89d73d7f661b Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 01:40:12 +0100 Subject: [PATCH 01/23] Initial pass at swizzle_dyn, ported from std::simd --- fearless_simd/src/generated/avx2.rs | 2490 +++++++++- fearless_simd/src/generated/avx512.rs | 448 ++ fearless_simd/src/generated/fallback.rs | 4288 ++++++++++++++++- fearless_simd/src/generated/neon.rs | 416 ++ fearless_simd/src/generated/simd_trait.rs | 50 + fearless_simd/src/generated/simd_types.rs | 120 + fearless_simd/src/generated/sse2.rs | 4288 ++++++++++++++++- fearless_simd/src/generated/sse4_2.rs | 3704 +++++++++++++- fearless_simd/src/generated/wasm.rs | 3432 ++++++++++++- fearless_simd_gen/src/generic.rs | 3 + fearless_simd_gen/src/mk_fallback.rs | 27 + fearless_simd_gen/src/mk_neon.rs | 39 + fearless_simd_gen/src/mk_simd_types.rs | 6 + fearless_simd_gen/src/mk_wasm.rs | 18 + fearless_simd_gen/src/mk_x86.rs | 70 + fearless_simd_gen/src/ops.rs | 20 +- .../tests/harness/lm_generated.rs | 1 + .../lm_generated/swizzle_dyn_precise.rs | 96 + 18 files changed, 18731 insertions(+), 785 deletions(-) create mode 100644 fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 670aedcc..a614d2ec 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -252,6 +252,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x4, indices: u8x16) -> f32x4 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_f32x4(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_f32x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -888,6 +906,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i8x16, indices: u8x16) -> i8x16 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i8x16(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_i8x16(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1406,6 +1442,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x16, indices: u8x16) -> u8x16 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u8x16(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_u8x16(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -2071,6 +2125,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i16x8, indices: u8x16) -> i16x8 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i16x8(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_i16x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2522,6 +2594,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u16x8, indices: u8x16) -> u16x8 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u16x8(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_u16x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -3134,6 +3224,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i32x4, indices: u8x16) -> i32x4 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i32x4(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_i32x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -3569,6 +3677,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u32x4, indices: u8x16) -> u32x4 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u32x4(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_u32x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -4169,6 +4295,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x2, indices: u8x16) -> f64x2 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_f64x2(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_f64x2(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -5624,6 +5768,34 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = token.cvt_to_bytes_f32x8(a); + let idxs = indices; + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let hi_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 32)) + .select(hi_shuf, u8x32::splat(token, 0)); + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let lo_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 16)) + .select(lo_shuf, result_bytes); + token.cvt_from_bytes_f32x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] @@ -6387,6 +6559,34 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = token.cvt_to_bytes_i8x32(a); + let idxs = indices; + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let hi_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 32)) + .select(hi_shuf, u8x32::splat(token, 0)); + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let lo_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 16)) + .select(lo_shuf, result_bytes); + token.cvt_from_bytes_i8x32(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] @@ -7089,6 +7289,34 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = token.cvt_to_bytes_u8x32(a); + let idxs = indices; + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let hi_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 32)) + .select(hi_shuf, u8x32::splat(token, 0)); + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let lo_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 16)) + .select(lo_shuf, result_bytes); + token.cvt_from_bytes_u8x32(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] @@ -7925,6 +8153,34 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = token.cvt_to_bytes_i16x16(a); + let idxs = indices; + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let hi_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 32)) + .select(hi_shuf, u8x32::splat(token, 0)); + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let lo_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 16)) + .select(lo_shuf, result_bytes); + token.cvt_from_bytes_i16x16(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] @@ -8519,6 +8775,34 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = token.cvt_to_bytes_u16x16(a); + let idxs = indices; + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let hi_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 32)) + .select(hi_shuf, u8x32::splat(token, 0)); + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let lo_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 16)) + .select(lo_shuf, result_bytes); + token.cvt_from_bytes_u16x16(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -9284,6 +9568,34 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = token.cvt_to_bytes_i32x8(a); + let idxs = indices; + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let hi_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 32)) + .select(hi_shuf, u8x32::splat(token, 0)); + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let lo_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 16)) + .select(lo_shuf, result_bytes); + token.cvt_from_bytes_i32x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] @@ -9801,6 +10113,34 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = token.cvt_to_bytes_u32x8(a); + let idxs = indices; + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let hi_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 32)) + .select(hi_shuf, u8x32::splat(token, 0)); + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let lo_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 16)) + .select(lo_shuf, result_bytes); + token.cvt_from_bytes_u32x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] @@ -10487,6 +10827,34 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = token.cvt_to_bytes_f64x4(a); + let idxs = indices; + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let hi_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 32)) + .select(hi_shuf, u8x32::splat(token, 0)); + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let lo_shuf = u8x32 { + val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + simd: token, + }; + let result_bytes = idxs + .simd_lt(u8x32::splat(token, 16)) + .select(lo_shuf, result_bytes); + token.cvt_from_bytes_f64x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { crate::kernel!( #[inline(always)] @@ -12170,6 +12538,270 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + let bytes = self.cvt_to_bytes_f32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f32x16(result) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -12910,11 +13542,275 @@ impl Simd for Avx2 { ) } #[inline(always)] - fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) - } + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let bytes = self.cvt_to_bytes_i8x64(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i8x64(result) + } + #[inline(always)] + fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) + } #[inline(always)] fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); @@ -13482,6 +14378,270 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let bytes = self.cvt_to_bytes_u8x64(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u8x64(result) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -14157,6 +15317,270 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + let bytes = self.cvt_to_bytes_i16x32(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i16x32(result) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -14614,6 +16038,270 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + let bytes = self.cvt_to_bytes_u16x32(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u16x32(result) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -15233,6 +16921,270 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + let bytes = self.cvt_to_bytes_i32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i32x16(result) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -15622,6 +17574,270 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + let bytes = self.cvt_to_bytes_u32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u32x16(result) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -16175,6 +18391,270 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let bytes = self.cvt_to_bytes_f64x8(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f64x8(result) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index 76f36efd..41b36cde 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -260,6 +260,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, indices: u8x16) -> f32x4 { + let bytes = token.cvt_to_bytes_f32x4(a).val.0; + let indices = indices.into(); + let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); + let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_f32x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -894,6 +912,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, indices: u8x16) -> i8x16 { + let bytes = token.cvt_to_bytes_i8x16(a).val.0; + let indices = indices.into(); + let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); + let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_i8x16(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1455,6 +1491,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, indices: u8x16) -> u8x16 { + let bytes = token.cvt_to_bytes_u8x16(a).val.0; + let indices = indices.into(); + let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); + let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_u8x16(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -2109,6 +2163,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, indices: u8x16) -> i16x8 { + let bytes = token.cvt_to_bytes_i16x8(a).val.0; + let indices = indices.into(); + let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); + let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_i16x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2589,6 +2661,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, indices: u8x16) -> u16x8 { + let bytes = token.cvt_to_bytes_u16x8(a).val.0; + let indices = indices.into(); + let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); + let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_u16x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -3178,6 +3268,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, indices: u8x16) -> i32x4 { + let bytes = token.cvt_to_bytes_i32x4(a).val.0; + let indices = indices.into(); + let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); + let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_i32x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -3642,6 +3750,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, indices: u8x16) -> u32x4 { + let bytes = token.cvt_to_bytes_u32x4(a).val.0; + let indices = indices.into(); + let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); + let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_u32x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -4214,6 +4340,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, indices: u8x16) -> f64x2 { + let bytes = token.cvt_to_bytes_f64x2(a).val.0; + let indices = indices.into(); + let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); + let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_f64x2(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -5775,6 +5919,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = token.cvt_to_bytes_f32x8(a).val.0; + let indices = indices.into(); + let in_range = + _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); + let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + token.cvt_from_bytes_f32x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] @@ -6574,6 +6737,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = token.cvt_to_bytes_i8x32(a).val.0; + let indices = indices.into(); + let in_range = + _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); + let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + token.cvt_from_bytes_i8x32(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] @@ -7310,6 +7492,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = token.cvt_to_bytes_u8x32(a).val.0; + let indices = indices.into(); + let in_range = + _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); + let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + token.cvt_from_bytes_u8x32(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] @@ -8121,6 +8322,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = token.cvt_to_bytes_i16x16(a).val.0; + let indices = indices.into(); + let in_range = + _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); + let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + token.cvt_from_bytes_i16x16(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] @@ -8728,6 +8948,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = token.cvt_to_bytes_u16x16(a).val.0; + let indices = indices.into(); + let in_range = + _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); + let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + token.cvt_from_bytes_u16x16(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -9448,6 +9687,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = token.cvt_to_bytes_i32x8(a).val.0; + let indices = indices.into(); + let in_range = + _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); + let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + token.cvt_from_bytes_i32x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] @@ -10007,6 +10265,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = token.cvt_to_bytes_u32x8(a).val.0; + let indices = indices.into(); + let in_range = + _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); + let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + token.cvt_from_bytes_u32x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] @@ -10680,6 +10957,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = token.cvt_to_bytes_f64x4(a).val.0; + let indices = indices.into(); + let in_range = + _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); + let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + token.cvt_from_bytes_f64x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { crate::kernel!( #[inline(always)] @@ -12489,6 +12785,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, indices: u8x64) -> f32x16 { + let bytes = token.cvt_to_bytes_f32x16(a).val.0; + let indices = indices.into(); + let in_range = + _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); + let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + token.cvt_from_bytes_f32x16(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { crate::kernel!( #[inline(always)] @@ -13459,6 +13774,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, indices: u8x64) -> i8x64 { + let bytes = token.cvt_to_bytes_i8x64(a).val.0; + let indices = indices.into(); + let in_range = + _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); + let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + token.cvt_from_bytes_i8x64(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { crate::kernel!( #[inline(always)] @@ -14338,6 +14672,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, indices: u8x64) -> u8x64 { + let bytes = token.cvt_to_bytes_u8x64(a).val.0; + let indices = indices.into(); + let in_range = + _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); + let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + token.cvt_from_bytes_u8x64(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { crate::kernel!( #[inline(always)] @@ -15243,6 +15596,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, indices: u8x64) -> i16x32 { + let bytes = token.cvt_to_bytes_i16x32(a).val.0; + let indices = indices.into(); + let in_range = + _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); + let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + token.cvt_from_bytes_i16x32(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { crate::kernel!( #[inline(always)] @@ -15924,6 +16296,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, indices: u8x64) -> u16x32 { + let bytes = token.cvt_to_bytes_u16x32(a).val.0; + let indices = indices.into(); + let in_range = + _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); + let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + token.cvt_from_bytes_u16x32(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { crate::kernel!( #[inline(always)] @@ -16718,6 +17109,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, indices: u8x64) -> i32x16 { + let bytes = token.cvt_to_bytes_i32x16(a).val.0; + let indices = indices.into(); + let in_range = + _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); + let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + token.cvt_from_bytes_i32x16(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { crate::kernel!( #[inline(always)] @@ -17327,6 +17737,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, indices: u8x64) -> u32x16 { + let bytes = token.cvt_to_bytes_u32x16(a).val.0; + let indices = indices.into(); + let in_range = + _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); + let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + token.cvt_from_bytes_u32x16(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { crate::kernel!( #[inline(always)] @@ -18050,6 +18479,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, indices: u8x64) -> f64x8 { + let bytes = token.cvt_to_bytes_f64x8(a).val.0; + let indices = indices.into(); + let in_range = + _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); + let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + token.cvt_from_bytes_f64x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { crate::kernel!( #[inline(always)] diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index e2e5bb43..b2f9f319 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -351,6 +351,78 @@ impl Simd for Fallback { self.cvt_from_bytes_f32x4(result) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + let bytes = self.cvt_to_bytes_f32x4(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f32x4(result) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { [ f32::abs(a[0usize]), @@ -931,6 +1003,78 @@ impl Simd for Fallback { self.cvt_from_bytes_i8x16(result) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + let bytes = self.cvt_to_bytes_i8x16(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i8x16(result) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { [ i8::wrapping_add(a[0usize], b[0usize]), @@ -1728,6 +1872,78 @@ impl Simd for Fallback { self.cvt_from_bytes_u8x16(result) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + let bytes = self.cvt_to_bytes_u8x16(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u8x16(result) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { [ u8::wrapping_add(a[0usize], b[0usize]), @@ -2835,6 +3051,78 @@ impl Simd for Fallback { self.cvt_from_bytes_i16x8(result) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + let bytes = self.cvt_to_bytes_i16x8(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i16x8(result) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { [ i16::wrapping_add(a[0usize], b[0usize]), @@ -3401,42 +3689,114 @@ impl Simd for Fallback { self.cvt_from_bytes_u16x8(result) } #[inline(always)] - fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - [ - u16::wrapping_add(a[0usize], b[0usize]), - u16::wrapping_add(a[1usize], b[1usize]), - u16::wrapping_add(a[2usize], b[2usize]), - u16::wrapping_add(a[3usize], b[3usize]), - u16::wrapping_add(a[4usize], b[4usize]), - u16::wrapping_add(a[5usize], b[5usize]), - u16::wrapping_add(a[6usize], b[6usize]), - u16::wrapping_add(a[7usize], b[7usize]), - ] - .simd_into(self) - } - #[inline(always)] - fn sub_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - [ - u16::wrapping_sub(a[0usize], b[0usize]), - u16::wrapping_sub(a[1usize], b[1usize]), - u16::wrapping_sub(a[2usize], b[2usize]), - u16::wrapping_sub(a[3usize], b[3usize]), - u16::wrapping_sub(a[4usize], b[4usize]), - u16::wrapping_sub(a[5usize], b[5usize]), - u16::wrapping_sub(a[6usize], b[6usize]), - u16::wrapping_sub(a[7usize], b[7usize]), - ] - .simd_into(self) - } - #[inline(always)] - fn mul_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - [ - u16::wrapping_mul(a[0usize], b[0usize]), - u16::wrapping_mul(a[1usize], b[1usize]), - u16::wrapping_mul(a[2usize], b[2usize]), - u16::wrapping_mul(a[3usize], b[3usize]), - u16::wrapping_mul(a[4usize], b[4usize]), - u16::wrapping_mul(a[5usize], b[5usize]), + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + let bytes = self.cvt_to_bytes_u16x8(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u16x8(result) + } + #[inline(always)] + fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + [ + u16::wrapping_add(a[0usize], b[0usize]), + u16::wrapping_add(a[1usize], b[1usize]), + u16::wrapping_add(a[2usize], b[2usize]), + u16::wrapping_add(a[3usize], b[3usize]), + u16::wrapping_add(a[4usize], b[4usize]), + u16::wrapping_add(a[5usize], b[5usize]), + u16::wrapping_add(a[6usize], b[6usize]), + u16::wrapping_add(a[7usize], b[7usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn sub_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + [ + u16::wrapping_sub(a[0usize], b[0usize]), + u16::wrapping_sub(a[1usize], b[1usize]), + u16::wrapping_sub(a[2usize], b[2usize]), + u16::wrapping_sub(a[3usize], b[3usize]), + u16::wrapping_sub(a[4usize], b[4usize]), + u16::wrapping_sub(a[5usize], b[5usize]), + u16::wrapping_sub(a[6usize], b[6usize]), + u16::wrapping_sub(a[7usize], b[7usize]), + ] + .simd_into(self) + } + #[inline(always)] + fn mul_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + [ + u16::wrapping_mul(a[0usize], b[0usize]), + u16::wrapping_mul(a[1usize], b[1usize]), + u16::wrapping_mul(a[2usize], b[2usize]), + u16::wrapping_mul(a[3usize], b[3usize]), + u16::wrapping_mul(a[4usize], b[4usize]), + u16::wrapping_mul(a[5usize], b[5usize]), u16::wrapping_mul(a[6usize], b[6usize]), u16::wrapping_mul(a[7usize], b[7usize]), ] @@ -4163,6 +4523,78 @@ impl Simd for Fallback { self.cvt_from_bytes_i32x4(result) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + let bytes = self.cvt_to_bytes_i32x4(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i32x4(result) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { [ i32::wrapping_add(a[0usize], b[0usize]), @@ -4615,6 +5047,78 @@ impl Simd for Fallback { self.cvt_from_bytes_u32x4(result) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + let bytes = self.cvt_to_bytes_u32x4(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u32x4(result) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { [ u32::wrapping_add(a[0usize], b[0usize]), @@ -5199,6 +5703,78 @@ impl Simd for Fallback { self.cvt_from_bytes_f64x2(result) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + let bytes = self.cvt_to_bytes_f64x2(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f64x2(result) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { [f64::abs(a[0usize]), f64::abs(a[1usize])].simd_into(self) } @@ -6420,6 +6996,142 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = self.cvt_to_bytes_f32x8(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f32x8(result) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -6950,6 +7662,142 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = self.cvt_to_bytes_i8x32(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i8x32(result) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -7387,6 +8235,142 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = self.cvt_to_bytes_u8x32(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u8x32(result) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -7878,10 +8862,146 @@ impl Simd for Fallback { ) } #[inline(always)] - fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.add_i16x8(a0, b0), self.add_i16x8(a1, b1)) + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = self.cvt_to_bytes_i16x16(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i16x16(result) + } + #[inline(always)] + fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.add_i16x8(a0, b0), self.add_i16x8(a1, b1)) } #[inline(always)] fn sub_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { @@ -8255,6 +9375,142 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = self.cvt_to_bytes_u16x16(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u16x16(result) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -8732,6 +9988,142 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = self.cvt_to_bytes_i32x8(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i32x8(result) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -9078,6 +10470,142 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = self.cvt_to_bytes_u32x8(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u32x8(result) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -9514,6 +11042,142 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = self.cvt_to_bytes_f64x4(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f64x4(result) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -10701,94 +12365,358 @@ impl Simd for Fallback { ) } #[inline(always)] - fn abs_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) - } - #[inline(always)] - fn neg_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) - } - #[inline(always)] - fn sqrt_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) - } - #[inline(always)] - fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8( - self.approximate_recip_f32x8(a0), - self.approximate_recip_f32x8(a1), - ) - } - #[inline(always)] - fn add_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.add_f32x8(a0, b0), self.add_f32x8(a1, b1)) - } - #[inline(always)] - fn sub_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.sub_f32x8(a0, b0), self.sub_f32x8(a1, b1)) - } - #[inline(always)] - fn mul_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.mul_f32x8(a0, b0), self.mul_f32x8(a1, b1)) - } - #[inline(always)] - fn div_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.div_f32x8(a0, b0), self.div_f32x8(a1, b1)) - } - #[inline(always)] - fn copysign_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.copysign_f32x8(a0, b0), self.copysign_f32x8(a1, b1)) - } - #[inline(always)] - fn simd_eq_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_eq_f32x8(a0, b0), self.simd_eq_f32x8(a1, b1)) - } - #[inline(always)] - fn simd_lt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_lt_f32x8(a0, b0), self.simd_lt_f32x8(a1, b1)) - } - #[inline(always)] - fn simd_le_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_le_f32x8(a0, b0), self.simd_le_f32x8(a1, b1)) - } - #[inline(always)] - fn simd_ge_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_ge_f32x8(a0, b0), self.simd_ge_f32x8(a1, b1)) - } - #[inline(always)] - fn simd_gt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_gt_f32x8(a0, b0), self.simd_gt_f32x8(a1, b1)) - } - #[inline(always)] - fn zip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, _) = self.split_f32x16(a); - let (b0, _) = self.split_f32x16(b); - self.combine_f32x8(self.zip_low_f32x8(a0, b0), self.zip_high_f32x8(a0, b0)) - } + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + let bytes = self.cvt_to_bytes_f32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f32x16(result) + } + #[inline(always)] + fn abs_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) + } + #[inline(always)] + fn neg_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) + } + #[inline(always)] + fn sqrt_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) + } + #[inline(always)] + fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8( + self.approximate_recip_f32x8(a0), + self.approximate_recip_f32x8(a1), + ) + } + #[inline(always)] + fn add_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.add_f32x8(a0, b0), self.add_f32x8(a1, b1)) + } + #[inline(always)] + fn sub_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.sub_f32x8(a0, b0), self.sub_f32x8(a1, b1)) + } + #[inline(always)] + fn mul_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.mul_f32x8(a0, b0), self.mul_f32x8(a1, b1)) + } + #[inline(always)] + fn div_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.div_f32x8(a0, b0), self.div_f32x8(a1, b1)) + } + #[inline(always)] + fn copysign_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.copysign_f32x8(a0, b0), self.copysign_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_eq_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_eq_f32x8(a0, b0), self.simd_eq_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_lt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_lt_f32x8(a0, b0), self.simd_lt_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_le_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_le_f32x8(a0, b0), self.simd_le_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_ge_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_ge_f32x8(a0, b0), self.simd_ge_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_gt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_gt_f32x8(a0, b0), self.simd_gt_f32x8(a1, b1)) + } + #[inline(always)] + fn zip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, _) = self.split_f32x16(a); + let (b0, _) = self.split_f32x16(b); + self.combine_f32x8(self.zip_low_f32x8(a0, b0), self.zip_high_f32x8(a0, b0)) + } #[inline(always)] fn zip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { let (_, a1) = self.split_f32x16(a); @@ -11382,26 +13310,290 @@ impl Simd for Fallback { ) } #[inline(always)] - fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) - } - #[inline(always)] - fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) - } - #[inline(always)] - fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) - } - #[inline(always)] - fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let bytes = self.cvt_to_bytes_i8x64(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i8x64(result) + } + #[inline(always)] + fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) + } + #[inline(always)] + fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) + } + #[inline(always)] + fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) + } + #[inline(always)] + fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); self.combine_i8x32(self.and_i8x32(a0, b0), self.and_i8x32(a1, b1)) } @@ -11940,94 +14132,358 @@ impl Simd for Fallback { ) } #[inline(always)] - fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.add_u8x32(a0, b0), self.add_u8x32(a1, b1)) - } - #[inline(always)] - fn sub_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.sub_u8x32(a0, b0), self.sub_u8x32(a1, b1)) - } - #[inline(always)] - fn mul_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.mul_u8x32(a0, b0), self.mul_u8x32(a1, b1)) - } - #[inline(always)] - fn and_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.and_u8x32(a0, b0), self.and_u8x32(a1, b1)) - } - #[inline(always)] - fn or_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.or_u8x32(a0, b0), self.or_u8x32(a1, b1)) - } - #[inline(always)] - fn xor_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.xor_u8x32(a0, b0), self.xor_u8x32(a1, b1)) - } - #[inline(always)] - fn not_u8x64(self, a: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.not_u8x32(a0), self.not_u8x32(a1)) - } - #[inline(always)] - fn shl_u8x64(self, a: u8x64, shift: u32) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.shl_u8x32(a0, shift), self.shl_u8x32(a1, shift)) - } - #[inline(always)] - fn shlv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.shlv_u8x32(a0, b0), self.shlv_u8x32(a1, b1)) - } - #[inline(always)] - fn shr_u8x64(self, a: u8x64, shift: u32) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.shr_u8x32(a0, shift), self.shr_u8x32(a1, shift)) - } - #[inline(always)] - fn shrv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.shrv_u8x32(a0, b0), self.shrv_u8x32(a1, b1)) - } - #[inline(always)] - fn simd_eq_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_eq_u8x32(a0, b0), self.simd_eq_u8x32(a1, b1)) - } - #[inline(always)] - fn simd_lt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_lt_u8x32(a0, b0), self.simd_lt_u8x32(a1, b1)) - } - #[inline(always)] - fn simd_le_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_le_u8x32(a0, b0), self.simd_le_u8x32(a1, b1)) - } - #[inline(always)] - fn simd_ge_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_ge_u8x32(a0, b0), self.simd_ge_u8x32(a1, b1)) - } - #[inline(always)] - fn simd_gt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let bytes = self.cvt_to_bytes_u8x64(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u8x64(result) + } + #[inline(always)] + fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.add_u8x32(a0, b0), self.add_u8x32(a1, b1)) + } + #[inline(always)] + fn sub_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.sub_u8x32(a0, b0), self.sub_u8x32(a1, b1)) + } + #[inline(always)] + fn mul_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.mul_u8x32(a0, b0), self.mul_u8x32(a1, b1)) + } + #[inline(always)] + fn and_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.and_u8x32(a0, b0), self.and_u8x32(a1, b1)) + } + #[inline(always)] + fn or_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.or_u8x32(a0, b0), self.or_u8x32(a1, b1)) + } + #[inline(always)] + fn xor_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.xor_u8x32(a0, b0), self.xor_u8x32(a1, b1)) + } + #[inline(always)] + fn not_u8x64(self, a: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.not_u8x32(a0), self.not_u8x32(a1)) + } + #[inline(always)] + fn shl_u8x64(self, a: u8x64, shift: u32) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.shl_u8x32(a0, shift), self.shl_u8x32(a1, shift)) + } + #[inline(always)] + fn shlv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.shlv_u8x32(a0, b0), self.shlv_u8x32(a1, b1)) + } + #[inline(always)] + fn shr_u8x64(self, a: u8x64, shift: u32) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.shr_u8x32(a0, shift), self.shr_u8x32(a1, shift)) + } + #[inline(always)] + fn shrv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.shrv_u8x32(a0, b0), self.shrv_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_eq_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_eq_u8x32(a0, b0), self.simd_eq_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_lt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_lt_u8x32(a0, b0), self.simd_lt_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_le_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_le_u8x32(a0, b0), self.simd_le_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_ge_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_ge_u8x32(a0, b0), self.simd_ge_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_gt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); self.combine_mask8x32(self.simd_gt_u8x32(a0, b0), self.simd_gt_u8x32(a1, b1)) @@ -12561,26 +15017,290 @@ impl Simd for Fallback { ) } #[inline(always)] - fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.add_i16x16(a0, b0), self.add_i16x16(a1, b1)) - } - #[inline(always)] - fn sub_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.sub_i16x16(a0, b0), self.sub_i16x16(a1, b1)) - } - #[inline(always)] - fn mul_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.mul_i16x16(a0, b0), self.mul_i16x16(a1, b1)) - } - #[inline(always)] - fn and_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + let bytes = self.cvt_to_bytes_i16x32(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i16x32(result) + } + #[inline(always)] + fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.add_i16x16(a0, b0), self.add_i16x16(a1, b1)) + } + #[inline(always)] + fn sub_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.sub_i16x16(a0, b0), self.sub_i16x16(a1, b1)) + } + #[inline(always)] + fn mul_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.mul_i16x16(a0, b0), self.mul_i16x16(a1, b1)) + } + #[inline(always)] + fn and_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); self.combine_i16x16(self.and_i16x16(a0, b0), self.and_i16x16(a1, b1)) } @@ -13004,6 +15724,270 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + let bytes = self.cvt_to_bytes_u16x32(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u16x32(result) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -13546,6 +16530,270 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + let bytes = self.cvt_to_bytes_i32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i32x16(result) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -13921,6 +17169,270 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + let bytes = self.cvt_to_bytes_u32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u32x16(result) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -14389,6 +17901,270 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let bytes = self.cvt_to_bytes_f64x8(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f64x8(result) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index c9e164a2..dde2299e 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -239,6 +239,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f32x4, indices: u8x16) -> f32x4 { + let result = vqtbl1q_u8(token.cvt_to_bytes_f32x4(a).val.0, indices.into()); + token.cvt_from_bytes_f32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -830,6 +844,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i8x16, indices: u8x16) -> i8x16 { + let result = vqtbl1q_u8(token.cvt_to_bytes_i8x16(a).val.0, indices.into()); + token.cvt_from_bytes_i8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1310,6 +1338,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u8x16, indices: u8x16) -> u8x16 { + let result = vqtbl1q_u8(token.cvt_to_bytes_u8x16(a).val.0, indices.into()); + token.cvt_from_bytes_u8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1937,6 +1979,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i16x8, indices: u8x16) -> i16x8 { + let result = vqtbl1q_u8(token.cvt_to_bytes_i16x8(a).val.0, indices.into()); + token.cvt_from_bytes_i16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2385,6 +2441,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u16x8, indices: u8x16) -> u16x8 { + let result = vqtbl1q_u8(token.cvt_to_bytes_u16x8(a).val.0, indices.into()); + token.cvt_from_bytes_u16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2986,6 +3056,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i32x4, indices: u8x16) -> i32x4 { + let result = vqtbl1q_u8(token.cvt_to_bytes_i32x4(a).val.0, indices.into()); + token.cvt_from_bytes_i32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -3428,6 +3512,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u32x4, indices: u8x16) -> u32x4 { + let result = vqtbl1q_u8(token.cvt_to_bytes_u32x4(a).val.0, indices.into()); + token.cvt_from_bytes_u32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -4020,6 +4118,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f64x2, indices: u8x16) -> f64x2 { + let result = vqtbl1q_u8(token.cvt_to_bytes_f64x2(a).val.0, indices.into()); + token.cvt_from_bytes_f64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -5540,6 +5652,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f32x8, indices: u8x32) -> f32x8 { + let table = token.cvt_to_bytes_f32x8(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + token.cvt_from_bytes_f32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -6107,6 +6236,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i8x32, indices: u8x32) -> i8x32 { + let table = token.cvt_to_bytes_i8x32(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + token.cvt_from_bytes_i8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -6581,6 +6727,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u8x32, indices: u8x32) -> u8x32 { + let table = token.cvt_to_bytes_u8x32(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + token.cvt_from_bytes_u8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -7116,6 +7279,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i16x16, indices: u8x32) -> i16x16 { + let table = token.cvt_to_bytes_i16x16(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + token.cvt_from_bytes_i16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -7530,6 +7710,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u16x16, indices: u8x32) -> u16x16 { + let table = token.cvt_to_bytes_u16x16(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + token.cvt_from_bytes_u16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -8042,6 +8239,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i32x8, indices: u8x32) -> i32x8 { + let table = token.cvt_to_bytes_i32x8(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + token.cvt_from_bytes_i32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -8425,6 +8639,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u32x8, indices: u8x32) -> u32x8 { + let table = token.cvt_to_bytes_u32x8(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + token.cvt_from_bytes_u32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -8905,6 +9136,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f64x4, indices: u8x32) -> f64x4 { + let table = token.cvt_to_bytes_f64x4(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + token.cvt_from_bytes_f64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -10228,6 +10476,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f32x16, indices: u8x64) -> f32x16 { + let table = token.cvt_to_bytes_f32x16(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + token.cvt_from_bytes_f32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -10940,6 +11209,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i8x64, indices: u8x64) -> i8x64 { + let table = token.cvt_to_bytes_i8x64(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + token.cvt_from_bytes_i8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -11551,6 +11841,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u8x64, indices: u8x64) -> u8x64 { + let table = token.cvt_to_bytes_u8x64(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + token.cvt_from_bytes_u8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -12153,6 +12464,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i16x32, indices: u8x64) -> i16x32 { + let table = token.cvt_to_bytes_i16x32(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + token.cvt_from_bytes_i16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -12649,6 +12981,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u16x32, indices: u8x64) -> u16x32 { + let table = token.cvt_to_bytes_u16x32(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + token.cvt_from_bytes_u16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -13209,6 +13562,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i32x16, indices: u8x64) -> i32x16 { + let table = token.cvt_to_bytes_i32x16(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + token.cvt_from_bytes_i32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -13637,6 +14011,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u32x16, indices: u8x64) -> u32x16 { + let table = token.cvt_to_bytes_u32x16(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + token.cvt_from_bytes_u32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -14141,6 +14536,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f64x8, indices: u8x64) -> f64x8 { + let table = token.cvt_to_bytes_f64x8(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + token.cvt_from_bytes_f64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index 4d6490e6..2efdf921 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -182,6 +182,8 @@ pub trait Simd: ) -> f32x4; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4; #[doc = "Compute the absolute value of each element."] fn abs_f32x4(self, a: f32x4) -> f32x4; #[doc = "Negate each element of the vector."] @@ -308,6 +310,8 @@ pub trait Simd: ) -> i8x16; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -410,6 +414,8 @@ pub trait Simd: ) -> u8x16; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -549,6 +555,8 @@ pub trait Simd: ) -> i16x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -651,6 +659,8 @@ pub trait Simd: ) -> u16x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -790,6 +800,8 @@ pub trait Simd: ) -> i32x4; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -894,6 +906,8 @@ pub trait Simd: ) -> u32x4; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1033,6 +1047,8 @@ pub trait Simd: ) -> f64x2; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2; #[doc = "Compute the absolute value of each element."] fn abs_f64x2(self, a: f64x2) -> f64x2; #[doc = "Negate each element of the vector."] @@ -1386,6 +1402,8 @@ pub trait Simd: ) -> f32x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8; #[doc = "Compute the absolute value of each element."] fn abs_f32x8(self, a: f32x8) -> f32x8; #[doc = "Negate each element of the vector."] @@ -1514,6 +1532,8 @@ pub trait Simd: ) -> i8x32; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1618,6 +1638,8 @@ pub trait Simd: ) -> u8x32; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1765,6 +1787,8 @@ pub trait Simd: a: i16x16, indices: u8x32, ) -> i16x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1873,6 +1897,8 @@ pub trait Simd: a: u16x16, indices: u8x32, ) -> u16x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2018,6 +2044,8 @@ pub trait Simd: ) -> i32x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2124,6 +2152,8 @@ pub trait Simd: ) -> u32x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2267,6 +2297,8 @@ pub trait Simd: ) -> f64x4; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4; #[doc = "Compute the absolute value of each element."] fn abs_f64x4(self, a: f64x4) -> f64x4; #[doc = "Negate each element of the vector."] @@ -2632,6 +2664,8 @@ pub trait Simd: a: f32x16, indices: u8x64, ) -> f32x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16; #[doc = "Compute the absolute value of each element."] fn abs_f32x16(self, a: f32x16) -> f32x16; #[doc = "Negate each element of the vector."] @@ -2762,6 +2796,8 @@ pub trait Simd: ) -> i8x64; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2864,6 +2900,8 @@ pub trait Simd: ) -> u8x64; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -3009,6 +3047,8 @@ pub trait Simd: a: i16x32, indices: u8x64, ) -> i16x32; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -3115,6 +3155,8 @@ pub trait Simd: a: u16x32, indices: u8x64, ) -> u16x32; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -3264,6 +3306,8 @@ pub trait Simd: a: i32x16, indices: u8x64, ) -> i32x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -3372,6 +3416,8 @@ pub trait Simd: a: u32x16, indices: u8x64, ) -> u32x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -3515,6 +3561,8 @@ pub trait Simd: ) -> f64x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8; #[doc = "Compute the absolute value of each element."] fn abs_f64x8(self, a: f64x8) -> f64x8; #[doc = "Negate each element of the vector."] @@ -3947,6 +3995,8 @@ pub trait SimdBase: fn shift_elements_right(self, padding: Self::Element) -> Self; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self; } #[doc = r" Functionality implemented by floating-point SIMD vectors."] pub trait SimdFloat: diff --git a/fearless_simd/src/generated/simd_types.rs b/fearless_simd/src/generated/simd_types.rs index 50d80f73..740c5000 100644 --- a/fearless_simd/src/generated/simd_types.rs +++ b/fearless_simd/src/generated/simd_types.rs @@ -152,6 +152,11 @@ impl SimdBase for f32x4 { self.simd .swizzle_dyn_within_blocks_f32x4(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_f32x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f32x4 { #[inline(always)] @@ -444,6 +449,11 @@ impl SimdBase for i8x16 { self.simd .swizzle_dyn_within_blocks_i8x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i8x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i8x16 { #[inline(always)] @@ -668,6 +678,11 @@ impl SimdBase for u8x16 { self.simd .swizzle_dyn_within_blocks_u8x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u8x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u8x16 { #[inline(always)] @@ -975,6 +990,11 @@ impl SimdBase for i16x8 { self.simd .swizzle_dyn_within_blocks_i16x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i16x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i16x8 { #[inline(always)] @@ -1191,6 +1211,11 @@ impl SimdBase for u16x8 { self.simd .swizzle_dyn_within_blocks_u16x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u16x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u16x8 { #[inline(always)] @@ -1489,6 +1514,11 @@ impl SimdBase for i32x4 { self.simd .swizzle_dyn_within_blocks_i32x4(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i32x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i32x4 { #[inline(always)] @@ -1708,6 +1738,11 @@ impl SimdBase for u32x4 { self.simd .swizzle_dyn_within_blocks_u32x4(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u32x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u32x4 { #[inline(always)] @@ -2018,6 +2053,11 @@ impl SimdBase for f64x2 { self.simd .swizzle_dyn_within_blocks_f64x2(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_f64x2(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f64x2 { #[inline(always)] @@ -2793,6 +2833,11 @@ impl SimdBase for f32x8 { self.simd .swizzle_dyn_within_blocks_f32x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_f32x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f32x8 { #[inline(always)] @@ -3108,6 +3153,11 @@ impl SimdBase for i8x32 { self.simd .swizzle_dyn_within_blocks_i8x32(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i8x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i8x32 { #[inline(always)] @@ -3355,6 +3405,11 @@ impl SimdBase for u8x32 { self.simd .swizzle_dyn_within_blocks_u8x32(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u8x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u8x32 { #[inline(always)] @@ -3683,6 +3738,11 @@ impl SimdBase for i16x16 { self.simd .swizzle_dyn_within_blocks_i16x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i16x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i16x16 { #[inline(always)] @@ -3921,6 +3981,11 @@ impl SimdBase for u16x16 { self.simd .swizzle_dyn_within_blocks_u16x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u16x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u16x16 { #[inline(always)] @@ -4236,6 +4301,11 @@ impl SimdBase for i32x8 { self.simd .swizzle_dyn_within_blocks_i32x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i32x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i32x8 { #[inline(always)] @@ -4471,6 +4541,11 @@ impl SimdBase for u32x8 { self.simd .swizzle_dyn_within_blocks_u32x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u32x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u32x8 { #[inline(always)] @@ -4788,6 +4863,11 @@ impl SimdBase for f64x4 { self.simd .swizzle_dyn_within_blocks_f64x4(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_f64x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f64x4 { #[inline(always)] @@ -5599,6 +5679,11 @@ impl SimdBase for f32x16 { self.simd .swizzle_dyn_within_blocks_f32x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_f32x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f32x16 { #[inline(always)] @@ -5941,6 +6026,11 @@ impl SimdBase for i8x64 { self.simd .swizzle_dyn_within_blocks_i8x64(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i8x64(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i8x64 { #[inline(always)] @@ -6214,6 +6304,11 @@ impl SimdBase for u8x64 { self.simd .swizzle_dyn_within_blocks_u8x64(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u8x64(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u8x64 { #[inline(always)] @@ -6552,6 +6647,11 @@ impl SimdBase for i16x32 { self.simd .swizzle_dyn_within_blocks_i16x32(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i16x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i16x32 { #[inline(always)] @@ -6800,6 +6900,11 @@ impl SimdBase for u16x32 { self.simd .swizzle_dyn_within_blocks_u16x32(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u16x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u16x32 { #[inline(always)] @@ -7123,6 +7228,11 @@ impl SimdBase for i32x16 { self.simd .swizzle_dyn_within_blocks_i32x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i32x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i32x16 { #[inline(always)] @@ -7367,6 +7477,11 @@ impl SimdBase for u32x16 { self.simd .swizzle_dyn_within_blocks_u32x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u32x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u32x16 { #[inline(always)] @@ -7688,6 +7803,11 @@ impl SimdBase for f64x8 { self.simd .swizzle_dyn_within_blocks_f64x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_f64x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f64x8 { #[inline(always)] diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index 92b92b42..ed3cb966 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -373,6 +373,78 @@ impl Simd for Sse2 { self.cvt_from_bytes_f32x4(result) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + let bytes = self.cvt_to_bytes_f32x4(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f32x4(result) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -1027,6 +1099,78 @@ impl Simd for Sse2 { self.cvt_from_bytes_i8x16(result) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + let bytes = self.cvt_to_bytes_i8x16(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i8x16(result) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1616,6 +1760,78 @@ impl Simd for Sse2 { self.cvt_from_bytes_u8x16(result) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + let bytes = self.cvt_to_bytes_u8x16(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u8x16(result) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -2363,6 +2579,78 @@ impl Simd for Sse2 { self.cvt_from_bytes_i16x8(result) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + let bytes = self.cvt_to_bytes_i16x8(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i16x8(result) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2869,42 +3157,114 @@ impl Simd for Sse2 { self.cvt_from_bytes_u16x8(result) } #[inline(always)] - fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, a: u16x8, b: u16x8) -> u16x8 { - _mm_add_epi16(a.into(), b.into()).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn sub_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, a: u16x8, b: u16x8) -> u16x8 { - _mm_sub_epi16(a.into(), b.into()).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn mul_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, a: u16x8, b: u16x8) -> u16x8 { - _mm_mullo_epi16(a.into(), b.into()).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn and_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, a: u16x8, b: u16x8) -> u16x8 { - _mm_and_si128(a.into(), b.into()).simd_into(token) - } + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + let bytes = self.cvt_to_bytes_u16x8(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u16x8(result) + } + #[inline(always)] + fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, a: u16x8, b: u16x8) -> u16x8 { + _mm_add_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, a: u16x8, b: u16x8) -> u16x8 { + _mm_sub_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, a: u16x8, b: u16x8) -> u16x8 { + _mm_mullo_epi16(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn and_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, a: u16x8, b: u16x8) -> u16x8 { + _mm_and_si128(a.into(), b.into()).simd_into(token) + } ); kernel(self, a, b) } @@ -3575,6 +3935,78 @@ impl Simd for Sse2 { self.cvt_from_bytes_i32x4(result) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + let bytes = self.cvt_to_bytes_i32x4(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i32x4(result) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -4089,6 +4521,78 @@ impl Simd for Sse2 { self.cvt_from_bytes_u32x4(result) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + let bytes = self.cvt_to_bytes_u32x4(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u32x4(result) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -4786,6 +5290,78 @@ impl Simd for Sse2 { self.cvt_from_bytes_f64x2(result) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + let bytes = self.cvt_to_bytes_f64x2(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f64x2(result) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -6322,6 +6898,142 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = self.cvt_to_bytes_f32x8(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f32x8(result) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -6866,6 +7578,142 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = self.cvt_to_bytes_i8x32(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i8x32(result) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -7317,6 +8165,142 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = self.cvt_to_bytes_u8x32(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u8x32(result) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -7827,10 +8811,146 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { - let (a0, a1) = self.split_i16x16(a); - let (b0, b1) = self.split_i16x16(b); - self.combine_i16x8(self.add_i16x8(a0, b0), self.add_i16x8(a1, b1)) + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = self.cvt_to_bytes_i16x16(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i16x16(result) + } + #[inline(always)] + fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (b0, b1) = self.split_i16x16(b); + self.combine_i16x8(self.add_i16x8(a0, b0), self.add_i16x8(a1, b1)) } #[inline(always)] fn sub_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { @@ -8218,6 +9338,142 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = self.cvt_to_bytes_u16x16(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u16x16(result) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -8713,6 +9969,142 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = self.cvt_to_bytes_i32x8(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i32x8(result) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -9073,6 +10465,142 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = self.cvt_to_bytes_u32x8(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u32x8(result) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -9528,6 +11056,142 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = self.cvt_to_bytes_f64x4(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f64x4(result) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -10762,94 +12426,358 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn abs_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) - } - #[inline(always)] - fn neg_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) - } - #[inline(always)] - fn sqrt_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) - } - #[inline(always)] - fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8( - self.approximate_recip_f32x8(a0), - self.approximate_recip_f32x8(a1), - ) - } - #[inline(always)] - fn add_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.add_f32x8(a0, b0), self.add_f32x8(a1, b1)) - } - #[inline(always)] - fn sub_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.sub_f32x8(a0, b0), self.sub_f32x8(a1, b1)) - } - #[inline(always)] - fn mul_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.mul_f32x8(a0, b0), self.mul_f32x8(a1, b1)) - } - #[inline(always)] - fn div_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.div_f32x8(a0, b0), self.div_f32x8(a1, b1)) - } - #[inline(always)] - fn copysign_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.copysign_f32x8(a0, b0), self.copysign_f32x8(a1, b1)) - } - #[inline(always)] - fn simd_eq_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_eq_f32x8(a0, b0), self.simd_eq_f32x8(a1, b1)) - } - #[inline(always)] - fn simd_lt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_lt_f32x8(a0, b0), self.simd_lt_f32x8(a1, b1)) - } - #[inline(always)] - fn simd_le_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_le_f32x8(a0, b0), self.simd_le_f32x8(a1, b1)) - } - #[inline(always)] - fn simd_ge_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_ge_f32x8(a0, b0), self.simd_ge_f32x8(a1, b1)) - } - #[inline(always)] - fn simd_gt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_gt_f32x8(a0, b0), self.simd_gt_f32x8(a1, b1)) - } - #[inline(always)] - fn zip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, _) = self.split_f32x16(a); - let (b0, _) = self.split_f32x16(b); - self.combine_f32x8(self.zip_low_f32x8(a0, b0), self.zip_high_f32x8(a0, b0)) - } + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + let bytes = self.cvt_to_bytes_f32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f32x16(result) + } + #[inline(always)] + fn abs_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) + } + #[inline(always)] + fn neg_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) + } + #[inline(always)] + fn sqrt_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) + } + #[inline(always)] + fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8( + self.approximate_recip_f32x8(a0), + self.approximate_recip_f32x8(a1), + ) + } + #[inline(always)] + fn add_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.add_f32x8(a0, b0), self.add_f32x8(a1, b1)) + } + #[inline(always)] + fn sub_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.sub_f32x8(a0, b0), self.sub_f32x8(a1, b1)) + } + #[inline(always)] + fn mul_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.mul_f32x8(a0, b0), self.mul_f32x8(a1, b1)) + } + #[inline(always)] + fn div_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.div_f32x8(a0, b0), self.div_f32x8(a1, b1)) + } + #[inline(always)] + fn copysign_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.copysign_f32x8(a0, b0), self.copysign_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_eq_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_eq_f32x8(a0, b0), self.simd_eq_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_lt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_lt_f32x8(a0, b0), self.simd_lt_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_le_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_le_f32x8(a0, b0), self.simd_le_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_ge_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_ge_f32x8(a0, b0), self.simd_ge_f32x8(a1, b1)) + } + #[inline(always)] + fn simd_gt_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_gt_f32x8(a0, b0), self.simd_gt_f32x8(a1, b1)) + } + #[inline(always)] + fn zip_low_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, _) = self.split_f32x16(a); + let (b0, _) = self.split_f32x16(b); + self.combine_f32x8(self.zip_low_f32x8(a0, b0), self.zip_high_f32x8(a0, b0)) + } #[inline(always)] fn zip_high_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { let (_, a1) = self.split_f32x16(a); @@ -11502,26 +13430,290 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) - } - #[inline(always)] - fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) - } - #[inline(always)] - fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) - } - #[inline(always)] - fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let bytes = self.cvt_to_bytes_i8x64(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i8x64(result) + } + #[inline(always)] + fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) + } + #[inline(always)] + fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) + } + #[inline(always)] + fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) + } + #[inline(always)] + fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); self.combine_i8x32(self.and_i8x32(a0, b0), self.and_i8x32(a1, b1)) } @@ -12074,94 +14266,358 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.add_u8x32(a0, b0), self.add_u8x32(a1, b1)) - } - #[inline(always)] - fn sub_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.sub_u8x32(a0, b0), self.sub_u8x32(a1, b1)) - } - #[inline(always)] - fn mul_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.mul_u8x32(a0, b0), self.mul_u8x32(a1, b1)) - } - #[inline(always)] - fn and_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.and_u8x32(a0, b0), self.and_u8x32(a1, b1)) - } - #[inline(always)] - fn or_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.or_u8x32(a0, b0), self.or_u8x32(a1, b1)) - } - #[inline(always)] - fn xor_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.xor_u8x32(a0, b0), self.xor_u8x32(a1, b1)) - } - #[inline(always)] - fn not_u8x64(self, a: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.not_u8x32(a0), self.not_u8x32(a1)) - } - #[inline(always)] - fn shl_u8x64(self, a: u8x64, shift: u32) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.shl_u8x32(a0, shift), self.shl_u8x32(a1, shift)) - } - #[inline(always)] - fn shlv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.shlv_u8x32(a0, b0), self.shlv_u8x32(a1, b1)) - } - #[inline(always)] - fn shr_u8x64(self, a: u8x64, shift: u32) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - self.combine_u8x32(self.shr_u8x32(a0, shift), self.shr_u8x32(a1, shift)) - } - #[inline(always)] - fn shrv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_u8x32(self.shrv_u8x32(a0, b0), self.shrv_u8x32(a1, b1)) - } - #[inline(always)] - fn simd_eq_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_eq_u8x32(a0, b0), self.simd_eq_u8x32(a1, b1)) - } - #[inline(always)] - fn simd_lt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_lt_u8x32(a0, b0), self.simd_lt_u8x32(a1, b1)) - } - #[inline(always)] - fn simd_le_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_le_u8x32(a0, b0), self.simd_le_u8x32(a1, b1)) - } - #[inline(always)] - fn simd_ge_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { - let (a0, a1) = self.split_u8x64(a); - let (b0, b1) = self.split_u8x64(b); - self.combine_mask8x32(self.simd_ge_u8x32(a0, b0), self.simd_ge_u8x32(a1, b1)) - } - #[inline(always)] - fn simd_gt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let bytes = self.cvt_to_bytes_u8x64(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u8x64(result) + } + #[inline(always)] + fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.add_u8x32(a0, b0), self.add_u8x32(a1, b1)) + } + #[inline(always)] + fn sub_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.sub_u8x32(a0, b0), self.sub_u8x32(a1, b1)) + } + #[inline(always)] + fn mul_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.mul_u8x32(a0, b0), self.mul_u8x32(a1, b1)) + } + #[inline(always)] + fn and_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.and_u8x32(a0, b0), self.and_u8x32(a1, b1)) + } + #[inline(always)] + fn or_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.or_u8x32(a0, b0), self.or_u8x32(a1, b1)) + } + #[inline(always)] + fn xor_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.xor_u8x32(a0, b0), self.xor_u8x32(a1, b1)) + } + #[inline(always)] + fn not_u8x64(self, a: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.not_u8x32(a0), self.not_u8x32(a1)) + } + #[inline(always)] + fn shl_u8x64(self, a: u8x64, shift: u32) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.shl_u8x32(a0, shift), self.shl_u8x32(a1, shift)) + } + #[inline(always)] + fn shlv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.shlv_u8x32(a0, b0), self.shlv_u8x32(a1, b1)) + } + #[inline(always)] + fn shr_u8x64(self, a: u8x64, shift: u32) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + self.combine_u8x32(self.shr_u8x32(a0, shift), self.shr_u8x32(a1, shift)) + } + #[inline(always)] + fn shrv_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_u8x32(self.shrv_u8x32(a0, b0), self.shrv_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_eq_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_eq_u8x32(a0, b0), self.simd_eq_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_lt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_lt_u8x32(a0, b0), self.simd_lt_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_le_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_le_u8x32(a0, b0), self.simd_le_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_ge_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { + let (a0, a1) = self.split_u8x64(a); + let (b0, b1) = self.split_u8x64(b); + self.combine_mask8x32(self.simd_ge_u8x32(a0, b0), self.simd_ge_u8x32(a1, b1)) + } + #[inline(always)] + fn simd_gt_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); self.combine_mask8x32(self.simd_gt_u8x32(a0, b0), self.simd_gt_u8x32(a1, b1)) @@ -12714,26 +15170,290 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.add_i16x16(a0, b0), self.add_i16x16(a1, b1)) - } - #[inline(always)] - fn sub_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.sub_i16x16(a0, b0), self.sub_i16x16(a1, b1)) - } - #[inline(always)] - fn mul_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); - let (b0, b1) = self.split_i16x32(b); - self.combine_i16x16(self.mul_i16x16(a0, b0), self.mul_i16x16(a1, b1)) - } - #[inline(always)] - fn and_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { - let (a0, a1) = self.split_i16x32(a); + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + let bytes = self.cvt_to_bytes_i16x32(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i16x32(result) + } + #[inline(always)] + fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.add_i16x16(a0, b0), self.add_i16x16(a1, b1)) + } + #[inline(always)] + fn sub_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.sub_i16x16(a0, b0), self.sub_i16x16(a1, b1)) + } + #[inline(always)] + fn mul_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (b0, b1) = self.split_i16x32(b); + self.combine_i16x16(self.mul_i16x16(a0, b0), self.mul_i16x16(a1, b1)) + } + #[inline(always)] + fn and_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); self.combine_i16x16(self.and_i16x16(a0, b0), self.and_i16x16(a1, b1)) } @@ -13171,6 +15891,270 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + let bytes = self.cvt_to_bytes_u16x32(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u16x32(result) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -13741,6 +16725,270 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + let bytes = self.cvt_to_bytes_i32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i32x16(result) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -14130,6 +17378,270 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + let bytes = self.cvt_to_bytes_u32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u32x16(result) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -14662,6 +18174,270 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let bytes = self.cvt_to_bytes_f64x8(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f64x8(result) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 8e6d9c30..212f52f8 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -249,6 +249,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f32x4, indices: u8x16) -> f32x4 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_f32x4(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_f32x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -870,6 +888,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i8x16, indices: u8x16) -> i8x16 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i8x16(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_i8x16(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1385,6 +1421,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u8x16, indices: u8x16) -> u8x16 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u8x16(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_u8x16(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -2047,6 +2101,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i16x8, indices: u8x16) -> i16x8 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i16x8(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_i16x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2495,6 +2567,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u16x8, indices: u8x16) -> u16x8 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u16x8(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_u16x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -3101,6 +3191,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i32x4, indices: u8x16) -> i32x4 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i32x4(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_i32x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -3533,6 +3641,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u32x4, indices: u8x16) -> u32x4 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u32x4(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_u32x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -4127,6 +4253,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f64x2, indices: u8x16) -> f64x2 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_f64x2(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_f64x2(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -5535,6 +5679,142 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = self.cvt_to_bytes_f32x8(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f32x8(result) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -6079,6 +6359,142 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = self.cvt_to_bytes_i8x32(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i8x32(result) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -6530,40 +6946,176 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.add_u8x16(a0, b0), self.add_u8x16(a1, b1)) - } - #[inline(always)] - fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.sub_u8x16(a0, b0), self.sub_u8x16(a1, b1)) - } - #[inline(always)] - fn mul_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.mul_u8x16(a0, b0), self.mul_u8x16(a1, b1)) - } - #[inline(always)] - fn and_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.and_u8x16(a0, b0), self.and_u8x16(a1, b1)) - } - #[inline(always)] - fn or_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.or_u8x16(a0, b0), self.or_u8x16(a1, b1)) - } - #[inline(always)] - fn xor_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - let (a0, a1) = self.split_u8x32(a); - let (b0, b1) = self.split_u8x32(b); - self.combine_u8x16(self.xor_u8x16(a0, b0), self.xor_u8x16(a1, b1)) + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = self.cvt_to_bytes_u8x32(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u8x32(result) + } + #[inline(always)] + fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.add_u8x16(a0, b0), self.add_u8x16(a1, b1)) + } + #[inline(always)] + fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.sub_u8x16(a0, b0), self.sub_u8x16(a1, b1)) + } + #[inline(always)] + fn mul_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.mul_u8x16(a0, b0), self.mul_u8x16(a1, b1)) + } + #[inline(always)] + fn and_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.and_u8x16(a0, b0), self.and_u8x16(a1, b1)) + } + #[inline(always)] + fn or_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.or_u8x16(a0, b0), self.or_u8x16(a1, b1)) + } + #[inline(always)] + fn xor_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (b0, b1) = self.split_u8x32(b); + self.combine_u8x16(self.xor_u8x16(a0, b0), self.xor_u8x16(a1, b1)) } #[inline(always)] fn not_u8x32(self, a: u8x32) -> u8x32 { @@ -7040,6 +7592,142 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = self.cvt_to_bytes_i16x16(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i16x16(result) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -7431,6 +8119,142 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = self.cvt_to_bytes_u16x16(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u16x16(result) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -7926,27 +8750,163 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.add_i32x4(a0, b0), self.add_i32x4(a1, b1)) - } - #[inline(always)] - fn sub_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.sub_i32x4(a0, b0), self.sub_i32x4(a1, b1)) - } - #[inline(always)] - fn mul_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.mul_i32x4(a0, b0), self.mul_i32x4(a1, b1)) - } - #[inline(always)] - fn and_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = self.cvt_to_bytes_i32x8(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i32x8(result) + } + #[inline(always)] + fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.add_i32x4(a0, b0), self.add_i32x4(a1, b1)) + } + #[inline(always)] + fn sub_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.sub_i32x4(a0, b0), self.sub_i32x4(a1, b1)) + } + #[inline(always)] + fn mul_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.mul_i32x4(a0, b0), self.mul_i32x4(a1, b1)) + } + #[inline(always)] + fn and_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); self.combine_i32x4(self.and_i32x4(a0, b0), self.and_i32x4(a1, b1)) } #[inline(always)] @@ -8286,6 +9246,142 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = self.cvt_to_bytes_u32x8(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u32x8(result) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -8741,6 +9837,142 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = self.cvt_to_bytes_f64x4(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f64x4(result) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -9975,40 +11207,304 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn abs_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) - } - #[inline(always)] - fn neg_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) - } - #[inline(always)] - fn sqrt_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) - } - #[inline(always)] - fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - self.combine_f32x8( - self.approximate_recip_f32x8(a0), - self.approximate_recip_f32x8(a1), - ) - } - #[inline(always)] - fn add_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.add_f32x8(a0, b0), self.add_f32x8(a1, b1)) - } - #[inline(always)] - fn sub_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_f32x8(self.sub_f32x8(a0, b0), self.sub_f32x8(a1, b1)) - } + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + let bytes = self.cvt_to_bytes_f32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f32x16(result) + } + #[inline(always)] + fn abs_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) + } + #[inline(always)] + fn neg_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.neg_f32x8(a0), self.neg_f32x8(a1)) + } + #[inline(always)] + fn sqrt_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8(self.sqrt_f32x8(a0), self.sqrt_f32x8(a1)) + } + #[inline(always)] + fn approximate_recip_f32x16(self, a: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + self.combine_f32x8( + self.approximate_recip_f32x8(a0), + self.approximate_recip_f32x8(a1), + ) + } + #[inline(always)] + fn add_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.add_f32x8(a0, b0), self.add_f32x8(a1, b1)) + } + #[inline(always)] + fn sub_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_f32x8(self.sub_f32x8(a0, b0), self.sub_f32x8(a1, b1)) + } #[inline(always)] fn mul_f32x16(self, a: f32x16, b: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); @@ -10715,97 +12211,361 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) - } - #[inline(always)] - fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) - } - #[inline(always)] - fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) - } - #[inline(always)] - fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.and_i8x32(a0, b0), self.and_i8x32(a1, b1)) - } - #[inline(always)] - fn or_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.or_i8x32(a0, b0), self.or_i8x32(a1, b1)) - } - #[inline(always)] - fn xor_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.xor_i8x32(a0, b0), self.xor_i8x32(a1, b1)) - } - #[inline(always)] - fn not_i8x64(self, a: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.not_i8x32(a0), self.not_i8x32(a1)) - } - #[inline(always)] - fn shl_i8x64(self, a: i8x64, shift: u32) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.shl_i8x32(a0, shift), self.shl_i8x32(a1, shift)) - } - #[inline(always)] - fn shlv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.shlv_i8x32(a0, b0), self.shlv_i8x32(a1, b1)) - } - #[inline(always)] - fn shr_i8x64(self, a: i8x64, shift: u32) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.shr_i8x32(a0, shift), self.shr_i8x32(a1, shift)) - } - #[inline(always)] - fn shrv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.shrv_i8x32(a0, b0), self.shrv_i8x32(a1, b1)) - } - #[inline(always)] - fn simd_eq_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_eq_i8x32(a0, b0), self.simd_eq_i8x32(a1, b1)) - } - #[inline(always)] - fn simd_lt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_lt_i8x32(a0, b0), self.simd_lt_i8x32(a1, b1)) - } - #[inline(always)] - fn simd_le_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_le_i8x32(a0, b0), self.simd_le_i8x32(a1, b1)) - } - #[inline(always)] - fn simd_ge_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_ge_i8x32(a0, b0), self.simd_ge_i8x32(a1, b1)) - } - #[inline(always)] - fn simd_gt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_mask8x32(self.simd_gt_i8x32(a0, b0), self.simd_gt_i8x32(a1, b1)) + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let bytes = self.cvt_to_bytes_i8x64(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i8x64(result) + } + #[inline(always)] + fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) + } + #[inline(always)] + fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) + } + #[inline(always)] + fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) + } + #[inline(always)] + fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.and_i8x32(a0, b0), self.and_i8x32(a1, b1)) + } + #[inline(always)] + fn or_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.or_i8x32(a0, b0), self.or_i8x32(a1, b1)) + } + #[inline(always)] + fn xor_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.xor_i8x32(a0, b0), self.xor_i8x32(a1, b1)) + } + #[inline(always)] + fn not_i8x64(self, a: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.not_i8x32(a0), self.not_i8x32(a1)) + } + #[inline(always)] + fn shl_i8x64(self, a: i8x64, shift: u32) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.shl_i8x32(a0, shift), self.shl_i8x32(a1, shift)) + } + #[inline(always)] + fn shlv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.shlv_i8x32(a0, b0), self.shlv_i8x32(a1, b1)) + } + #[inline(always)] + fn shr_i8x64(self, a: i8x64, shift: u32) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.shr_i8x32(a0, shift), self.shr_i8x32(a1, shift)) + } + #[inline(always)] + fn shrv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.shrv_i8x32(a0, b0), self.shrv_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_eq_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_eq_i8x32(a0, b0), self.simd_eq_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_lt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_lt_i8x32(a0, b0), self.simd_lt_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_le_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_le_i8x32(a0, b0), self.simd_le_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_ge_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_ge_i8x32(a0, b0), self.simd_ge_i8x32(a1, b1)) + } + #[inline(always)] + fn simd_gt_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_mask8x32(self.simd_gt_i8x32(a0, b0), self.simd_gt_i8x32(a1, b1)) } #[inline(always)] fn zip_low_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { @@ -11287,6 +13047,270 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let bytes = self.cvt_to_bytes_u8x64(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u8x64(result) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -11968,6 +13992,270 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + let bytes = self.cvt_to_bytes_i16x32(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i16x32(result) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -12425,6 +14713,270 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + let bytes = self.cvt_to_bytes_u16x32(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u16x32(result) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -13032,6 +15584,270 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + let bytes = self.cvt_to_bytes_i32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i32x16(result) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -13421,6 +16237,270 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + let bytes = self.cvt_to_bytes_u32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u32x16(result) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -13953,6 +17033,270 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let bytes = self.cvt_to_bytes_f64x8(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f64x8(result) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index 834cab1b..a78202d6 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -226,6 +226,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + let result = u8x16_swizzle(self.cvt_to_bytes_f32x4(a).val.0, indices.into()); + self.cvt_from_bytes_f32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { f32x4_abs(a.into()).simd_into(self) } @@ -628,6 +636,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + let result = u8x16_swizzle(self.cvt_to_bytes_i8x16(a).val.0, indices.into()); + self.cvt_from_bytes_i8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { i8x16_add(a.into(), b.into()).simd_into(self) } @@ -985,6 +1001,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + let result = u8x16_swizzle(self.cvt_to_bytes_u8x16(a).val.0, indices.into()); + self.cvt_from_bytes_u8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { u8x16_add(a.into(), b.into()).simd_into(self) } @@ -1407,6 +1431,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + let result = u8x16_swizzle(self.cvt_to_bytes_i16x8(a).val.0, indices.into()); + self.cvt_from_bytes_i16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { i16x8_add(a.into(), b.into()).simd_into(self) } @@ -1700,6 +1732,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + let result = u8x16_swizzle(self.cvt_to_bytes_u16x8(a).val.0, indices.into()); + self.cvt_from_bytes_u16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { u16x8_add(a.into(), b.into()).simd_into(self) } @@ -2070,6 +2110,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + let result = u8x16_swizzle(self.cvt_to_bytes_i32x4(a).val.0, indices.into()); + self.cvt_from_bytes_i32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { i32x4_add(a.into(), b.into()).simd_into(self) } @@ -2343,6 +2391,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + let result = u8x16_swizzle(self.cvt_to_bytes_u32x4(a).val.0, indices.into()); + self.cvt_from_bytes_u32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { u32x4_add(a.into(), b.into()).simd_into(self) } @@ -2697,6 +2753,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + let result = u8x16_swizzle(self.cvt_to_bytes_f64x2(a).val.0, indices.into()); + self.cvt_from_bytes_f64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { f64x2_abs(a.into()).simd_into(self) } @@ -3675,6 +3739,142 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = self.cvt_to_bytes_f32x8(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f32x8(result) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -4218,6 +4418,142 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = self.cvt_to_bytes_i8x32(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i8x32(result) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -4668,6 +5004,142 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = self.cvt_to_bytes_u8x32(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u8x32(result) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -5177,6 +5649,142 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = self.cvt_to_bytes_i16x16(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i16x16(result) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -5567,6 +6175,142 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = self.cvt_to_bytes_u16x16(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u16x16(result) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -6049,31 +6793,167 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.add_i32x4(a0, b0), self.add_i32x4(a1, b1)) - } - #[inline(always)] - fn sub_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.sub_i32x4(a0, b0), self.sub_i32x4(a1, b1)) - } - #[inline(always)] - fn mul_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.mul_i32x4(a0, b0), self.mul_i32x4(a1, b1)) - } - #[inline(always)] - fn and_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { - let (a0, a1) = self.split_i32x8(a); - let (b0, b1) = self.split_i32x8(b); - self.combine_i32x4(self.and_i32x4(a0, b0), self.and_i32x4(a1, b1)) - } - #[inline(always)] - fn or_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = self.cvt_to_bytes_i32x8(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i32x8(result) + } + #[inline(always)] + fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.add_i32x4(a0, b0), self.add_i32x4(a1, b1)) + } + #[inline(always)] + fn sub_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.sub_i32x4(a0, b0), self.sub_i32x4(a1, b1)) + } + #[inline(always)] + fn mul_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.mul_i32x4(a0, b0), self.mul_i32x4(a1, b1)) + } + #[inline(always)] + fn and_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (b0, b1) = self.split_i32x8(b); + self.combine_i32x4(self.and_i32x4(a0, b0), self.and_i32x4(a1, b1)) + } + #[inline(always)] + fn or_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); self.combine_i32x4(self.or_i32x4(a0, b0), self.or_i32x4(a1, b1)) @@ -6408,6 +7288,142 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = self.cvt_to_bytes_u32x8(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u32x8(result) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -6862,6 +7878,142 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = self.cvt_to_bytes_f64x4(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f64x4(result) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -8093,6 +9245,270 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + let bytes = self.cvt_to_bytes_f32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f32x16(result) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -8803,65 +10219,329 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) - } - #[inline(always)] - fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) - } - #[inline(always)] - fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) - } - #[inline(always)] - fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.and_i8x32(a0, b0), self.and_i8x32(a1, b1)) - } - #[inline(always)] - fn or_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.or_i8x32(a0, b0), self.or_i8x32(a1, b1)) - } - #[inline(always)] - fn xor_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.xor_i8x32(a0, b0), self.xor_i8x32(a1, b1)) - } - #[inline(always)] - fn not_i8x64(self, a: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.not_i8x32(a0), self.not_i8x32(a1)) - } - #[inline(always)] - fn shl_i8x64(self, a: i8x64, shift: u32) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.shl_i8x32(a0, shift), self.shl_i8x32(a1, shift)) - } - #[inline(always)] - fn shlv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.shlv_i8x32(a0, b0), self.shlv_i8x32(a1, b1)) - } - #[inline(always)] - fn shr_i8x64(self, a: i8x64, shift: u32) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.shr_i8x32(a0, shift), self.shr_i8x32(a1, shift)) - } - #[inline(always)] - fn shrv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let bytes = self.cvt_to_bytes_i8x64(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i8x64(result) + } + #[inline(always)] + fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.add_i8x32(a0, b0), self.add_i8x32(a1, b1)) + } + #[inline(always)] + fn sub_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.sub_i8x32(a0, b0), self.sub_i8x32(a1, b1)) + } + #[inline(always)] + fn mul_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.mul_i8x32(a0, b0), self.mul_i8x32(a1, b1)) + } + #[inline(always)] + fn and_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.and_i8x32(a0, b0), self.and_i8x32(a1, b1)) + } + #[inline(always)] + fn or_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.or_i8x32(a0, b0), self.or_i8x32(a1, b1)) + } + #[inline(always)] + fn xor_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.xor_i8x32(a0, b0), self.xor_i8x32(a1, b1)) + } + #[inline(always)] + fn not_i8x64(self, a: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.not_i8x32(a0), self.not_i8x32(a1)) + } + #[inline(always)] + fn shl_i8x64(self, a: i8x64, shift: u32) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.shl_i8x32(a0, shift), self.shl_i8x32(a1, shift)) + } + #[inline(always)] + fn shlv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.shlv_i8x32(a0, b0), self.shlv_i8x32(a1, b1)) + } + #[inline(always)] + fn shr_i8x64(self, a: i8x64, shift: u32) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.shr_i8x32(a0, shift), self.shr_i8x32(a1, shift)) + } + #[inline(always)] + fn shrv_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); self.combine_i8x32(self.shrv_i8x32(a0, b0), self.shrv_i8x32(a1, b1)) } @@ -9374,6 +11054,270 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let bytes = self.cvt_to_bytes_u8x64(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u8x64(result) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -9998,6 +11942,270 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + let bytes = self.cvt_to_bytes_i16x32(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i16x32(result) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -10454,6 +12662,270 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + let bytes = self.cvt_to_bytes_u16x32(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u16x32(result) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -11012,6 +13484,270 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + let bytes = self.cvt_to_bytes_i32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i32x16(result) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -11400,6 +14136,270 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + let bytes = self.cvt_to_bytes_u32x16(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u32x16(result) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -11902,6 +14902,270 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let bytes = self.cvt_to_bytes_f64x8(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_f64x8(result) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index 4ed67d0b..5e579a4e 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -88,6 +88,9 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { } } } + OpSig::SwizzleDynPrecise => { + panic!("whole-vector swizzles cannot be done via split/combine"); + } OpSig::Ternary => { quote! { #method_sig { diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index db9350e2..3ea5d546 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -436,6 +436,33 @@ impl Level for Fallback { } } } + OpSig::SwizzleDynPrecise => { + let bytes_ty = vec_ty.bytes_ty(); + let bytes_rust = bytes_ty.rust(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + let byte_count = bytes_ty.len; + let items = make_list( + (0..byte_count) + .map(|idx| { + quote! { + { + let index = indices[#idx] as usize; + bytes.get(index).copied().unwrap_or(0) + } + } + }) + .collect::>(), + ); + + quote! { + #method_sig { + let bytes = self.#to_bytes(a); + let result: #bytes_rust = #items.simd_into(self); + self.#from_bytes(result) + } + } + } OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index 80d39f39..cf7722c1 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -456,6 +456,45 @@ impl Level for Neon { } }) } + OpSig::SwizzleDynPrecise => { + let bytes_ty = vec_ty.bytes_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + + self.kernel_method(op, vec_ty, |token| { + let body = match vec_ty.n_bits() { + 128 => quote! { + let result = vqtbl1q_u8(#token.#to_bytes(a).val.0, indices.into()); + }, + 256 => quote! { + let table = #token.#to_bytes(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = uint8x16x2_t( + vqtbl2q_u8(table, indices.0), + vqtbl2q_u8(table, indices.1), + ); + }, + 512 => quote! { + let table = #token.#to_bytes(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + }, + _ => unreachable!(), + }; + + quote! { + #body + #token.#from_bytes(#bytes { val: #wrapper(result), simd: #token }) + } + }) + } OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_simd_types.rs b/fearless_simd_gen/src/mk_simd_types.rs index b40e2dc0..c334caae 100644 --- a/fearless_simd_gen/src/mk_simd_types.rs +++ b/fearless_simd_gen/src/mk_simd_types.rs @@ -435,6 +435,7 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { let shift_elements_left_op = generic_op_name("shift_elements_left", ty); let shift_elements_right_op = generic_op_name("shift_elements_right", ty); let swizzle_dyn_within_blocks_op = generic_op_name("swizzle_dyn_within_blocks", ty); + let swizzle_dyn_precise_op = generic_op_name("swizzle_dyn_precise", ty); quote! { impl SimdBase for #name { type Element = #scalar; @@ -517,6 +518,11 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd.#swizzle_dyn_within_blocks_op(self, indices.simd_into(self.simd)) } + + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd.#swizzle_dyn_precise_op(self, indices.simd_into(self.simd)) + } } impl crate::#vec_trait_id for #name { #( #methods )* diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index 1baf395b..29dc7e9a 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -517,6 +517,24 @@ impl Level for WasmSimd128 { } } } + OpSig::SwizzleDynPrecise => { + if vec_ty.n_bits() != self.native_width() { + return crate::mk_fallback::Fallback.make_method(op, vec_ty); + } + + let bytes_ty = vec_ty.bytes_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + + quote! { + #method_sig { + let result = u8x16_swizzle(self.#to_bytes(a).val.0, indices.into()); + self.#from_bytes(#bytes { val: #wrapper(result), simd: self }) + } + } + } OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index ac8cd462..acc193ac 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -313,6 +313,7 @@ impl Level for X86 { unreachable!("element moves use generic lowering") } OpSig::SwizzleDynWithinBlocks => self.handle_swizzle_dyn_within_blocks(op, vec_ty), + OpSig::SwizzleDynPrecise => self.handle_swizzle_dyn_precise(op, vec_ty), OpSig::Cvt { target_ty, scalar_bits, @@ -2758,6 +2759,75 @@ impl X86 { }) } + pub(crate) fn handle_swizzle_dyn_precise(&self, op: Op, vec_ty: &VecType) -> TokenStream { + let bytes_ty = vec_ty.bytes_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + + if *self == Self::Sse2 + || (*self == Self::Sse4_2 && vec_ty.n_bits() != 128) + || (*self == Self::Avx2 && vec_ty.n_bits() == 512) + { + return fallback_method(op, vec_ty); + } + + self.kernel_method(op, vec_ty, |token| { + let body = match (*self, vec_ty.n_bits()) { + (Self::Sse4_2 | Self::Avx2, 128) => quote! { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(#token.#to_bytes(a).val.0, zeroing_indices); + let result_bytes = #bytes { val: #wrapper(result), simd: #token }; + }, + (Self::Avx2, 256) => quote! { + // carefully tuned implementation reused from std::simd: + // https://github.com/rust-lang/portable-simd/blob/7d497cca160ae6062acc1a2db838667f83c0b58e/crates/core_simd/src/swizzle_dyn.rs#L205-L224 + let bytes = #token.#to_bytes(a); + let idxs = indices; + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let hi_shuf = #bytes { + val: #wrapper(_mm256_shuffle_epi8(hihi, idxs.into())), + simd: #token, + }; + let result_bytes = idxs + .simd_lt(#bytes::splat(#token, 32)) + .select(hi_shuf, #bytes::splat(#token, 0)); + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let lo_shuf = #bytes { + val: #wrapper(_mm256_shuffle_epi8(lolo, idxs.into())), + simd: #token, + }; + let result_bytes = idxs + .simd_lt(#bytes::splat(#token, 16)) + .select(lo_shuf, result_bytes); + }, + (Self::Avx512, 128 | 256 | 512) => { + let cmp = intrinsic_ident("cmp", "epu8_mask", vec_ty.n_bits()); + let maskz_permute = + intrinsic_ident("maskz_permutexvar", "epi8", vec_ty.n_bits()); + let set1 = set1_intrinsic(&bytes_ty); + let byte_count = signed_literal(bytes_ty.len as u64, 8); + quote! { + let bytes = #token.#to_bytes(a).val.0; + let indices = indices.into(); + let in_range = #cmp::<{ _MM_CMPINT_LT }>(indices, #set1(#byte_count)); + let result = #maskz_permute(in_range, indices, bytes); + let result_bytes = #bytes { val: #wrapper(result), simd: #token }; + } + } + _ => unreachable!(), + }; + + quote! { + #body + #token.#from_bytes(result_bytes) + } + }) + } + pub(crate) fn handle_cvt( &self, op: Op, diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index ad2c1832..49e298c5 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -96,6 +96,9 @@ pub(crate) enum OpSig { /// Takes a vector and a same-width byte-index vector, and returns the original vector type with its bytes /// dynamically swizzled within each 128-bit block. SwizzleDynWithinBlocks, + /// Takes a vector and a same-width byte-index vector, and returns the original vector type with its bytes + /// dynamically swizzled across the whole vector. Out-of-range indices produce zero bytes. + SwizzleDynPrecise, /// Takes a single argument of the source vector type, and returns a vector type of the target scalar type and the /// same length. Cvt { @@ -322,7 +325,7 @@ impl Op { OpSig::ElementRotate { .. } => (vec![vec.clone()], vec), OpSig::ElementShift { .. } => (vec![vec.clone(), splat_arg_ty(vec_ty)], vec), OpSig::Slide { .. } => (vec![vec.clone(), vec.clone()], vec), - OpSig::SwizzleDynWithinBlocks => { + OpSig::SwizzleDynWithinBlocks | OpSig::SwizzleDynPrecise => { let bytes_ty = vec_ty.bytes_ty().rust(); (vec![vec.clone(), quote! { #bytes_ty<#simd_ty> }], vec) } @@ -452,7 +455,7 @@ impl Op { let arg1 = &arg_names[1]; quote! { (#arg0, #arg1: impl SimdInto) -> Self } } - OpSig::SwizzleDynWithinBlocks => { + OpSig::SwizzleDynWithinBlocks | OpSig::SwizzleDynPrecise => { let arg0 = &arg_names[0]; let arg1 = &arg_names[1]; quote! { (#arg0, #arg1: impl SimdInto) -> Self } @@ -667,6 +670,13 @@ const BASE_OPS: &[Op] = &[ The `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\n\ Out-of-range index behavior varies by platform.", ), + Op::new( + "swizzle_dyn_precise", + OpKind::BaseTraitMethod, + OpSig::SwizzleDynPrecise, + "Dynamically swizzle this vector's bytes across the whole vector.\n\n\ + The `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes.", + ), ]; const MASK_REPRESENTATION_OPS: &[Op] = &[ @@ -1655,6 +1665,7 @@ impl OpSig { | Self::AsArray { .. } | Self::StoreArray | Self::MaskSet + | Self::SwizzleDynPrecise | Self::Slide { granularity: SlideGranularity::AcrossBlocks, .. @@ -1695,7 +1706,7 @@ impl OpSig { | Self::AsArray { .. } | Self::FromBytes | Self::ToBytes => &["a"], - Self::SwizzleDynWithinBlocks => &["a", "indices"], + Self::SwizzleDynWithinBlocks | Self::SwizzleDynPrecise => &["a", "indices"], Self::Binary | Self::Compare | Self::Combine { .. } @@ -1730,7 +1741,7 @@ impl OpSig { | Self::MaskReduce { .. } | Self::AsArray { .. } | Self::ToBytes => &["self"], - Self::SwizzleDynWithinBlocks => &["self", "indices"], + Self::SwizzleDynWithinBlocks | Self::SwizzleDynPrecise => &["self", "indices"], Self::Binary | Self::Compare | Self::Zip { .. } @@ -1797,6 +1808,7 @@ impl OpSig { | Self::FromBytes | Self::ToBytes | Self::SwizzleDynWithinBlocks + | Self::SwizzleDynPrecise | Self::Slide { .. } => return None, }; Some(args) diff --git a/fearless_simd_tests/tests/harness/lm_generated.rs b/fearless_simd_tests/tests/harness/lm_generated.rs index 7034cc6d..18d91a29 100644 --- a/fearless_simd_tests/tests/harness/lm_generated.rs +++ b/fearless_simd_tests/tests/harness/lm_generated.rs @@ -8,3 +8,4 @@ mod mask_roundtrip; mod mask_roundtrip_x86; mod mod_256; mod mod_512; +mod swizzle_dyn_precise; diff --git a/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs b/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs new file mode 100644 index 00000000..5dd8933d --- /dev/null +++ b/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs @@ -0,0 +1,96 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +fn expected_swizzle_precise(bytes: [u8; N], indices: [u8; N]) -> [u8; N] { + core::array::from_fn(|i| { + let index = indices[i] as usize; + bytes.get(index).copied().unwrap_or(0) + }) +} + +#[simd_test] +fn swizzle_dyn_precise_u8x16(simd: S) { + let bytes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let indices = [15, 14, 0, 1, 16, 17, 31, 127, 128, 255, 8, 7, 6, 5, 4, 3]; + let expected = expected_swizzle_precise(bytes, indices); + + let value = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let result = value.swizzle_dyn_precise(index_vec); + + assert_eq!(*result, expected); +} + +#[simd_test] +fn swizzle_dyn_precise_u8x32_crosses_blocks(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| u8::try_from(i + 1).unwrap()); + let indices = [ + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 0, 1, 2, 3, 4, 5, 6, 7, 32, + 33, 127, 128, 255, 15, 16, 31, + ]; + let expected = expected_swizzle_precise(bytes, indices); + + let value = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let result = value.swizzle_dyn_precise(index_vec); + + assert_eq!(*result, expected); +} + +#[simd_test] +fn swizzle_dyn_precise_u8x64_crosses_blocks(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(|i| u8::try_from(i + 1).unwrap()); + let indices = [ + 63, 48, 32, 16, 0, 15, 31, 47, 62, 49, 33, 17, 1, 14, 30, 46, 45, 29, 13, 2, 18, 34, 50, + 61, 60, 44, 28, 12, 3, 19, 35, 51, 52, 36, 20, 4, 11, 27, 43, 59, 58, 42, 26, 10, 5, 21, + 37, 53, 64, 65, 127, 128, 255, 54, 38, 22, 6, 9, 25, 41, 57, 56, 40, 24, + ]; + let expected = expected_swizzle_precise(bytes, indices); + + let value = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let result = value.swizzle_dyn_precise(index_vec); + + assert_eq!(*result, expected); +} + +#[simd_test] +fn swizzle_dyn_precise_bitcast_f32x8(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| u8::try_from(i * 3 + 1).unwrap()); + let indices = [ + 16, 17, 18, 19, 0, 1, 2, 3, 31, 30, 29, 28, 32, 33, 128, 255, 4, 5, 6, 7, 20, 21, 22, 23, + 27, 26, 25, 24, 15, 14, 13, 12, + ]; + let expected = expected_swizzle_precise(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let value: f32x8 = byte_vec.bitcast(); + let index_vec = u8x32::simd_from(simd, indices); + let result_bytes: u8x32 = value.swizzle_dyn_precise(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_precise_generic_indices(simd: S) { + #[inline(always)] + fn do_swizzle>(value: V, indices: V::Bytes) -> V { + value.swizzle_dyn_precise(indices) + } + + let bytes: [u8; 32] = core::array::from_fn(|i| u8::try_from(i + 1).unwrap()); + let indices = [ + 16, 17, 18, 19, 20, 21, 22, 23, 31, 30, 29, 28, 27, 26, 25, 24, 0, 1, 2, 3, 4, 5, 6, 7, 32, + 64, 127, 128, 255, 8, 9, 10, + ]; + let expected = expected_swizzle_precise(bytes, indices); + + let value = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let result = do_swizzle::>(value, index_vec); + + assert_eq!(*result, expected); +} From 54d67603a97ddc75e9ec9aa4615ab86665ec6c43 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 02:11:57 +0100 Subject: [PATCH 02/23] Add a 256-bit SSE4.2 implementation as an experiment --- fearless_simd/src/generated/sse4_2.rs | 1256 ++++--------------------- fearless_simd_gen/src/mk_x86.rs | 48 +- 2 files changed, 239 insertions(+), 1065 deletions(-) diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 212f52f8..26e30dd6 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -5680,139 +5680,30 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { - let bytes = self.cvt_to_bytes_f32x8(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_f32x8(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = token.cvt_to_bytes_f32x8(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + token.cvt_from_bytes_f32x8(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { @@ -6360,139 +6251,30 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { - let bytes = self.cvt_to_bytes_i8x32(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_i8x32(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = token.cvt_to_bytes_i8x32(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + token.cvt_from_bytes_i8x32(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { @@ -6947,139 +6729,30 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { - let bytes = self.cvt_to_bytes_u8x32(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_u8x32(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = token.cvt_to_bytes_u8x32(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + token.cvt_from_bytes_u8x32(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { @@ -7593,139 +7266,30 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { - let bytes = self.cvt_to_bytes_i16x16(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_i16x16(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = token.cvt_to_bytes_i16x16(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + token.cvt_from_bytes_i16x16(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { @@ -8120,139 +7684,30 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { - let bytes = self.cvt_to_bytes_u16x16(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_u16x16(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = token.cvt_to_bytes_u16x16(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + token.cvt_from_bytes_u16x16(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { @@ -8751,139 +8206,30 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { - let bytes = self.cvt_to_bytes_i32x8(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_i32x8(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = token.cvt_to_bytes_i32x8(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + token.cvt_from_bytes_i32x8(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { @@ -9247,139 +8593,30 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { - let bytes = self.cvt_to_bytes_u32x8(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_u32x8(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = token.cvt_to_bytes_u32x8(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + token.cvt_from_bytes_u32x8(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { @@ -9838,139 +9075,30 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { - let bytes = self.cvt_to_bytes_f64x4(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_f64x4(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = token.cvt_to_bytes_f64x4(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + token.cvt_from_bytes_f64x4(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index acc193ac..039a043e 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2767,7 +2767,7 @@ impl X86 { let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); if *self == Self::Sse2 - || (*self == Self::Sse4_2 && vec_ty.n_bits() != 128) + || (*self == Self::Sse4_2 && vec_ty.n_bits() == 512) || (*self == Self::Avx2 && vec_ty.n_bits() == 512) { return fallback_method(op, vec_ty); @@ -2782,6 +2782,52 @@ impl X86 { let result = _mm_shuffle_epi8(#token.#to_bytes(a).val.0, zeroing_indices); let result_bytes = #bytes { val: #wrapper(result), simd: #token }; }, + (Self::Sse4_2, 256) => { + // We can take advantage of the "precise" property (out-of-range is 0) + // to assemble a double-vector-width arbitrary shuffle. + // The trick is to split the input into two, then run for each half of output indices J: + // ``` + // from_low = swizzle_H(low_table, J); + // from_high = swizzle_H(high_table, J.wrapping_sub(H)); + // result = from_low | from_high; + // ``` + // Since each element is out-of-range for at least one half, + // the final combine is very cheap, it's just a bitwise or. + // So all we need is 4 half-size shuffles and a little bit of math! + let half_bytes_ty = VecType::new(ScalarType::Unsigned, 8, bytes_ty.len / 2); + let split_bytes = generic_op_name("split", &bytes_ty); + let combine_half_bytes = generic_op_name("combine", &half_bytes_ty); + let swizzle_half = generic_op_name("swizzle_dyn_precise", &half_bytes_ty); + let splat_half = generic_op_name("splat", &half_bytes_ty); + let sub_half = generic_op_name("sub", &half_bytes_ty); + let or_half = generic_op_name("or", &half_bytes_ty); + let half_len = Literal::u8_unsuffixed((bytes_ty.len / 2) as u8); + + quote! { + let bytes = #token.#to_bytes(a); + let (table_low, table_high) = #token.#split_bytes(bytes); + let (indices_low, indices_high) = #token.#split_bytes(indices); + let high_table_offset = #token.#splat_half(#half_len); + + let output_low_from_low = + #token.#swizzle_half(table_low, indices_low); + let output_low_from_high = #token.#swizzle_half( + table_high, + #token.#sub_half(indices_low, high_table_offset), + ); + let output_low = #token.#or_half(output_low_from_low, output_low_from_high); + + let output_high_from_low = + #token.#swizzle_half(table_low, indices_high); + let output_high_from_high = #token.#swizzle_half( + table_high, + #token.#sub_half(indices_high, high_table_offset), + ); + let output_high = #token.#or_half(output_high_from_low, output_high_from_high); + + let result_bytes = #token.#combine_half_bytes(output_low, output_high); + } + } (Self::Avx2, 256) => quote! { // carefully tuned implementation reused from std::simd: // https://github.com/rust-lang/portable-simd/blob/7d497cca160ae6062acc1a2db838667f83c0b58e/crates/core_simd/src/swizzle_dyn.rs#L205-L224 From 29262e1d2bf5acf820f1b8e5d587095d7a8bcac8 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 02:33:16 +0100 Subject: [PATCH 03/23] Split out recursive swizzle formulation into a helper and apply it to AVX2 --- fearless_simd/src/generated/avx2.rs | 2280 +++------------------------ fearless_simd_gen/src/mk_x86.rs | 98 +- 2 files changed, 241 insertions(+), 2137 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index a614d2ec..e638dd6c 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -12539,267 +12539,30 @@ impl Simd for Avx2 { } #[inline(always)] fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { - let bytes = self.cvt_to_bytes_f32x16(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_f32x16(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x16, indices: u8x64) -> f32x16 { + let bytes = token.cvt_to_bytes_f32x16(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + token.cvt_from_bytes_f32x16(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { @@ -13543,267 +13306,30 @@ impl Simd for Avx2 { } #[inline(always)] fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { - let bytes = self.cvt_to_bytes_i8x64(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_i8x64(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i8x64, indices: u8x64) -> i8x64 { + let bytes = token.cvt_to_bytes_i8x64(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + token.cvt_from_bytes_i8x64(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { @@ -14379,267 +13905,30 @@ impl Simd for Avx2 { } #[inline(always)] fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { - let bytes = self.cvt_to_bytes_u8x64(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_u8x64(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x64, indices: u8x64) -> u8x64 { + let bytes = token.cvt_to_bytes_u8x64(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + token.cvt_from_bytes_u8x64(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { @@ -15318,267 +14607,30 @@ impl Simd for Avx2 { } #[inline(always)] fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { - let bytes = self.cvt_to_bytes_i16x32(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_i16x32(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i16x32, indices: u8x64) -> i16x32 { + let bytes = token.cvt_to_bytes_i16x32(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + token.cvt_from_bytes_i16x32(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { @@ -16039,267 +15091,30 @@ impl Simd for Avx2 { } #[inline(always)] fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { - let bytes = self.cvt_to_bytes_u16x32(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_u16x32(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u16x32, indices: u8x64) -> u16x32 { + let bytes = token.cvt_to_bytes_u16x32(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + token.cvt_from_bytes_u16x32(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { @@ -16922,267 +15737,30 @@ impl Simd for Avx2 { } #[inline(always)] fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { - let bytes = self.cvt_to_bytes_i32x16(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_i32x16(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i32x16, indices: u8x64) -> i32x16 { + let bytes = token.cvt_to_bytes_i32x16(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + token.cvt_from_bytes_i32x16(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { @@ -17575,267 +16153,30 @@ impl Simd for Avx2 { } #[inline(always)] fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { - let bytes = self.cvt_to_bytes_u32x16(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_u32x16(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u32x16, indices: u8x64) -> u32x16 { + let bytes = token.cvt_to_bytes_u32x16(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + token.cvt_from_bytes_u32x16(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { @@ -18392,267 +16733,30 @@ impl Simd for Avx2 { } #[inline(always)] fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { - let bytes = self.cvt_to_bytes_f64x8(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_f64x8(result) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x8, indices: u8x64) -> f64x8 { + let bytes = token.cvt_to_bytes_f64x8(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + token.cvt_from_bytes_f64x8(result_bytes) + } + ); + kernel(self, a, indices) } #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 039a043e..4ec5ed5d 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2766,10 +2766,7 @@ impl X86 { let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); - if *self == Self::Sse2 - || (*self == Self::Sse4_2 && vec_ty.n_bits() == 512) - || (*self == Self::Avx2 && vec_ty.n_bits() == 512) - { + if *self == Self::Sse2 || (*self == Self::Sse4_2 && vec_ty.n_bits() == 512) { return fallback_method(op, vec_ty); } @@ -2782,51 +2779,8 @@ impl X86 { let result = _mm_shuffle_epi8(#token.#to_bytes(a).val.0, zeroing_indices); let result_bytes = #bytes { val: #wrapper(result), simd: #token }; }, - (Self::Sse4_2, 256) => { - // We can take advantage of the "precise" property (out-of-range is 0) - // to assemble a double-vector-width arbitrary shuffle. - // The trick is to split the input into two, then run for each half of output indices J: - // ``` - // from_low = swizzle_H(low_table, J); - // from_high = swizzle_H(high_table, J.wrapping_sub(H)); - // result = from_low | from_high; - // ``` - // Since each element is out-of-range for at least one half, - // the final combine is very cheap, it's just a bitwise or. - // So all we need is 4 half-size shuffles and a little bit of math! - let half_bytes_ty = VecType::new(ScalarType::Unsigned, 8, bytes_ty.len / 2); - let split_bytes = generic_op_name("split", &bytes_ty); - let combine_half_bytes = generic_op_name("combine", &half_bytes_ty); - let swizzle_half = generic_op_name("swizzle_dyn_precise", &half_bytes_ty); - let splat_half = generic_op_name("splat", &half_bytes_ty); - let sub_half = generic_op_name("sub", &half_bytes_ty); - let or_half = generic_op_name("or", &half_bytes_ty); - let half_len = Literal::u8_unsuffixed((bytes_ty.len / 2) as u8); - - quote! { - let bytes = #token.#to_bytes(a); - let (table_low, table_high) = #token.#split_bytes(bytes); - let (indices_low, indices_high) = #token.#split_bytes(indices); - let high_table_offset = #token.#splat_half(#half_len); - - let output_low_from_low = - #token.#swizzle_half(table_low, indices_low); - let output_low_from_high = #token.#swizzle_half( - table_high, - #token.#sub_half(indices_low, high_table_offset), - ); - let output_low = #token.#or_half(output_low_from_low, output_low_from_high); - - let output_high_from_low = - #token.#swizzle_half(table_low, indices_high); - let output_high_from_high = #token.#swizzle_half( - table_high, - #token.#sub_half(indices_high, high_table_offset), - ); - let output_high = #token.#or_half(output_high_from_low, output_high_from_high); - - let result_bytes = #token.#combine_half_bytes(output_low, output_high); - } + (Self::Sse4_2, 256) | (Self::Avx2, 512) => { + Self::recursive_swizzle_dyn_precise_body(vec_ty, token) } (Self::Avx2, 256) => quote! { // carefully tuned implementation reused from std::simd: @@ -2874,6 +2828,52 @@ impl X86 { }) } + fn recursive_swizzle_dyn_precise_body(vec_ty: &VecType, token: &Ident) -> TokenStream { + // We can take advantage of the "precise" property (out-of-range is 0) + // to assemble a double-vector-width arbitrary shuffle. + // The trick is to split the input into two, then run for each half of output indices J: + // ``` + // from_low = swizzle_H(low_table, J); + // from_high = swizzle_H(high_table, J.wrapping_sub(H)); + // result = from_low | from_high; + // ``` + // Since each element is out-of-range for at least one half, + // the final combine is very cheap: a bitwise or. + let bytes_ty = vec_ty.bytes_ty(); + let half_bytes_ty = VecType::new(ScalarType::Unsigned, 8, bytes_ty.len / 2); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let split_bytes = generic_op_name("split", &bytes_ty); + let combine_half_bytes = generic_op_name("combine", &half_bytes_ty); + let swizzle_half = generic_op_name("swizzle_dyn_precise", &half_bytes_ty); + let splat_half = generic_op_name("splat", &half_bytes_ty); + let sub_half = generic_op_name("sub", &half_bytes_ty); + let or_half = generic_op_name("or", &half_bytes_ty); + let half_len = Literal::u8_unsuffixed((bytes_ty.len / 2) as u8); + + quote! { + let bytes = #token.#to_bytes(a); + let (table_low, table_high) = #token.#split_bytes(bytes); + let (indices_low, indices_high) = #token.#split_bytes(indices); + let high_table_offset = #token.#splat_half(#half_len); + + let output_low_from_low = #token.#swizzle_half(table_low, indices_low); + let output_low_from_high = #token.#swizzle_half( + table_high, + #token.#sub_half(indices_low, high_table_offset), + ); + let output_low = #token.#or_half(output_low_from_low, output_low_from_high); + + let output_high_from_low = #token.#swizzle_half(table_low, indices_high); + let output_high_from_high = #token.#swizzle_half( + table_high, + #token.#sub_half(indices_high, high_table_offset), + ); + let output_high = #token.#or_half(output_high_from_low, output_high_from_high); + + let result_bytes = #token.#combine_half_bytes(output_low, output_high); + } + } + pub(crate) fn handle_cvt( &self, op: Op, From 51113e86eeffdff51857c52d98b07ad73f390d12 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 03:06:43 +0100 Subject: [PATCH 04/23] Implement a dedicated AVX2 512-bit shuffle path --- fearless_simd/src/generated/avx2.rs | 344 +++++++++++++++++----------- fearless_simd_gen/src/mk_x86.rs | 58 ++++- 2 files changed, 271 insertions(+), 131 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index e638dd6c..c39d9d78 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -12543,22 +12543,33 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: f32x16, indices: u8x64) -> f32x16 { let bytes = token.cvt_to_bytes_f32x16(a); - let (table_low, table_high) = token.split_u8x64(bytes); - let (indices_low, indices_high) = token.split_u8x64(indices); - let high_table_offset = token.splat_u8x32(32); - let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); - let output_low_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_low, high_table_offset), - ); - let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); - let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); - let output_high_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_high, high_table_offset), - ); - let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); - let result_bytes = token.combine_u8x32(output_low, output_high); + let bytes01 = bytes.val.0[0]; + let bytes23 = bytes.val.0[1]; + let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); + let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); + let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); + let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); + let oob_bias = _mm256_set1_epi8(64); + let swizzle_half = |idxs: __m256i| -> __m256i { + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + }; + let result = [ + swizzle_half(indices.val.0[0]), + swizzle_half(indices.val.0[1]), + ]; + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; token.cvt_from_bytes_f32x16(result_bytes) } ); @@ -13310,22 +13321,33 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: i8x64, indices: u8x64) -> i8x64 { let bytes = token.cvt_to_bytes_i8x64(a); - let (table_low, table_high) = token.split_u8x64(bytes); - let (indices_low, indices_high) = token.split_u8x64(indices); - let high_table_offset = token.splat_u8x32(32); - let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); - let output_low_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_low, high_table_offset), - ); - let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); - let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); - let output_high_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_high, high_table_offset), - ); - let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); - let result_bytes = token.combine_u8x32(output_low, output_high); + let bytes01 = bytes.val.0[0]; + let bytes23 = bytes.val.0[1]; + let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); + let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); + let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); + let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); + let oob_bias = _mm256_set1_epi8(64); + let swizzle_half = |idxs: __m256i| -> __m256i { + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + }; + let result = [ + swizzle_half(indices.val.0[0]), + swizzle_half(indices.val.0[1]), + ]; + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; token.cvt_from_bytes_i8x64(result_bytes) } ); @@ -13909,22 +13931,33 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: u8x64, indices: u8x64) -> u8x64 { let bytes = token.cvt_to_bytes_u8x64(a); - let (table_low, table_high) = token.split_u8x64(bytes); - let (indices_low, indices_high) = token.split_u8x64(indices); - let high_table_offset = token.splat_u8x32(32); - let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); - let output_low_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_low, high_table_offset), - ); - let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); - let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); - let output_high_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_high, high_table_offset), - ); - let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); - let result_bytes = token.combine_u8x32(output_low, output_high); + let bytes01 = bytes.val.0[0]; + let bytes23 = bytes.val.0[1]; + let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); + let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); + let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); + let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); + let oob_bias = _mm256_set1_epi8(64); + let swizzle_half = |idxs: __m256i| -> __m256i { + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + }; + let result = [ + swizzle_half(indices.val.0[0]), + swizzle_half(indices.val.0[1]), + ]; + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; token.cvt_from_bytes_u8x64(result_bytes) } ); @@ -14611,22 +14644,33 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: i16x32, indices: u8x64) -> i16x32 { let bytes = token.cvt_to_bytes_i16x32(a); - let (table_low, table_high) = token.split_u8x64(bytes); - let (indices_low, indices_high) = token.split_u8x64(indices); - let high_table_offset = token.splat_u8x32(32); - let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); - let output_low_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_low, high_table_offset), - ); - let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); - let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); - let output_high_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_high, high_table_offset), - ); - let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); - let result_bytes = token.combine_u8x32(output_low, output_high); + let bytes01 = bytes.val.0[0]; + let bytes23 = bytes.val.0[1]; + let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); + let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); + let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); + let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); + let oob_bias = _mm256_set1_epi8(64); + let swizzle_half = |idxs: __m256i| -> __m256i { + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + }; + let result = [ + swizzle_half(indices.val.0[0]), + swizzle_half(indices.val.0[1]), + ]; + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; token.cvt_from_bytes_i16x32(result_bytes) } ); @@ -15095,22 +15139,33 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: u16x32, indices: u8x64) -> u16x32 { let bytes = token.cvt_to_bytes_u16x32(a); - let (table_low, table_high) = token.split_u8x64(bytes); - let (indices_low, indices_high) = token.split_u8x64(indices); - let high_table_offset = token.splat_u8x32(32); - let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); - let output_low_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_low, high_table_offset), - ); - let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); - let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); - let output_high_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_high, high_table_offset), - ); - let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); - let result_bytes = token.combine_u8x32(output_low, output_high); + let bytes01 = bytes.val.0[0]; + let bytes23 = bytes.val.0[1]; + let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); + let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); + let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); + let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); + let oob_bias = _mm256_set1_epi8(64); + let swizzle_half = |idxs: __m256i| -> __m256i { + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + }; + let result = [ + swizzle_half(indices.val.0[0]), + swizzle_half(indices.val.0[1]), + ]; + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; token.cvt_from_bytes_u16x32(result_bytes) } ); @@ -15741,22 +15796,33 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: i32x16, indices: u8x64) -> i32x16 { let bytes = token.cvt_to_bytes_i32x16(a); - let (table_low, table_high) = token.split_u8x64(bytes); - let (indices_low, indices_high) = token.split_u8x64(indices); - let high_table_offset = token.splat_u8x32(32); - let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); - let output_low_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_low, high_table_offset), - ); - let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); - let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); - let output_high_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_high, high_table_offset), - ); - let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); - let result_bytes = token.combine_u8x32(output_low, output_high); + let bytes01 = bytes.val.0[0]; + let bytes23 = bytes.val.0[1]; + let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); + let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); + let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); + let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); + let oob_bias = _mm256_set1_epi8(64); + let swizzle_half = |idxs: __m256i| -> __m256i { + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + }; + let result = [ + swizzle_half(indices.val.0[0]), + swizzle_half(indices.val.0[1]), + ]; + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; token.cvt_from_bytes_i32x16(result_bytes) } ); @@ -16157,22 +16223,33 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: u32x16, indices: u8x64) -> u32x16 { let bytes = token.cvt_to_bytes_u32x16(a); - let (table_low, table_high) = token.split_u8x64(bytes); - let (indices_low, indices_high) = token.split_u8x64(indices); - let high_table_offset = token.splat_u8x32(32); - let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); - let output_low_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_low, high_table_offset), - ); - let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); - let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); - let output_high_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_high, high_table_offset), - ); - let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); - let result_bytes = token.combine_u8x32(output_low, output_high); + let bytes01 = bytes.val.0[0]; + let bytes23 = bytes.val.0[1]; + let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); + let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); + let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); + let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); + let oob_bias = _mm256_set1_epi8(64); + let swizzle_half = |idxs: __m256i| -> __m256i { + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + }; + let result = [ + swizzle_half(indices.val.0[0]), + swizzle_half(indices.val.0[1]), + ]; + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; token.cvt_from_bytes_u32x16(result_bytes) } ); @@ -16737,22 +16814,33 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: f64x8, indices: u8x64) -> f64x8 { let bytes = token.cvt_to_bytes_f64x8(a); - let (table_low, table_high) = token.split_u8x64(bytes); - let (indices_low, indices_high) = token.split_u8x64(indices); - let high_table_offset = token.splat_u8x32(32); - let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); - let output_low_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_low, high_table_offset), - ); - let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); - let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); - let output_high_from_high = token.swizzle_dyn_precise_u8x32( - table_high, - token.sub_u8x32(indices_high, high_table_offset), - ); - let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); - let result_bytes = token.combine_u8x32(output_low, output_high); + let bytes01 = bytes.val.0[0]; + let bytes23 = bytes.val.0[1]; + let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); + let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); + let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); + let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); + let oob_bias = _mm256_set1_epi8(64); + let swizzle_half = |idxs: __m256i| -> __m256i { + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + }; + let result = [ + swizzle_half(indices.val.0[0]), + swizzle_half(indices.val.0[1]), + ]; + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; token.cvt_from_bytes_f64x8(result_bytes) } ); diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 4ec5ed5d..a2bef721 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2779,9 +2779,7 @@ impl X86 { let result = _mm_shuffle_epi8(#token.#to_bytes(a).val.0, zeroing_indices); let result_bytes = #bytes { val: #wrapper(result), simd: #token }; }, - (Self::Sse4_2, 256) | (Self::Avx2, 512) => { - Self::recursive_swizzle_dyn_precise_body(vec_ty, token) - } + (Self::Sse4_2, 256) => Self::recursive_swizzle_dyn_precise_body(vec_ty, token), (Self::Avx2, 256) => quote! { // carefully tuned implementation reused from std::simd: // https://github.com/rust-lang/portable-simd/blob/7d497cca160ae6062acc1a2db838667f83c0b58e/crates/core_simd/src/swizzle_dyn.rs#L205-L224 @@ -2804,6 +2802,60 @@ impl X86 { .simd_lt(#bytes::splat(#token, 16)) .select(lo_shuf, result_bytes); }, + (Self::Avx2, 512) => { + // llvm-mca measurements: + // On Haswell this is ~20% slower than the generic splt-combine fallback SSE4.2 uses, + // due to a higher shuffle port pressure. On Skylake this is a tie (no difference vs generic). + // On recent client Intel (Tiger Lake, Rocket Lake) this is ~35% faster. + // On Zen 1-3 this is ~50% faster. + let bytes_ty = vec_ty.bytes_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + quote! { + let bytes = #token.#to_bytes(a); + let bytes01 = bytes.val.0[0]; + let bytes23 = bytes.val.0[1]; + + // Broadcast each 16-byte source quarter once, then reuse the tables for + // both 32-byte output halves. + let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); + let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); + let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); + let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); + + let oob_bias = _mm256_set1_epi8(64); + let swizzle_half = |idxs: __m256i| -> __m256i { + // For in-range indices, adding 64 preserves bits 0..=5 and keeps + // the sign bit clear. Out-of-range indices get a set sign bit, so + // VPSHUFB supplies the required zeroing. + let control = _mm256_adds_epu8(idxs, oob_bias); + + // Shift bit 4 or 5 of each byte into that byte's sign bit for + // VPBLENDVB selection. These bits do not cross byte boundaries. + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + }; + + let result = [ + swizzle_half(indices.val.0[0]), + swizzle_half(indices.val.0[1]), + ]; + let result_bytes = #bytes { + val: #wrapper(result), + simd: #token, + }; + } + } (Self::Avx512, 128 | 256 | 512) => { let cmp = intrinsic_ident("cmp", "epu8_mask", vec_ty.n_bits()); let maskz_permute = From e0379042f11fc9f68621de78c2f1a978c1ddbee8 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 03:23:42 +0100 Subject: [PATCH 05/23] Apply the or trick to the dedicated AVX2 512-bit implementation to see what happens --- fearless_simd/src/generated/avx2.rs | 256 ++++++++++++++++++---------- fearless_simd_gen/src/mk_x86.rs | 53 +++--- 2 files changed, 197 insertions(+), 112 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index c39d9d78..5d566436 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -12549,18 +12549,28 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); + let oob_bias = _mm256_set1_epi8(0x70); + let offset_q1 = _mm256_set1_epi8(16); + let offset_q2 = _mm256_set1_epi8(32); + let offset_q3 = _mm256_set1_epi8(48); let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); + let from_q1 = _mm256_shuffle_epi8( + q1, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), + ); + let from_q2 = _mm256_shuffle_epi8( + q2, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), + ); + let from_q3 = _mm256_shuffle_epi8( + q3, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), + ); + _mm256_or_si256( + _mm256_or_si256(from_q0, from_q1), + _mm256_or_si256(from_q2, from_q3), + ) }; let result = [ swizzle_half(indices.val.0[0]), @@ -13327,18 +13337,28 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); + let oob_bias = _mm256_set1_epi8(0x70); + let offset_q1 = _mm256_set1_epi8(16); + let offset_q2 = _mm256_set1_epi8(32); + let offset_q3 = _mm256_set1_epi8(48); let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); + let from_q1 = _mm256_shuffle_epi8( + q1, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), + ); + let from_q2 = _mm256_shuffle_epi8( + q2, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), + ); + let from_q3 = _mm256_shuffle_epi8( + q3, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), + ); + _mm256_or_si256( + _mm256_or_si256(from_q0, from_q1), + _mm256_or_si256(from_q2, from_q3), + ) }; let result = [ swizzle_half(indices.val.0[0]), @@ -13937,18 +13957,28 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); + let oob_bias = _mm256_set1_epi8(0x70); + let offset_q1 = _mm256_set1_epi8(16); + let offset_q2 = _mm256_set1_epi8(32); + let offset_q3 = _mm256_set1_epi8(48); let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); + let from_q1 = _mm256_shuffle_epi8( + q1, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), + ); + let from_q2 = _mm256_shuffle_epi8( + q2, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), + ); + let from_q3 = _mm256_shuffle_epi8( + q3, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), + ); + _mm256_or_si256( + _mm256_or_si256(from_q0, from_q1), + _mm256_or_si256(from_q2, from_q3), + ) }; let result = [ swizzle_half(indices.val.0[0]), @@ -14650,18 +14680,28 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); + let oob_bias = _mm256_set1_epi8(0x70); + let offset_q1 = _mm256_set1_epi8(16); + let offset_q2 = _mm256_set1_epi8(32); + let offset_q3 = _mm256_set1_epi8(48); let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); + let from_q1 = _mm256_shuffle_epi8( + q1, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), + ); + let from_q2 = _mm256_shuffle_epi8( + q2, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), + ); + let from_q3 = _mm256_shuffle_epi8( + q3, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), + ); + _mm256_or_si256( + _mm256_or_si256(from_q0, from_q1), + _mm256_or_si256(from_q2, from_q3), + ) }; let result = [ swizzle_half(indices.val.0[0]), @@ -15145,18 +15185,28 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); + let oob_bias = _mm256_set1_epi8(0x70); + let offset_q1 = _mm256_set1_epi8(16); + let offset_q2 = _mm256_set1_epi8(32); + let offset_q3 = _mm256_set1_epi8(48); let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); + let from_q1 = _mm256_shuffle_epi8( + q1, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), + ); + let from_q2 = _mm256_shuffle_epi8( + q2, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), + ); + let from_q3 = _mm256_shuffle_epi8( + q3, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), + ); + _mm256_or_si256( + _mm256_or_si256(from_q0, from_q1), + _mm256_or_si256(from_q2, from_q3), + ) }; let result = [ swizzle_half(indices.val.0[0]), @@ -15802,18 +15852,28 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); + let oob_bias = _mm256_set1_epi8(0x70); + let offset_q1 = _mm256_set1_epi8(16); + let offset_q2 = _mm256_set1_epi8(32); + let offset_q3 = _mm256_set1_epi8(48); let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); + let from_q1 = _mm256_shuffle_epi8( + q1, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), + ); + let from_q2 = _mm256_shuffle_epi8( + q2, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), + ); + let from_q3 = _mm256_shuffle_epi8( + q3, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), + ); + _mm256_or_si256( + _mm256_or_si256(from_q0, from_q1), + _mm256_or_si256(from_q2, from_q3), + ) }; let result = [ swizzle_half(indices.val.0[0]), @@ -16229,18 +16289,28 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); + let oob_bias = _mm256_set1_epi8(0x70); + let offset_q1 = _mm256_set1_epi8(16); + let offset_q2 = _mm256_set1_epi8(32); + let offset_q3 = _mm256_set1_epi8(48); let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); + let from_q1 = _mm256_shuffle_epi8( + q1, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), + ); + let from_q2 = _mm256_shuffle_epi8( + q2, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), + ); + let from_q3 = _mm256_shuffle_epi8( + q3, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), + ); + _mm256_or_si256( + _mm256_or_si256(from_q0, from_q1), + _mm256_or_si256(from_q2, from_q3), + ) }; let result = [ swizzle_half(indices.val.0[0]), @@ -16820,18 +16890,28 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); + let oob_bias = _mm256_set1_epi8(0x70); + let offset_q1 = _mm256_set1_epi8(16); + let offset_q2 = _mm256_set1_epi8(32); + let offset_q3 = _mm256_set1_epi8(48); let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); + let from_q1 = _mm256_shuffle_epi8( + q1, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), + ); + let from_q2 = _mm256_shuffle_epi8( + q2, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), + ); + let from_q3 = _mm256_shuffle_epi8( + q3, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), + ); + _mm256_or_si256( + _mm256_or_si256(from_q0, from_q1), + _mm256_or_si256(from_q2, from_q3), + ) }; let result = [ swizzle_half(indices.val.0[0]), diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index a2bef721..46c7952c 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2803,11 +2803,6 @@ impl X86 { .select(lo_shuf, result_bytes); }, (Self::Avx2, 512) => { - // llvm-mca measurements: - // On Haswell this is ~20% slower than the generic splt-combine fallback SSE4.2 uses, - // due to a higher shuffle port pressure. On Skylake this is a tie (no difference vs generic). - // On recent client Intel (Tiger Lake, Rocket Lake) this is ~35% faster. - // On Zen 1-3 this is ~50% faster. let bytes_ty = vec_ty.bytes_ty(); let bytes = bytes_ty.rust(); let wrapper = bytes_ty.aligned_wrapper(); @@ -2824,26 +2819,36 @@ impl X86 { let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); + let oob_bias = _mm256_set1_epi8(0x70); + let offset_q1 = _mm256_set1_epi8(16); + let offset_q2 = _mm256_set1_epi8(32); + let offset_q3 = _mm256_set1_epi8(48); let swizzle_half = |idxs: __m256i| -> __m256i { - // For in-range indices, adding 64 preserves bits 0..=5 and keeps - // the sign bit clear. Out-of-range indices get a set sign bit, so - // VPSHUFB supplies the required zeroing. - let control = _mm256_adds_epu8(idxs, oob_bias); - - // Shift bit 4 or 5 of each byte into that byte's sign bit for - // VPBLENDVB selection. These bits do not cross byte boundaries. - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) + // Make each quarter lookup self-zeroing. For the matching 16-byte + // quarter, `idx - base` is 0..15, so adding 0x70 gives 0x70..0x7f: + // the high bit is clear and VPSHUFB uses the low nibble as the index. + // For every other quarter, wrapping subtraction or an index above the + // quarter range leaves a value >= 16; saturating +0x70 sets the high + // bit, so VPSHUFB returns zero. Exactly one shuffled value can be + // nonzero, which makes bitwise OR a valid combine. + let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); + let from_q1 = _mm256_shuffle_epi8( + q1, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), + ); + let from_q2 = _mm256_shuffle_epi8( + q2, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), + ); + let from_q3 = _mm256_shuffle_epi8( + q3, + _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), + ); + + _mm256_or_si256( + _mm256_or_si256(from_q0, from_q1), + _mm256_or_si256(from_q2, from_q3), + ) }; let result = [ From 6fd5d86ea7ecfb752bc0b7a54efa97ad5e610d4a Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 03:24:40 +0100 Subject: [PATCH 06/23] Revert "Apply the or trick to the dedicated AVX2 512-bit implementation to see what happens" This reverts commit de4a2c7b4a4937a2018a21efeed06089f4b47b16. --- fearless_simd/src/generated/avx2.rs | 256 ++++++++++------------------ fearless_simd_gen/src/mk_x86.rs | 53 +++--- 2 files changed, 112 insertions(+), 197 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 5d566436..c39d9d78 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -12549,28 +12549,18 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(0x70); - let offset_q1 = _mm256_set1_epi8(16); - let offset_q2 = _mm256_set1_epi8(32); - let offset_q3 = _mm256_set1_epi8(48); + let oob_bias = _mm256_set1_epi8(64); let swizzle_half = |idxs: __m256i| -> __m256i { - let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); - let from_q1 = _mm256_shuffle_epi8( - q1, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), - ); - let from_q2 = _mm256_shuffle_epi8( - q2, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), - ); - let from_q3 = _mm256_shuffle_epi8( - q3, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), - ); - _mm256_or_si256( - _mm256_or_si256(from_q0, from_q1), - _mm256_or_si256(from_q2, from_q3), - ) + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) }; let result = [ swizzle_half(indices.val.0[0]), @@ -13337,28 +13327,18 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(0x70); - let offset_q1 = _mm256_set1_epi8(16); - let offset_q2 = _mm256_set1_epi8(32); - let offset_q3 = _mm256_set1_epi8(48); + let oob_bias = _mm256_set1_epi8(64); let swizzle_half = |idxs: __m256i| -> __m256i { - let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); - let from_q1 = _mm256_shuffle_epi8( - q1, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), - ); - let from_q2 = _mm256_shuffle_epi8( - q2, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), - ); - let from_q3 = _mm256_shuffle_epi8( - q3, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), - ); - _mm256_or_si256( - _mm256_or_si256(from_q0, from_q1), - _mm256_or_si256(from_q2, from_q3), - ) + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) }; let result = [ swizzle_half(indices.val.0[0]), @@ -13957,28 +13937,18 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(0x70); - let offset_q1 = _mm256_set1_epi8(16); - let offset_q2 = _mm256_set1_epi8(32); - let offset_q3 = _mm256_set1_epi8(48); + let oob_bias = _mm256_set1_epi8(64); let swizzle_half = |idxs: __m256i| -> __m256i { - let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); - let from_q1 = _mm256_shuffle_epi8( - q1, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), - ); - let from_q2 = _mm256_shuffle_epi8( - q2, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), - ); - let from_q3 = _mm256_shuffle_epi8( - q3, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), - ); - _mm256_or_si256( - _mm256_or_si256(from_q0, from_q1), - _mm256_or_si256(from_q2, from_q3), - ) + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) }; let result = [ swizzle_half(indices.val.0[0]), @@ -14680,28 +14650,18 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(0x70); - let offset_q1 = _mm256_set1_epi8(16); - let offset_q2 = _mm256_set1_epi8(32); - let offset_q3 = _mm256_set1_epi8(48); + let oob_bias = _mm256_set1_epi8(64); let swizzle_half = |idxs: __m256i| -> __m256i { - let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); - let from_q1 = _mm256_shuffle_epi8( - q1, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), - ); - let from_q2 = _mm256_shuffle_epi8( - q2, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), - ); - let from_q3 = _mm256_shuffle_epi8( - q3, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), - ); - _mm256_or_si256( - _mm256_or_si256(from_q0, from_q1), - _mm256_or_si256(from_q2, from_q3), - ) + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) }; let result = [ swizzle_half(indices.val.0[0]), @@ -15185,28 +15145,18 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(0x70); - let offset_q1 = _mm256_set1_epi8(16); - let offset_q2 = _mm256_set1_epi8(32); - let offset_q3 = _mm256_set1_epi8(48); + let oob_bias = _mm256_set1_epi8(64); let swizzle_half = |idxs: __m256i| -> __m256i { - let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); - let from_q1 = _mm256_shuffle_epi8( - q1, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), - ); - let from_q2 = _mm256_shuffle_epi8( - q2, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), - ); - let from_q3 = _mm256_shuffle_epi8( - q3, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), - ); - _mm256_or_si256( - _mm256_or_si256(from_q0, from_q1), - _mm256_or_si256(from_q2, from_q3), - ) + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) }; let result = [ swizzle_half(indices.val.0[0]), @@ -15852,28 +15802,18 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(0x70); - let offset_q1 = _mm256_set1_epi8(16); - let offset_q2 = _mm256_set1_epi8(32); - let offset_q3 = _mm256_set1_epi8(48); + let oob_bias = _mm256_set1_epi8(64); let swizzle_half = |idxs: __m256i| -> __m256i { - let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); - let from_q1 = _mm256_shuffle_epi8( - q1, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), - ); - let from_q2 = _mm256_shuffle_epi8( - q2, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), - ); - let from_q3 = _mm256_shuffle_epi8( - q3, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), - ); - _mm256_or_si256( - _mm256_or_si256(from_q0, from_q1), - _mm256_or_si256(from_q2, from_q3), - ) + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) }; let result = [ swizzle_half(indices.val.0[0]), @@ -16289,28 +16229,18 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(0x70); - let offset_q1 = _mm256_set1_epi8(16); - let offset_q2 = _mm256_set1_epi8(32); - let offset_q3 = _mm256_set1_epi8(48); + let oob_bias = _mm256_set1_epi8(64); let swizzle_half = |idxs: __m256i| -> __m256i { - let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); - let from_q1 = _mm256_shuffle_epi8( - q1, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), - ); - let from_q2 = _mm256_shuffle_epi8( - q2, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), - ); - let from_q3 = _mm256_shuffle_epi8( - q3, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), - ); - _mm256_or_si256( - _mm256_or_si256(from_q0, from_q1), - _mm256_or_si256(from_q2, from_q3), - ) + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) }; let result = [ swizzle_half(indices.val.0[0]), @@ -16890,28 +16820,18 @@ impl Simd for Avx2 { let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(0x70); - let offset_q1 = _mm256_set1_epi8(16); - let offset_q2 = _mm256_set1_epi8(32); - let offset_q3 = _mm256_set1_epi8(48); + let oob_bias = _mm256_set1_epi8(64); let swizzle_half = |idxs: __m256i| -> __m256i { - let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); - let from_q1 = _mm256_shuffle_epi8( - q1, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), - ); - let from_q2 = _mm256_shuffle_epi8( - q2, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), - ); - let from_q3 = _mm256_shuffle_epi8( - q3, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), - ); - _mm256_or_si256( - _mm256_or_si256(from_q0, from_q1), - _mm256_or_si256(from_q2, from_q3), - ) + let control = _mm256_adds_epu8(idxs, oob_bias); + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) }; let result = [ swizzle_half(indices.val.0[0]), diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 46c7952c..a2bef721 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2803,6 +2803,11 @@ impl X86 { .select(lo_shuf, result_bytes); }, (Self::Avx2, 512) => { + // llvm-mca measurements: + // On Haswell this is ~20% slower than the generic splt-combine fallback SSE4.2 uses, + // due to a higher shuffle port pressure. On Skylake this is a tie (no difference vs generic). + // On recent client Intel (Tiger Lake, Rocket Lake) this is ~35% faster. + // On Zen 1-3 this is ~50% faster. let bytes_ty = vec_ty.bytes_ty(); let bytes = bytes_ty.rust(); let wrapper = bytes_ty.aligned_wrapper(); @@ -2819,36 +2824,26 @@ impl X86 { let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(0x70); - let offset_q1 = _mm256_set1_epi8(16); - let offset_q2 = _mm256_set1_epi8(32); - let offset_q3 = _mm256_set1_epi8(48); + let oob_bias = _mm256_set1_epi8(64); let swizzle_half = |idxs: __m256i| -> __m256i { - // Make each quarter lookup self-zeroing. For the matching 16-byte - // quarter, `idx - base` is 0..15, so adding 0x70 gives 0x70..0x7f: - // the high bit is clear and VPSHUFB uses the low nibble as the index. - // For every other quarter, wrapping subtraction or an index above the - // quarter range leaves a value >= 16; saturating +0x70 sets the high - // bit, so VPSHUFB returns zero. Exactly one shuffled value can be - // nonzero, which makes bitwise OR a valid combine. - let from_q0 = _mm256_shuffle_epi8(q0, _mm256_adds_epu8(idxs, oob_bias)); - let from_q1 = _mm256_shuffle_epi8( - q1, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q1), oob_bias), - ); - let from_q2 = _mm256_shuffle_epi8( - q2, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q2), oob_bias), - ); - let from_q3 = _mm256_shuffle_epi8( - q3, - _mm256_adds_epu8(_mm256_sub_epi8(idxs, offset_q3), oob_bias), - ); - - _mm256_or_si256( - _mm256_or_si256(from_q0, from_q1), - _mm256_or_si256(from_q2, from_q3), - ) + // For in-range indices, adding 64 preserves bits 0..=5 and keeps + // the sign bit clear. Out-of-range indices get a set sign bit, so + // VPSHUFB supplies the required zeroing. + let control = _mm256_adds_epu8(idxs, oob_bias); + + // Shift bit 4 or 5 of each byte into that byte's sign bit for + // VPBLENDVB selection. These bits do not cross byte boundaries. + let select_q1_q3 = _mm256_slli_epi16::<3>(control); + let select_q2_q3 = _mm256_slli_epi16::<2>(control); + + let from_q0 = _mm256_shuffle_epi8(q0, control); + let from_q1 = _mm256_shuffle_epi8(q1, control); + let from_q2 = _mm256_shuffle_epi8(q2, control); + let from_q3 = _mm256_shuffle_epi8(q3, control); + + let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); + let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); + _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) }; let result = [ From c79bf96c29c338229b6a09b4c16aa3956a3b8713 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 03:34:50 +0100 Subject: [PATCH 07/23] More detailed comment on the AVX2-optimized shuffle --- fearless_simd_gen/src/mk_x86.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index a2bef721..7bcf7518 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2803,11 +2803,24 @@ impl X86 { .select(lo_shuf, result_bytes); }, (Self::Avx2, 512) => { - // llvm-mca measurements: - // On Haswell this is ~20% slower than the generic splt-combine fallback SSE4.2 uses, - // due to a higher shuffle port pressure. On Skylake this is a tie (no difference vs generic). - // On recent client Intel (Tiger Lake, Rocket Lake) this is ~35% faster. - // On Zen 1-3 this is ~50% faster. + // llvm-mca measurements (troughput): + // + // On Haswell this is 24 cycles, as opposed to 20 cycles for split-combine that SSE4.2 uses. + // Skylake doesn't care (12 cycles either way). + // Recent Intel (Icelake-client, Tigerlake, Rocketlake) take 6.5 cycles vs 8.8 for generic + // AMD Zen 1-2 takes 7.3 vs 11.8 for generic, Zen 3 takes 6.5 vs 8.5 for generic. + // Uses 11 registers instead of 14 registers in generic split-combine. + // + // It's also possible to apply the bitwise or trick from the generic impl. + // This can be seen in commit history, just before this comment was written. + // That helps haswell a lot (12 cycles) by reducing shuffle port pressure, + // but regresses everything else by using an extra register and by putting + // more pressure on the register file (llvm-mca --register-file-stats). + // Zen1 also drops all the way back to 11.8 cycles and nearly hits the register file limit + // from this one operation alone. + // + // Since registers are the most scarce resource on AVX2, + // we're not going to hamstring all CPUs from the past decade to make Haswell faster. let bytes_ty = vec_ty.bytes_ty(); let bytes = bytes_ty.rust(); let wrapper = bytes_ty.aligned_wrapper(); From e1e605d7ef1a2de6adddf0214b89af9a17e0b242 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 03:49:50 +0100 Subject: [PATCH 08/23] Reuse the recursive decomposition for WASM 256-bit case --- fearless_simd/src/generated/wasm.rs | 1160 +++------------------------ fearless_simd_gen/src/generic.rs | 49 ++ fearless_simd_gen/src/mk_wasm.rs | 44 +- fearless_simd_gen/src/mk_x86.rs | 49 +- 4 files changed, 184 insertions(+), 1118 deletions(-) diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index a78202d6..3b0642c4 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -3741,138 +3741,19 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { let bytes = self.cvt_to_bytes_f32x8(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_f32x8(result) + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + self.cvt_from_bytes_f32x8(result_bytes) } #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { @@ -4420,138 +4301,19 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { let bytes = self.cvt_to_bytes_i8x32(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_i8x32(result) + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + self.cvt_from_bytes_i8x32(result_bytes) } #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { @@ -5006,138 +4768,19 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { let bytes = self.cvt_to_bytes_u8x32(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_u8x32(result) + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + self.cvt_from_bytes_u8x32(result_bytes) } #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { @@ -5651,138 +5294,19 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { let bytes = self.cvt_to_bytes_i16x16(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_i16x16(result) + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + self.cvt_from_bytes_i16x16(result_bytes) } #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { @@ -6177,138 +5701,19 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { let bytes = self.cvt_to_bytes_u16x16(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_u16x16(result) + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + self.cvt_from_bytes_u16x16(result_bytes) } #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { @@ -6795,138 +6200,19 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { let bytes = self.cvt_to_bytes_i32x8(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_i32x8(result) + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + self.cvt_from_bytes_i32x8(result_bytes) } #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { @@ -7290,138 +6576,19 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { let bytes = self.cvt_to_bytes_u32x8(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_u32x8(result) + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + self.cvt_from_bytes_u32x8(result_bytes) } #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { @@ -7880,138 +7047,19 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { let bytes = self.cvt_to_bytes_f64x4(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); - self.cvt_from_bytes_f64x4(result) + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + self.cvt_from_bytes_f64x4(result_bytes) } #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index 5e579a4e..31949cad 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -18,6 +18,55 @@ pub(crate) fn fallback_method(op: Op, vec_ty: &VecType) -> TokenStream { crate::mk_fallback::Fallback.make_method(op, vec_ty) } +pub(crate) fn recursive_swizzle_dyn_precise_body( + vec_ty: &VecType, + token: &T, +) -> TokenStream { + // We can take advantage of the "precise" property (out-of-range is 0) + // to assemble a double-vector-width arbitrary shuffle. + // The trick is to split the input into two, then run for each half of output indices J: + // ``` + // from_low = swizzle_H(low_table, J); + // from_high = swizzle_H(high_table, J.wrapping_sub(H)); + // result = from_low | from_high; + // ``` + // Since each element is out-of-range for at least one half, + // the final combine is very cheap: a bitwise or. + let bytes_ty = vec_ty.bytes_ty(); + let half_bytes_ty = VecType::new(ScalarType::Unsigned, 8, bytes_ty.len / 2); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let split_bytes = generic_op_name("split", &bytes_ty); + let combine_half_bytes = generic_op_name("combine", &half_bytes_ty); + let swizzle_half = generic_op_name("swizzle_dyn_precise", &half_bytes_ty); + let splat_half = generic_op_name("splat", &half_bytes_ty); + let sub_half = generic_op_name("sub", &half_bytes_ty); + let or_half = generic_op_name("or", &half_bytes_ty); + let half_len = Literal::u8_unsuffixed((bytes_ty.len / 2) as u8); + + quote! { + let bytes = #token.#to_bytes(a); + let (table_low, table_high) = #token.#split_bytes(bytes); + let (indices_low, indices_high) = #token.#split_bytes(indices); + let high_table_offset = #token.#splat_half(#half_len); + + let output_low_from_low = #token.#swizzle_half(table_low, indices_low); + let output_low_from_high = #token.#swizzle_half( + table_high, + #token.#sub_half(indices_low, high_table_offset), + ); + let output_low = #token.#or_half(output_low_from_low, output_low_from_high); + + let output_high_from_low = #token.#swizzle_half(table_low, indices_high); + let output_high_from_high = #token.#swizzle_half( + table_high, + #token.#sub_half(indices_high, high_table_offset), + ); + let output_high = #token.#or_half(output_high_from_low, output_high_from_high); + + let result_bytes = #token.#combine_half_bytes(output_low, output_high); + } +} + /// For backends that store masks as all-zero/all-one integer lanes, convert the public /// `bool` mask splat argument into the backend's lane representation. pub(crate) fn integer_lane_mask_splat_arg(vec_ty: &VecType) -> TokenStream { diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index 29dc7e9a..e61ff5a4 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -8,7 +8,7 @@ use crate::arch::wasm::{arch_prefix, v128_intrinsic}; use crate::generic::{ fallback_method, generic_as_array, generic_block_combine, generic_block_split, generic_from_array, generic_from_bytes, generic_mask_set, generic_op_name, generic_store_array, - generic_to_bytes, integer_lane_mask_splat_arg, + generic_to_bytes, integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, }; use crate::level::Level; use crate::ops::{Op, Quantifier, SlideGranularity, valid_reinterpret}; @@ -517,24 +517,38 @@ impl Level for WasmSimd128 { } } } - OpSig::SwizzleDynPrecise => { - if vec_ty.n_bits() != self.native_width() { - return crate::mk_fallback::Fallback.make_method(op, vec_ty); - } + OpSig::SwizzleDynPrecise => match vec_ty.n_bits() { + 128 => { + let bytes_ty = vec_ty.bytes_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); - let bytes_ty = vec_ty.bytes_ty(); - let bytes = bytes_ty.rust(); - let wrapper = bytes_ty.aligned_wrapper(); - let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); - let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + quote! { + #method_sig { + let result = u8x16_swizzle(self.#to_bytes(a).val.0, indices.into()); + self.#from_bytes(#bytes { val: #wrapper(result), simd: self }) + } + } + } + 256 => { + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + let body = recursive_swizzle_dyn_precise_body(vec_ty, "e! { self }); - quote! { - #method_sig { - let result = u8x16_swizzle(self.#to_bytes(a).val.0, indices.into()); - self.#from_bytes(#bytes { val: #wrapper(result), simd: self }) + quote! { + #method_sig { + #body + self.#from_bytes(result_bytes) + } } } - } + // We don't use the recursive decomposition for the 512-bit case + // because register spills get way too bad. + // We can only hope that the compiler will recognize the shuffle operation + // and express it in terms of 256-bit or 512-bit vectors. + _ => crate::mk_fallback::Fallback.make_method(op, vec_ty), + }, OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 7bcf7518..29561384 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -10,6 +10,7 @@ use crate::generic::{ fallback_method, generic_as_array, generic_block_combine, generic_block_split, generic_from_array, generic_from_bytes, generic_mask_from_bitmask, generic_mask_set, generic_op_name, generic_store_array, generic_to_bytes, integer_lane_mask_splat_arg, + recursive_swizzle_dyn_precise_body, }; use crate::level::Level; use crate::ops::{Op, OpSig, Quantifier, SlideGranularity, valid_reinterpret}; @@ -2779,7 +2780,7 @@ impl X86 { let result = _mm_shuffle_epi8(#token.#to_bytes(a).val.0, zeroing_indices); let result_bytes = #bytes { val: #wrapper(result), simd: #token }; }, - (Self::Sse4_2, 256) => Self::recursive_swizzle_dyn_precise_body(vec_ty, token), + (Self::Sse4_2, 256) => recursive_swizzle_dyn_precise_body(vec_ty, token), (Self::Avx2, 256) => quote! { // carefully tuned implementation reused from std::simd: // https://github.com/rust-lang/portable-simd/blob/7d497cca160ae6062acc1a2db838667f83c0b58e/crates/core_simd/src/swizzle_dyn.rs#L205-L224 @@ -2893,52 +2894,6 @@ impl X86 { }) } - fn recursive_swizzle_dyn_precise_body(vec_ty: &VecType, token: &Ident) -> TokenStream { - // We can take advantage of the "precise" property (out-of-range is 0) - // to assemble a double-vector-width arbitrary shuffle. - // The trick is to split the input into two, then run for each half of output indices J: - // ``` - // from_low = swizzle_H(low_table, J); - // from_high = swizzle_H(high_table, J.wrapping_sub(H)); - // result = from_low | from_high; - // ``` - // Since each element is out-of-range for at least one half, - // the final combine is very cheap: a bitwise or. - let bytes_ty = vec_ty.bytes_ty(); - let half_bytes_ty = VecType::new(ScalarType::Unsigned, 8, bytes_ty.len / 2); - let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); - let split_bytes = generic_op_name("split", &bytes_ty); - let combine_half_bytes = generic_op_name("combine", &half_bytes_ty); - let swizzle_half = generic_op_name("swizzle_dyn_precise", &half_bytes_ty); - let splat_half = generic_op_name("splat", &half_bytes_ty); - let sub_half = generic_op_name("sub", &half_bytes_ty); - let or_half = generic_op_name("or", &half_bytes_ty); - let half_len = Literal::u8_unsuffixed((bytes_ty.len / 2) as u8); - - quote! { - let bytes = #token.#to_bytes(a); - let (table_low, table_high) = #token.#split_bytes(bytes); - let (indices_low, indices_high) = #token.#split_bytes(indices); - let high_table_offset = #token.#splat_half(#half_len); - - let output_low_from_low = #token.#swizzle_half(table_low, indices_low); - let output_low_from_high = #token.#swizzle_half( - table_high, - #token.#sub_half(indices_low, high_table_offset), - ); - let output_low = #token.#or_half(output_low_from_low, output_low_from_high); - - let output_high_from_low = #token.#swizzle_half(table_low, indices_high); - let output_high_from_high = #token.#swizzle_half( - table_high, - #token.#sub_half(indices_high, high_table_offset), - ); - let output_high = #token.#or_half(output_high_from_low, output_high_from_high); - - let result_bytes = #token.#combine_half_bytes(output_low, output_high); - } - } - pub(crate) fn handle_cvt( &self, op: Op, From 1ff3d165a269416832d66241a7fafe390d019560 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 04:20:43 +0100 Subject: [PATCH 09/23] Add random testing for swizzle_dyn_precise correctness, since it's precise and we can assert it has to always match the scalar fallback --- Cargo.lock | 7 +++ fearless_simd_tests/Cargo.toml | 1 + .../lm_generated/swizzle_dyn_precise.rs | 45 +++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 47f3baa2..d5513942 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,6 +104,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "fearless_simd" version = "0.6.0" @@ -134,6 +140,7 @@ dependencies = [ name = "fearless_simd_tests" version = "0.0.0" dependencies = [ + "fastrand", "fearless_simd", "fearless_simd_dev_macros", ] diff --git a/fearless_simd_tests/Cargo.toml b/fearless_simd_tests/Cargo.toml index 16fcf8f8..ef64bb00 100644 --- a/fearless_simd_tests/Cargo.toml +++ b/fearless_simd_tests/Cargo.toml @@ -22,3 +22,4 @@ workspace = true [dependencies] fearless_simd = { workspace = true, features = ["std"] } fearless_simd_dev_macros = { workspace = true } +fastrand = "2.4.1" \ No newline at end of file diff --git a/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs b/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs index 5dd8933d..56e4ec51 100644 --- a/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs +++ b/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs @@ -94,3 +94,48 @@ fn swizzle_dyn_precise_generic_indices(simd: S) { assert_eq!(*result, expected); } + +#[simd_test] +fn swizzle_dyn_precise_random_u8_widths(simd: S) { + let mut rng = fastrand::Rng::with_seed(0x5eed_5eed_cafe_f00d); + + for iteration in 0..100_000 { + let mut bytes = [0u8; 64]; + let mut indices = [0u8; 64]; + rng.fill(&mut bytes); + rng.fill(&mut indices); + + let bytes16 = &bytes[..16]; + let indices16 = &indices[..16]; + let expected16 = expected_swizzle_precise_from_slice::<16>(bytes16, indices16); + let value16 = u8x16::from_slice(simd, bytes16); + let index_vec16 = u8x16::from_slice(simd, indices16); + let result16 = value16.swizzle_dyn_precise(index_vec16); + assert_eq!(*result16, expected16, "iteration {iteration}, width 16"); + + let bytes32 = &bytes[..32]; + let indices32 = &indices[..32]; + let expected32 = expected_swizzle_precise_from_slice::<32>(bytes32, indices32); + let value32 = u8x32::from_slice(simd, bytes32); + let index_vec32 = u8x32::from_slice(simd, indices32); + let result32 = value32.swizzle_dyn_precise(index_vec32); + assert_eq!(*result32, expected32, "iteration {iteration}, width 32"); + + let bytes64 = &bytes[..64]; + let indices64 = &indices[..64]; + let expected64 = expected_swizzle_precise_from_slice::<64>(bytes64, indices64); + let value64 = u8x64::from_slice(simd, bytes64); + let index_vec64 = u8x64::from_slice(simd, indices64); + let result64 = value64.swizzle_dyn_precise(index_vec64); + assert_eq!(*result64, expected64, "iteration {iteration}, width 64"); + } +} + +fn expected_swizzle_precise_from_slice(bytes: &[u8], indices: &[u8]) -> [u8; N] { + assert_eq!(bytes.len(), N); + assert_eq!(indices.len(), N); + core::array::from_fn(|i| { + let index = indices[i] as usize; + bytes.get(index).copied().unwrap_or(0) + }) +} From 1d9536644045f2f5b9139bb072ccfaef162f1192 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 04:20:52 +0100 Subject: [PATCH 10/23] Revert "Add random testing for swizzle_dyn_precise correctness, since it's precise and we can assert it has to always match the scalar fallback" to avoid adding extra dependencies This reverts commit 4e8a56accb23ab4f0b96db89147b3c1fadc8f07e. --- Cargo.lock | 7 --- fearless_simd_tests/Cargo.toml | 1 - .../lm_generated/swizzle_dyn_precise.rs | 45 ------------------- 3 files changed, 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5513942..47f3baa2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,12 +104,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" -[[package]] -name = "fastrand" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" - [[package]] name = "fearless_simd" version = "0.6.0" @@ -140,7 +134,6 @@ dependencies = [ name = "fearless_simd_tests" version = "0.0.0" dependencies = [ - "fastrand", "fearless_simd", "fearless_simd_dev_macros", ] diff --git a/fearless_simd_tests/Cargo.toml b/fearless_simd_tests/Cargo.toml index ef64bb00..16fcf8f8 100644 --- a/fearless_simd_tests/Cargo.toml +++ b/fearless_simd_tests/Cargo.toml @@ -22,4 +22,3 @@ workspace = true [dependencies] fearless_simd = { workspace = true, features = ["std"] } fearless_simd_dev_macros = { workspace = true } -fastrand = "2.4.1" \ No newline at end of file diff --git a/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs b/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs index 56e4ec51..5dd8933d 100644 --- a/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs +++ b/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs @@ -94,48 +94,3 @@ fn swizzle_dyn_precise_generic_indices(simd: S) { assert_eq!(*result, expected); } - -#[simd_test] -fn swizzle_dyn_precise_random_u8_widths(simd: S) { - let mut rng = fastrand::Rng::with_seed(0x5eed_5eed_cafe_f00d); - - for iteration in 0..100_000 { - let mut bytes = [0u8; 64]; - let mut indices = [0u8; 64]; - rng.fill(&mut bytes); - rng.fill(&mut indices); - - let bytes16 = &bytes[..16]; - let indices16 = &indices[..16]; - let expected16 = expected_swizzle_precise_from_slice::<16>(bytes16, indices16); - let value16 = u8x16::from_slice(simd, bytes16); - let index_vec16 = u8x16::from_slice(simd, indices16); - let result16 = value16.swizzle_dyn_precise(index_vec16); - assert_eq!(*result16, expected16, "iteration {iteration}, width 16"); - - let bytes32 = &bytes[..32]; - let indices32 = &indices[..32]; - let expected32 = expected_swizzle_precise_from_slice::<32>(bytes32, indices32); - let value32 = u8x32::from_slice(simd, bytes32); - let index_vec32 = u8x32::from_slice(simd, indices32); - let result32 = value32.swizzle_dyn_precise(index_vec32); - assert_eq!(*result32, expected32, "iteration {iteration}, width 32"); - - let bytes64 = &bytes[..64]; - let indices64 = &indices[..64]; - let expected64 = expected_swizzle_precise_from_slice::<64>(bytes64, indices64); - let value64 = u8x64::from_slice(simd, bytes64); - let index_vec64 = u8x64::from_slice(simd, indices64); - let result64 = value64.swizzle_dyn_precise(index_vec64); - assert_eq!(*result64, expected64, "iteration {iteration}, width 64"); - } -} - -fn expected_swizzle_precise_from_slice(bytes: &[u8], indices: &[u8]) -> [u8; N] { - assert_eq!(bytes.len(), N); - assert_eq!(indices.len(), N); - core::array::from_fn(|i| { - let index = indices[i] as usize; - bytes.get(index).copied().unwrap_or(0) - }) -} From e0a32252e23b89e110403ca84b304b22391ffa64 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 04:40:53 +0100 Subject: [PATCH 11/23] Use an unwrap instead of a cast to appease clippy --- fearless_simd_gen/src/generic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index 31949cad..c737e035 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -41,7 +41,7 @@ pub(crate) fn recursive_swizzle_dyn_precise_body( let splat_half = generic_op_name("splat", &half_bytes_ty); let sub_half = generic_op_name("sub", &half_bytes_ty); let or_half = generic_op_name("or", &half_bytes_ty); - let half_len = Literal::u8_unsuffixed((bytes_ty.len / 2) as u8); + let half_len = Literal::u8_unsuffixed(u8::try_from(bytes_ty.len / 2).unwrap()); quote! { let bytes = #token.#to_bytes(a); From 710825b33aa91d4f7dca5dcdc34a5ee1b3273ad2 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 04:41:55 +0100 Subject: [PATCH 12/23] Fix typo --- fearless_simd_gen/src/mk_x86.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 29561384..dc966c91 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2804,7 +2804,7 @@ impl X86 { .select(lo_shuf, result_bytes); }, (Self::Avx2, 512) => { - // llvm-mca measurements (troughput): + // llvm-mca measurements (throughput): // // On Haswell this is 24 cycles, as opposed to 20 cycles for split-combine that SSE4.2 uses. // Skylake doesn't care (12 cycles either way). From 79de6c0c48aae27213835179c57121c59301e95b Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 19 Jul 2026 14:15:05 +0100 Subject: [PATCH 13/23] Remove 512-bit shuffle specialized for AVX2 and route through the generic split/combine since there's no difference on benchmarks on real Zen2 hardware, which was the best-case scenario for this implementation according to llvm-mca --- fearless_simd/src/generated/avx2.rs | 344 +++++++++++----------------- fearless_simd_gen/src/mk_x86.rs | 71 +----- 2 files changed, 131 insertions(+), 284 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index c39d9d78..e638dd6c 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -12543,33 +12543,22 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: f32x16, indices: u8x64) -> f32x16 { let bytes = token.cvt_to_bytes_f32x16(a); - let bytes01 = bytes.val.0[0]; - let bytes23 = bytes.val.0[1]; - let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); - let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); - let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); - let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); - let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) - }; - let result = [ - swizzle_half(indices.val.0[0]), - swizzle_half(indices.val.0[1]), - ]; - let result_bytes = u8x64 { - val: crate::support::Aligned512(result), - simd: token, - }; + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); token.cvt_from_bytes_f32x16(result_bytes) } ); @@ -13321,33 +13310,22 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: i8x64, indices: u8x64) -> i8x64 { let bytes = token.cvt_to_bytes_i8x64(a); - let bytes01 = bytes.val.0[0]; - let bytes23 = bytes.val.0[1]; - let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); - let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); - let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); - let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); - let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) - }; - let result = [ - swizzle_half(indices.val.0[0]), - swizzle_half(indices.val.0[1]), - ]; - let result_bytes = u8x64 { - val: crate::support::Aligned512(result), - simd: token, - }; + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); token.cvt_from_bytes_i8x64(result_bytes) } ); @@ -13931,33 +13909,22 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: u8x64, indices: u8x64) -> u8x64 { let bytes = token.cvt_to_bytes_u8x64(a); - let bytes01 = bytes.val.0[0]; - let bytes23 = bytes.val.0[1]; - let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); - let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); - let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); - let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); - let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) - }; - let result = [ - swizzle_half(indices.val.0[0]), - swizzle_half(indices.val.0[1]), - ]; - let result_bytes = u8x64 { - val: crate::support::Aligned512(result), - simd: token, - }; + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); token.cvt_from_bytes_u8x64(result_bytes) } ); @@ -14644,33 +14611,22 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: i16x32, indices: u8x64) -> i16x32 { let bytes = token.cvt_to_bytes_i16x32(a); - let bytes01 = bytes.val.0[0]; - let bytes23 = bytes.val.0[1]; - let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); - let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); - let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); - let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); - let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) - }; - let result = [ - swizzle_half(indices.val.0[0]), - swizzle_half(indices.val.0[1]), - ]; - let result_bytes = u8x64 { - val: crate::support::Aligned512(result), - simd: token, - }; + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); token.cvt_from_bytes_i16x32(result_bytes) } ); @@ -15139,33 +15095,22 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: u16x32, indices: u8x64) -> u16x32 { let bytes = token.cvt_to_bytes_u16x32(a); - let bytes01 = bytes.val.0[0]; - let bytes23 = bytes.val.0[1]; - let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); - let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); - let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); - let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); - let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) - }; - let result = [ - swizzle_half(indices.val.0[0]), - swizzle_half(indices.val.0[1]), - ]; - let result_bytes = u8x64 { - val: crate::support::Aligned512(result), - simd: token, - }; + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); token.cvt_from_bytes_u16x32(result_bytes) } ); @@ -15796,33 +15741,22 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: i32x16, indices: u8x64) -> i32x16 { let bytes = token.cvt_to_bytes_i32x16(a); - let bytes01 = bytes.val.0[0]; - let bytes23 = bytes.val.0[1]; - let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); - let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); - let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); - let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); - let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) - }; - let result = [ - swizzle_half(indices.val.0[0]), - swizzle_half(indices.val.0[1]), - ]; - let result_bytes = u8x64 { - val: crate::support::Aligned512(result), - simd: token, - }; + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); token.cvt_from_bytes_i32x16(result_bytes) } ); @@ -16223,33 +16157,22 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: u32x16, indices: u8x64) -> u32x16 { let bytes = token.cvt_to_bytes_u32x16(a); - let bytes01 = bytes.val.0[0]; - let bytes23 = bytes.val.0[1]; - let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); - let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); - let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); - let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); - let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) - }; - let result = [ - swizzle_half(indices.val.0[0]), - swizzle_half(indices.val.0[1]), - ]; - let result_bytes = u8x64 { - val: crate::support::Aligned512(result), - simd: token, - }; + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); token.cvt_from_bytes_u32x16(result_bytes) } ); @@ -16814,33 +16737,22 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: f64x8, indices: u8x64) -> f64x8 { let bytes = token.cvt_to_bytes_f64x8(a); - let bytes01 = bytes.val.0[0]; - let bytes23 = bytes.val.0[1]; - let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); - let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); - let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); - let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - let oob_bias = _mm256_set1_epi8(64); - let swizzle_half = |idxs: __m256i| -> __m256i { - let control = _mm256_adds_epu8(idxs, oob_bias); - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) - }; - let result = [ - swizzle_half(indices.val.0[0]), - swizzle_half(indices.val.0[1]), - ]; - let result_bytes = u8x64 { - val: crate::support::Aligned512(result), - simd: token, - }; + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); token.cvt_from_bytes_f64x8(result_bytes) } ); diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index dc966c91..9ba62f57 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2780,7 +2780,9 @@ impl X86 { let result = _mm_shuffle_epi8(#token.#to_bytes(a).val.0, zeroing_indices); let result_bytes = #bytes { val: #wrapper(result), simd: #token }; }, - (Self::Sse4_2, 256) => recursive_swizzle_dyn_precise_body(vec_ty, token), + (Self::Sse4_2, 256) | (Self::Avx2, 512) => { + recursive_swizzle_dyn_precise_body(vec_ty, token) + } (Self::Avx2, 256) => quote! { // carefully tuned implementation reused from std::simd: // https://github.com/rust-lang/portable-simd/blob/7d497cca160ae6062acc1a2db838667f83c0b58e/crates/core_simd/src/swizzle_dyn.rs#L205-L224 @@ -2803,73 +2805,6 @@ impl X86 { .simd_lt(#bytes::splat(#token, 16)) .select(lo_shuf, result_bytes); }, - (Self::Avx2, 512) => { - // llvm-mca measurements (throughput): - // - // On Haswell this is 24 cycles, as opposed to 20 cycles for split-combine that SSE4.2 uses. - // Skylake doesn't care (12 cycles either way). - // Recent Intel (Icelake-client, Tigerlake, Rocketlake) take 6.5 cycles vs 8.8 for generic - // AMD Zen 1-2 takes 7.3 vs 11.8 for generic, Zen 3 takes 6.5 vs 8.5 for generic. - // Uses 11 registers instead of 14 registers in generic split-combine. - // - // It's also possible to apply the bitwise or trick from the generic impl. - // This can be seen in commit history, just before this comment was written. - // That helps haswell a lot (12 cycles) by reducing shuffle port pressure, - // but regresses everything else by using an extra register and by putting - // more pressure on the register file (llvm-mca --register-file-stats). - // Zen1 also drops all the way back to 11.8 cycles and nearly hits the register file limit - // from this one operation alone. - // - // Since registers are the most scarce resource on AVX2, - // we're not going to hamstring all CPUs from the past decade to make Haswell faster. - let bytes_ty = vec_ty.bytes_ty(); - let bytes = bytes_ty.rust(); - let wrapper = bytes_ty.aligned_wrapper(); - let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); - quote! { - let bytes = #token.#to_bytes(a); - let bytes01 = bytes.val.0[0]; - let bytes23 = bytes.val.0[1]; - - // Broadcast each 16-byte source quarter once, then reuse the tables for - // both 32-byte output halves. - let q0 = _mm256_permute2x128_si256::<0x00>(bytes01, bytes01); - let q1 = _mm256_permute2x128_si256::<0x11>(bytes01, bytes01); - let q2 = _mm256_permute2x128_si256::<0x00>(bytes23, bytes23); - let q3 = _mm256_permute2x128_si256::<0x11>(bytes23, bytes23); - - let oob_bias = _mm256_set1_epi8(64); - let swizzle_half = |idxs: __m256i| -> __m256i { - // For in-range indices, adding 64 preserves bits 0..=5 and keeps - // the sign bit clear. Out-of-range indices get a set sign bit, so - // VPSHUFB supplies the required zeroing. - let control = _mm256_adds_epu8(idxs, oob_bias); - - // Shift bit 4 or 5 of each byte into that byte's sign bit for - // VPBLENDVB selection. These bits do not cross byte boundaries. - let select_q1_q3 = _mm256_slli_epi16::<3>(control); - let select_q2_q3 = _mm256_slli_epi16::<2>(control); - - let from_q0 = _mm256_shuffle_epi8(q0, control); - let from_q1 = _mm256_shuffle_epi8(q1, control); - let from_q2 = _mm256_shuffle_epi8(q2, control); - let from_q3 = _mm256_shuffle_epi8(q3, control); - - let from_q01 = _mm256_blendv_epi8(from_q0, from_q1, select_q1_q3); - let from_q23 = _mm256_blendv_epi8(from_q2, from_q3, select_q1_q3); - _mm256_blendv_epi8(from_q01, from_q23, select_q2_q3) - }; - - let result = [ - swizzle_half(indices.val.0[0]), - swizzle_half(indices.val.0[1]), - ]; - let result_bytes = #bytes { - val: #wrapper(result), - simd: #token, - }; - } - } (Self::Avx512, 128 | 256 | 512) => { let cmp = intrinsic_ident("cmp", "epu8_mask", vec_ty.n_bits()); let maskz_permute = From 8aee1131ebd542ab69595e91983bd0a138508f07 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Mon, 20 Jul 2026 16:08:49 +0100 Subject: [PATCH 14/23] Implement a custom AVX2 shuffle instead of reusing the std::simd one. llvm-mca results: Haswell, Broadwell and Skylake effectively unchanged for the 256-bit case; Zen1, Tiger Lake show 23% improvement, Zen3 7% to 20% improvement. The 512-bit case that decomposes into this is about the same, but with a 20% improvement on Skylake and a clear 20% improvement on Zen3. --- fearless_simd/src/generated/avx2.rs | 304 ++- fearless_simd/src/generated/avx512.rs | 112 + fearless_simd/src/generated/fallback.rs | 1418 ++++++++++--- fearless_simd/src/generated/neon.rs | 104 + fearless_simd/src/generated/simd_trait.rs | 12 + fearless_simd/src/generated/simd_types.rs | 30 + fearless_simd/src/generated/sse2.rs | 1808 +++++++++++++---- fearless_simd/src/generated/sse4_2.rs | 974 +++++++-- fearless_simd/src/generated/wasm.rs | 1004 +++++++-- fearless_simd_gen/src/mk_x86.rs | 30 +- .../lm_generated/swizzle_dyn_precise.rs | 16 + 11 files changed, 4631 insertions(+), 1181 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index e638dd6c..8ec5648f 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -4780,6 +4780,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x2, indices: u8x16) -> i64x2 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i64x2(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_i64x2(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -5171,6 +5189,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x2, indices: u8x16) -> u64x2 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u64x2(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_u64x2(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5774,22 +5810,17 @@ impl Simd for Avx2 { fn kernel(token: Avx2, a: f32x8, indices: u8x32) -> f32x8 { let bytes = token.cvt_to_bytes_f32x8(a); let idxs = indices; - let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); - let hi_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), - simd: token, - }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 32)) - .select(hi_shuf, u8x32::splat(token, 0)); let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); - let lo_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), simd: token, }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 16)) - .select(lo_shuf, result_bytes); token.cvt_from_bytes_f32x8(result_bytes) } ); @@ -6565,22 +6596,17 @@ impl Simd for Avx2 { fn kernel(token: Avx2, a: i8x32, indices: u8x32) -> i8x32 { let bytes = token.cvt_to_bytes_i8x32(a); let idxs = indices; - let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); - let hi_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), - simd: token, - }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 32)) - .select(hi_shuf, u8x32::splat(token, 0)); let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); - let lo_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), simd: token, }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 16)) - .select(lo_shuf, result_bytes); token.cvt_from_bytes_i8x32(result_bytes) } ); @@ -7295,22 +7321,17 @@ impl Simd for Avx2 { fn kernel(token: Avx2, a: u8x32, indices: u8x32) -> u8x32 { let bytes = token.cvt_to_bytes_u8x32(a); let idxs = indices; - let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); - let hi_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), - simd: token, - }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 32)) - .select(hi_shuf, u8x32::splat(token, 0)); let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); - let lo_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), simd: token, }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 16)) - .select(lo_shuf, result_bytes); token.cvt_from_bytes_u8x32(result_bytes) } ); @@ -8159,22 +8180,17 @@ impl Simd for Avx2 { fn kernel(token: Avx2, a: i16x16, indices: u8x32) -> i16x16 { let bytes = token.cvt_to_bytes_i16x16(a); let idxs = indices; - let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); - let hi_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), - simd: token, - }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 32)) - .select(hi_shuf, u8x32::splat(token, 0)); let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); - let lo_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), simd: token, }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 16)) - .select(lo_shuf, result_bytes); token.cvt_from_bytes_i16x16(result_bytes) } ); @@ -8781,22 +8797,17 @@ impl Simd for Avx2 { fn kernel(token: Avx2, a: u16x16, indices: u8x32) -> u16x16 { let bytes = token.cvt_to_bytes_u16x16(a); let idxs = indices; - let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); - let hi_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), - simd: token, - }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 32)) - .select(hi_shuf, u8x32::splat(token, 0)); let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); - let lo_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), simd: token, }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 16)) - .select(lo_shuf, result_bytes); token.cvt_from_bytes_u16x16(result_bytes) } ); @@ -9574,22 +9585,17 @@ impl Simd for Avx2 { fn kernel(token: Avx2, a: i32x8, indices: u8x32) -> i32x8 { let bytes = token.cvt_to_bytes_i32x8(a); let idxs = indices; - let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); - let hi_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), - simd: token, - }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 32)) - .select(hi_shuf, u8x32::splat(token, 0)); let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); - let lo_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), simd: token, }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 16)) - .select(lo_shuf, result_bytes); token.cvt_from_bytes_i32x8(result_bytes) } ); @@ -10119,22 +10125,17 @@ impl Simd for Avx2 { fn kernel(token: Avx2, a: u32x8, indices: u8x32) -> u32x8 { let bytes = token.cvt_to_bytes_u32x8(a); let idxs = indices; - let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); - let hi_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), - simd: token, - }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 32)) - .select(hi_shuf, u8x32::splat(token, 0)); let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); - let lo_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), simd: token, }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 16)) - .select(lo_shuf, result_bytes); token.cvt_from_bytes_u32x8(result_bytes) } ); @@ -10833,22 +10834,17 @@ impl Simd for Avx2 { fn kernel(token: Avx2, a: f64x4, indices: u8x32) -> f64x4 { let bytes = token.cvt_to_bytes_f64x4(a); let idxs = indices; - let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); - let hi_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(hihi, idxs.into())), - simd: token, - }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 32)) - .select(hi_shuf, u8x32::splat(token, 0)); let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); - let lo_shuf = u8x32 { - val: crate::support::Aligned256(_mm256_shuffle_epi8(lolo, idxs.into())), + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), simd: token, }; - let result_bytes = idxs - .simd_lt(u8x32::splat(token, 16)) - .select(lo_shuf, result_bytes); token.cvt_from_bytes_f64x4(result_bytes) } ); @@ -11385,6 +11381,29 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, indices: u8x32) -> i64x4 { + let bytes = token.cvt_to_bytes_i64x4(a); + let idxs = indices; + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + token.cvt_from_bytes_i64x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { crate::kernel!( #[inline(always)] @@ -11858,6 +11877,29 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, indices: u8x32) -> u64x4 { + let bytes = token.cvt_to_bytes_u64x4(a); + let idxs = indices; + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + token.cvt_from_bytes_u64x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { crate::kernel!( #[inline(always)] @@ -17153,6 +17195,33 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x8, indices: u8x64) -> i64x8 { + let bytes = token.cvt_to_bytes_i64x8(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + token.cvt_from_bytes_i64x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { let (a0, a1) = self.split_i64x8(a); let (b0, b1) = self.split_i64x8(b); @@ -17501,6 +17570,33 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x8, indices: u8x64) -> u64x8 { + let bytes = token.cvt_to_bytes_u64x8(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + token.cvt_from_bytes_u64x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { let (a0, a1) = self.split_u64x8(a); let (b0, b1) = self.split_u64x8(b); diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index 41b36cde..287de307 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -4853,6 +4853,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, indices: u8x16) -> i64x2 { + let bytes = token.cvt_to_bytes_i64x2(a).val.0; + let indices = indices.into(); + let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); + let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_i64x2(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -5297,6 +5315,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, indices: u8x16) -> u64x2 { + let bytes = token.cvt_to_bytes_u64x2(a).val.0; + let indices = indices.into(); + let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); + let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_u64x2(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -11558,6 +11594,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, indices: u8x32) -> i64x4 { + let bytes = token.cvt_to_bytes_i64x4(a).val.0; + let indices = indices.into(); + let in_range = + _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); + let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + token.cvt_from_bytes_i64x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { crate::kernel!( #[inline(always)] @@ -12075,6 +12130,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, indices: u8x32) -> u64x4 { + let bytes = token.cvt_to_bytes_u64x4(a).val.0; + let indices = indices.into(); + let in_range = + _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); + let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + token.cvt_from_bytes_u64x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { crate::kernel!( #[inline(always)] @@ -19107,6 +19181,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, indices: u8x64) -> i64x8 { + let bytes = token.cvt_to_bytes_i64x8(a).val.0; + let indices = indices.into(); + let in_range = + _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); + let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + token.cvt_from_bytes_i64x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { crate::kernel!( #[inline(always)] @@ -19647,6 +19740,25 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, indices: u8x64) -> u64x8 { + let bytes = token.cvt_to_bytes_u64x8(a).val.0; + let indices = indices.into(); + let in_range = + _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); + let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + token.cvt_from_bytes_u64x8(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { crate::kernel!( #[inline(always)] diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index b2f9f319..28cee689 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -6165,6 +6165,78 @@ impl Simd for Fallback { self.cvt_from_bytes_i64x2(result) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + let bytes = self.cvt_to_bytes_i64x2(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i64x2(result) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { [ i64::wrapping_add(a[0usize], b[0usize]), @@ -6535,6 +6607,78 @@ impl Simd for Fallback { self.cvt_from_bytes_u64x2(result) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + let bytes = self.cvt_to_bytes_u64x2(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u64x2(result) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { [ u64::wrapping_add(a[0usize], b[0usize]), @@ -11549,120 +11693,256 @@ impl Simd for Fallback { ) } #[inline(always)] - fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.add_i64x2(a0, b0), self.add_i64x2(a1, b1)) - } - #[inline(always)] - fn sub_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.sub_i64x2(a0, b0), self.sub_i64x2(a1, b1)) - } - #[inline(always)] - fn mul_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.mul_i64x2(a0, b0), self.mul_i64x2(a1, b1)) - } - #[inline(always)] - fn and_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.and_i64x2(a0, b0), self.and_i64x2(a1, b1)) - } - #[inline(always)] - fn or_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.or_i64x2(a0, b0), self.or_i64x2(a1, b1)) - } - #[inline(always)] - fn xor_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.xor_i64x2(a0, b0), self.xor_i64x2(a1, b1)) - } - #[inline(always)] - fn not_i64x4(self, a: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - self.combine_i64x2(self.not_i64x2(a0), self.not_i64x2(a1)) - } - #[inline(always)] - fn shl_i64x4(self, a: i64x4, shift: u32) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - self.combine_i64x2(self.shl_i64x2(a0, shift), self.shl_i64x2(a1, shift)) - } - #[inline(always)] - fn shlv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.shlv_i64x2(a0, b0), self.shlv_i64x2(a1, b1)) - } - #[inline(always)] - fn shr_i64x4(self, a: i64x4, shift: u32) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - self.combine_i64x2(self.shr_i64x2(a0, shift), self.shr_i64x2(a1, shift)) - } - #[inline(always)] - fn shrv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.shrv_i64x2(a0, b0), self.shrv_i64x2(a1, b1)) - } - #[inline(always)] - fn simd_eq_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_mask64x2(self.simd_eq_i64x2(a0, b0), self.simd_eq_i64x2(a1, b1)) - } - #[inline(always)] - fn simd_lt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_mask64x2(self.simd_lt_i64x2(a0, b0), self.simd_lt_i64x2(a1, b1)) - } - #[inline(always)] - fn simd_le_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_mask64x2(self.simd_le_i64x2(a0, b0), self.simd_le_i64x2(a1, b1)) - } - #[inline(always)] - fn simd_ge_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_mask64x2(self.simd_ge_i64x2(a0, b0), self.simd_ge_i64x2(a1, b1)) - } - #[inline(always)] - fn simd_gt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_mask64x2(self.simd_gt_i64x2(a0, b0), self.simd_gt_i64x2(a1, b1)) - } - #[inline(always)] - fn zip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, _) = self.split_i64x4(a); - let (b0, _) = self.split_i64x4(b); - self.combine_i64x2(self.zip_low_i64x2(a0, b0), self.zip_high_i64x2(a0, b0)) - } - #[inline(always)] - fn zip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (_, a1) = self.split_i64x4(a); - let (_, b1) = self.split_i64x4(b); - self.combine_i64x2(self.zip_low_i64x2(a1, b1), self.zip_high_i64x2(a1, b1)) - } - #[inline(always)] - fn unzip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.unzip_low_i64x2(a0, a1), self.unzip_low_i64x2(b0, b1)) - } - #[inline(always)] - fn unzip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + let bytes = self.cvt_to_bytes_i64x4(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i64x4(result) + } + #[inline(always)] + fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.add_i64x2(a0, b0), self.add_i64x2(a1, b1)) + } + #[inline(always)] + fn sub_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.sub_i64x2(a0, b0), self.sub_i64x2(a1, b1)) + } + #[inline(always)] + fn mul_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.mul_i64x2(a0, b0), self.mul_i64x2(a1, b1)) + } + #[inline(always)] + fn and_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.and_i64x2(a0, b0), self.and_i64x2(a1, b1)) + } + #[inline(always)] + fn or_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.or_i64x2(a0, b0), self.or_i64x2(a1, b1)) + } + #[inline(always)] + fn xor_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.xor_i64x2(a0, b0), self.xor_i64x2(a1, b1)) + } + #[inline(always)] + fn not_i64x4(self, a: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.not_i64x2(a0), self.not_i64x2(a1)) + } + #[inline(always)] + fn shl_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.shl_i64x2(a0, shift), self.shl_i64x2(a1, shift)) + } + #[inline(always)] + fn shlv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.shlv_i64x2(a0, b0), self.shlv_i64x2(a1, b1)) + } + #[inline(always)] + fn shr_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.shr_i64x2(a0, shift), self.shr_i64x2(a1, shift)) + } + #[inline(always)] + fn shrv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.shrv_i64x2(a0, b0), self.shrv_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_eq_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_eq_i64x2(a0, b0), self.simd_eq_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_lt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_lt_i64x2(a0, b0), self.simd_lt_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_le_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_le_i64x2(a0, b0), self.simd_le_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_ge_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_ge_i64x2(a0, b0), self.simd_ge_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_gt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_gt_i64x2(a0, b0), self.simd_gt_i64x2(a1, b1)) + } + #[inline(always)] + fn zip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, _) = self.split_i64x4(a); + let (b0, _) = self.split_i64x4(b); + self.combine_i64x2(self.zip_low_i64x2(a0, b0), self.zip_high_i64x2(a0, b0)) + } + #[inline(always)] + fn zip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (_, a1) = self.split_i64x4(a); + let (_, b1) = self.split_i64x4(b); + self.combine_i64x2(self.zip_low_i64x2(a1, b1), self.zip_high_i64x2(a1, b1)) + } + #[inline(always)] + fn unzip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.unzip_low_i64x2(a0, a1), self.unzip_low_i64x2(b0, b1)) + } + #[inline(always)] + fn unzip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); self.combine_i64x2(self.unzip_high_i64x2(a0, a1), self.unzip_high_i64x2(b0, b1)) } #[inline(always)] @@ -11874,15 +12154,151 @@ impl Simd for Fallback { ) } #[inline(always)] - fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { - let (a0, a1) = self.split_u64x4(a); - let (b0, b1) = self.split_u64x4(b); - self.combine_u64x2(self.add_u64x2(a0, b0), self.add_u64x2(a1, b1)) - } - #[inline(always)] - fn sub_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { - let (a0, a1) = self.split_u64x4(a); - let (b0, b1) = self.split_u64x4(b); + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + let bytes = self.cvt_to_bytes_u64x4(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u64x4(result) + } + #[inline(always)] + fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.add_u64x2(a0, b0), self.add_u64x2(a1, b1)) + } + #[inline(always)] + fn sub_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); self.combine_u64x2(self.sub_u64x2(a0, b0), self.sub_u64x2(a1, b1)) } #[inline(always)] @@ -18545,120 +18961,384 @@ impl Simd for Fallback { ) } #[inline(always)] - fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.add_i64x4(a0, b0), self.add_i64x4(a1, b1)) - } - #[inline(always)] - fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.sub_i64x4(a0, b0), self.sub_i64x4(a1, b1)) - } - #[inline(always)] - fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1)) - } - #[inline(always)] - fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.and_i64x4(a0, b0), self.and_i64x4(a1, b1)) - } - #[inline(always)] - fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.or_i64x4(a0, b0), self.or_i64x4(a1, b1)) - } - #[inline(always)] - fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.xor_i64x4(a0, b0), self.xor_i64x4(a1, b1)) - } - #[inline(always)] - fn not_i64x8(self, a: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.not_i64x4(a0), self.not_i64x4(a1)) - } - #[inline(always)] - fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.shl_i64x4(a0, shift), self.shl_i64x4(a1, shift)) - } - #[inline(always)] - fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.shlv_i64x4(a0, b0), self.shlv_i64x4(a1, b1)) - } - #[inline(always)] - fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.shr_i64x4(a0, shift), self.shr_i64x4(a1, shift)) - } - #[inline(always)] - fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.shrv_i64x4(a0, b0), self.shrv_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_eq_i64x4(a0, b0), self.simd_eq_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_lt_i64x4(a0, b0), self.simd_lt_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_le_i64x4(a0, b0), self.simd_le_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_ge_i64x4(a0, b0), self.simd_ge_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_gt_i64x4(a0, b0), self.simd_gt_i64x4(a1, b1)) - } - #[inline(always)] - fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, _) = self.split_i64x8(a); - let (b0, _) = self.split_i64x8(b); - self.combine_i64x4(self.zip_low_i64x4(a0, b0), self.zip_high_i64x4(a0, b0)) - } - #[inline(always)] - fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (_, a1) = self.split_i64x8(a); - let (_, b1) = self.split_i64x8(b); - self.combine_i64x4(self.zip_low_i64x4(a1, b1), self.zip_high_i64x4(a1, b1)) - } - #[inline(always)] - fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.unzip_low_i64x4(a0, a1), self.unzip_low_i64x4(b0, b1)) - } - #[inline(always)] - fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + let bytes = self.cvt_to_bytes_i64x8(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i64x8(result) + } + #[inline(always)] + fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.add_i64x4(a0, b0), self.add_i64x4(a1, b1)) + } + #[inline(always)] + fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.sub_i64x4(a0, b0), self.sub_i64x4(a1, b1)) + } + #[inline(always)] + fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1)) + } + #[inline(always)] + fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.and_i64x4(a0, b0), self.and_i64x4(a1, b1)) + } + #[inline(always)] + fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.or_i64x4(a0, b0), self.or_i64x4(a1, b1)) + } + #[inline(always)] + fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.xor_i64x4(a0, b0), self.xor_i64x4(a1, b1)) + } + #[inline(always)] + fn not_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.not_i64x4(a0), self.not_i64x4(a1)) + } + #[inline(always)] + fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shl_i64x4(a0, shift), self.shl_i64x4(a1, shift)) + } + #[inline(always)] + fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shlv_i64x4(a0, b0), self.shlv_i64x4(a1, b1)) + } + #[inline(always)] + fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shr_i64x4(a0, shift), self.shr_i64x4(a1, shift)) + } + #[inline(always)] + fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shrv_i64x4(a0, b0), self.shrv_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_eq_i64x4(a0, b0), self.simd_eq_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_lt_i64x4(a0, b0), self.simd_lt_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_le_i64x4(a0, b0), self.simd_le_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_ge_i64x4(a0, b0), self.simd_ge_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_gt_i64x4(a0, b0), self.simd_gt_i64x4(a1, b1)) + } + #[inline(always)] + fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, _) = self.split_i64x8(a); + let (b0, _) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a0, b0), self.zip_high_i64x4(a0, b0)) + } + #[inline(always)] + fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (_, a1) = self.split_i64x8(a); + let (_, b1) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a1, b1), self.zip_high_i64x4(a1, b1)) + } + #[inline(always)] + fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_low_i64x4(a0, a1), self.unzip_low_i64x4(b0, b1)) + } + #[inline(always)] + fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); self.combine_i64x4(self.unzip_high_i64x4(a0, a1), self.unzip_high_i64x4(b0, b1)) } #[inline(always)] @@ -18879,6 +19559,270 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + let bytes = self.cvt_to_bytes_u64x8(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u64x8(result) + } + #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { let (a0, a1) = self.split_u64x8(a); let (b0, b1) = self.split_u64x8(b); diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index dde2299e..7c41c5c0 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -4609,6 +4609,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, indices: u8x16) -> i64x2 { + let result = vqtbl1q_u8(token.cvt_to_bytes_i64x2(a).val.0, indices.into()); + token.cvt_from_bytes_i64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -5027,6 +5041,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, indices: u8x16) -> u64x2 { + let result = vqtbl1q_u8(token.cvt_to_bytes_u64x2(a).val.0, indices.into()); + token.cvt_from_bytes_u64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -9561,6 +9589,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x4, indices: u8x32) -> i64x4 { + let table = token.cvt_to_bytes_i64x4(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + token.cvt_from_bytes_i64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { let (a0, a1) = self.split_i64x4(a); let (b0, b1) = self.split_i64x4(b); @@ -9923,6 +9968,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x4, indices: u8x32) -> u64x4 { + let table = token.cvt_to_bytes_u64x4(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + token.cvt_from_bytes_u64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { let (a0, a1) = self.split_u64x4(a); let (b0, b1) = self.split_u64x4(b); @@ -14990,6 +15052,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x8, indices: u8x64) -> i64x8 { + let table = token.cvt_to_bytes_i64x8(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + token.cvt_from_bytes_i64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { let (a0, a1) = self.split_i64x8(a); let (b0, b1) = self.split_i64x8(b); @@ -15377,6 +15460,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x8, indices: u8x64) -> u64x8 { + let table = token.cvt_to_bytes_u64x8(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + token.cvt_from_bytes_u64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { let (a0, a1) = self.split_u64x8(a); let (b0, b1) = self.split_u64x8(b); diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index 2efdf921..550d120d 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -1161,6 +1161,8 @@ pub trait Simd: ) -> i64x2; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i64x2(self, a: i64x2, indices: u8x16) -> i64x2; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1263,6 +1265,8 @@ pub trait Simd: ) -> u64x2; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u64x2(self, a: u64x2, indices: u8x16) -> u64x2; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2413,6 +2417,8 @@ pub trait Simd: ) -> i64x4; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i64x4(self, a: i64x4, indices: u8x32) -> i64x4; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2517,6 +2523,8 @@ pub trait Simd: ) -> u64x4; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u64x4(self, a: u64x4, indices: u8x32) -> u64x4; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -3675,6 +3683,8 @@ pub trait Simd: ) -> i64x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i64x8(self, a: i64x8, indices: u8x64) -> i64x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -3777,6 +3787,8 @@ pub trait Simd: ) -> u64x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u64x8(self, a: u64x8, indices: u8x64) -> u64x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] diff --git a/fearless_simd/src/generated/simd_types.rs b/fearless_simd/src/generated/simd_types.rs index 740c5000..973d28be 100644 --- a/fearless_simd/src/generated/simd_types.rs +++ b/fearless_simd/src/generated/simd_types.rs @@ -2319,6 +2319,11 @@ impl SimdBase for i64x2 { self.simd .swizzle_dyn_within_blocks_i64x2(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i64x2(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i64x2 { #[inline(always)] @@ -2526,6 +2531,11 @@ impl SimdBase for u64x2 { self.simd .swizzle_dyn_within_blocks_u64x2(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u64x2(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u64x2 { #[inline(always)] @@ -5136,6 +5146,11 @@ impl SimdBase for i64x4 { self.simd .swizzle_dyn_within_blocks_i64x4(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i64x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i64x4 { #[inline(always)] @@ -5350,6 +5365,11 @@ impl SimdBase for u64x4 { self.simd .swizzle_dyn_within_blocks_u64x4(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u64x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u64x4 { #[inline(always)] @@ -8079,6 +8099,11 @@ impl SimdBase for i64x8 { self.simd .swizzle_dyn_within_blocks_i64x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i64x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i64x8 { #[inline(always)] @@ -8296,6 +8321,11 @@ impl SimdBase for u64x8 { self.simd .swizzle_dyn_within_blocks_u64x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u64x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u64x8 { #[inline(always)] diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index ed3cb966..643e64ba 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -5864,6 +5864,78 @@ impl Simd for Sse2 { self.cvt_from_bytes_i64x2(result) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + let bytes = self.cvt_to_bytes_i64x2(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i64x2(result) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -6317,6 +6389,78 @@ impl Simd for Sse2 { self.cvt_from_bytes_u64x2(result) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + let bytes = self.cvt_to_bytes_u64x2(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u64x2(result) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -11577,120 +11721,256 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.add_i64x2(a0, b0), self.add_i64x2(a1, b1)) - } - #[inline(always)] - fn sub_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.sub_i64x2(a0, b0), self.sub_i64x2(a1, b1)) - } - #[inline(always)] - fn mul_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.mul_i64x2(a0, b0), self.mul_i64x2(a1, b1)) - } - #[inline(always)] - fn and_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.and_i64x2(a0, b0), self.and_i64x2(a1, b1)) - } - #[inline(always)] - fn or_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.or_i64x2(a0, b0), self.or_i64x2(a1, b1)) - } - #[inline(always)] - fn xor_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.xor_i64x2(a0, b0), self.xor_i64x2(a1, b1)) - } - #[inline(always)] - fn not_i64x4(self, a: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - self.combine_i64x2(self.not_i64x2(a0), self.not_i64x2(a1)) - } - #[inline(always)] - fn shl_i64x4(self, a: i64x4, shift: u32) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - self.combine_i64x2(self.shl_i64x2(a0, shift), self.shl_i64x2(a1, shift)) - } - #[inline(always)] - fn shlv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.shlv_i64x2(a0, b0), self.shlv_i64x2(a1, b1)) - } - #[inline(always)] - fn shr_i64x4(self, a: i64x4, shift: u32) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - self.combine_i64x2(self.shr_i64x2(a0, shift), self.shr_i64x2(a1, shift)) - } - #[inline(always)] - fn shrv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.shrv_i64x2(a0, b0), self.shrv_i64x2(a1, b1)) - } - #[inline(always)] - fn simd_eq_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_mask64x2(self.simd_eq_i64x2(a0, b0), self.simd_eq_i64x2(a1, b1)) - } - #[inline(always)] - fn simd_lt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_mask64x2(self.simd_lt_i64x2(a0, b0), self.simd_lt_i64x2(a1, b1)) - } - #[inline(always)] - fn simd_le_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_mask64x2(self.simd_le_i64x2(a0, b0), self.simd_le_i64x2(a1, b1)) - } - #[inline(always)] - fn simd_ge_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_mask64x2(self.simd_ge_i64x2(a0, b0), self.simd_ge_i64x2(a1, b1)) - } - #[inline(always)] - fn simd_gt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_mask64x2(self.simd_gt_i64x2(a0, b0), self.simd_gt_i64x2(a1, b1)) - } - #[inline(always)] - fn zip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, _) = self.split_i64x4(a); - let (b0, _) = self.split_i64x4(b); - self.combine_i64x2(self.zip_low_i64x2(a0, b0), self.zip_high_i64x2(a0, b0)) - } - #[inline(always)] - fn zip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (_, a1) = self.split_i64x4(a); - let (_, b1) = self.split_i64x4(b); - self.combine_i64x2(self.zip_low_i64x2(a1, b1), self.zip_high_i64x2(a1, b1)) - } - #[inline(always)] - fn unzip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); - self.combine_i64x2(self.unzip_low_i64x2(a0, a1), self.unzip_low_i64x2(b0, b1)) - } - #[inline(always)] - fn unzip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { - let (a0, a1) = self.split_i64x4(a); - let (b0, b1) = self.split_i64x4(b); + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + let bytes = self.cvt_to_bytes_i64x4(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i64x4(result) + } + #[inline(always)] + fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.add_i64x2(a0, b0), self.add_i64x2(a1, b1)) + } + #[inline(always)] + fn sub_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.sub_i64x2(a0, b0), self.sub_i64x2(a1, b1)) + } + #[inline(always)] + fn mul_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.mul_i64x2(a0, b0), self.mul_i64x2(a1, b1)) + } + #[inline(always)] + fn and_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.and_i64x2(a0, b0), self.and_i64x2(a1, b1)) + } + #[inline(always)] + fn or_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.or_i64x2(a0, b0), self.or_i64x2(a1, b1)) + } + #[inline(always)] + fn xor_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.xor_i64x2(a0, b0), self.xor_i64x2(a1, b1)) + } + #[inline(always)] + fn not_i64x4(self, a: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.not_i64x2(a0), self.not_i64x2(a1)) + } + #[inline(always)] + fn shl_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.shl_i64x2(a0, shift), self.shl_i64x2(a1, shift)) + } + #[inline(always)] + fn shlv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.shlv_i64x2(a0, b0), self.shlv_i64x2(a1, b1)) + } + #[inline(always)] + fn shr_i64x4(self, a: i64x4, shift: u32) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + self.combine_i64x2(self.shr_i64x2(a0, shift), self.shr_i64x2(a1, shift)) + } + #[inline(always)] + fn shrv_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.shrv_i64x2(a0, b0), self.shrv_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_eq_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_eq_i64x2(a0, b0), self.simd_eq_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_lt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_lt_i64x2(a0, b0), self.simd_lt_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_le_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_le_i64x2(a0, b0), self.simd_le_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_ge_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_ge_i64x2(a0, b0), self.simd_ge_i64x2(a1, b1)) + } + #[inline(always)] + fn simd_gt_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_mask64x2(self.simd_gt_i64x2(a0, b0), self.simd_gt_i64x2(a1, b1)) + } + #[inline(always)] + fn zip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, _) = self.split_i64x4(a); + let (b0, _) = self.split_i64x4(b); + self.combine_i64x2(self.zip_low_i64x2(a0, b0), self.zip_high_i64x2(a0, b0)) + } + #[inline(always)] + fn zip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (_, a1) = self.split_i64x4(a); + let (_, b1) = self.split_i64x4(b); + self.combine_i64x2(self.zip_low_i64x2(a1, b1), self.zip_high_i64x2(a1, b1)) + } + #[inline(always)] + fn unzip_low_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); + self.combine_i64x2(self.unzip_low_i64x2(a0, a1), self.unzip_low_i64x2(b0, b1)) + } + #[inline(always)] + fn unzip_high_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { + let (a0, a1) = self.split_i64x4(a); + let (b0, b1) = self.split_i64x4(b); self.combine_i64x2(self.unzip_high_i64x2(a0, a1), self.unzip_high_i64x2(b0, b1)) } #[inline(always)] @@ -11916,15 +12196,151 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { - let (a0, a1) = self.split_u64x4(a); - let (b0, b1) = self.split_u64x4(b); - self.combine_u64x2(self.add_u64x2(a0, b0), self.add_u64x2(a1, b1)) - } - #[inline(always)] - fn sub_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { - let (a0, a1) = self.split_u64x4(a); - let (b0, b1) = self.split_u64x4(b); + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + let bytes = self.cvt_to_bytes_u64x4(a); + let result: u8x32 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u64x4(result) + } + #[inline(always)] + fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); + self.combine_u64x2(self.add_u64x2(a0, b0), self.add_u64x2(a1, b1)) + } + #[inline(always)] + fn sub_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { + let (a0, a1) = self.split_u64x4(a); + let (b0, b1) = self.split_u64x4(b); self.combine_u64x2(self.sub_u64x2(a0, b0), self.sub_u64x2(a1, b1)) } #[inline(always)] @@ -18832,120 +19248,384 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.add_i64x4(a0, b0), self.add_i64x4(a1, b1)) - } - #[inline(always)] - fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.sub_i64x4(a0, b0), self.sub_i64x4(a1, b1)) - } - #[inline(always)] - fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1)) - } - #[inline(always)] - fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.and_i64x4(a0, b0), self.and_i64x4(a1, b1)) - } - #[inline(always)] - fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.or_i64x4(a0, b0), self.or_i64x4(a1, b1)) - } - #[inline(always)] - fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.xor_i64x4(a0, b0), self.xor_i64x4(a1, b1)) - } - #[inline(always)] - fn not_i64x8(self, a: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.not_i64x4(a0), self.not_i64x4(a1)) - } - #[inline(always)] - fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.shl_i64x4(a0, shift), self.shl_i64x4(a1, shift)) - } - #[inline(always)] - fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.shlv_i64x4(a0, b0), self.shlv_i64x4(a1, b1)) - } - #[inline(always)] - fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.shr_i64x4(a0, shift), self.shr_i64x4(a1, shift)) - } - #[inline(always)] - fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.shrv_i64x4(a0, b0), self.shrv_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_eq_i64x4(a0, b0), self.simd_eq_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_lt_i64x4(a0, b0), self.simd_lt_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_le_i64x4(a0, b0), self.simd_le_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_ge_i64x4(a0, b0), self.simd_ge_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_gt_i64x4(a0, b0), self.simd_gt_i64x4(a1, b1)) - } - #[inline(always)] - fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, _) = self.split_i64x8(a); - let (b0, _) = self.split_i64x8(b); - self.combine_i64x4(self.zip_low_i64x4(a0, b0), self.zip_high_i64x4(a0, b0)) - } - #[inline(always)] - fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (_, a1) = self.split_i64x8(a); - let (_, b1) = self.split_i64x8(b); - self.combine_i64x4(self.zip_low_i64x4(a1, b1), self.zip_high_i64x4(a1, b1)) - } - #[inline(always)] - fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.unzip_low_i64x4(a0, a1), self.unzip_low_i64x4(b0, b1)) - } - #[inline(always)] - fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + let bytes = self.cvt_to_bytes_i64x8(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i64x8(result) + } + #[inline(always)] + fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.add_i64x4(a0, b0), self.add_i64x4(a1, b1)) + } + #[inline(always)] + fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.sub_i64x4(a0, b0), self.sub_i64x4(a1, b1)) + } + #[inline(always)] + fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1)) + } + #[inline(always)] + fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.and_i64x4(a0, b0), self.and_i64x4(a1, b1)) + } + #[inline(always)] + fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.or_i64x4(a0, b0), self.or_i64x4(a1, b1)) + } + #[inline(always)] + fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.xor_i64x4(a0, b0), self.xor_i64x4(a1, b1)) + } + #[inline(always)] + fn not_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.not_i64x4(a0), self.not_i64x4(a1)) + } + #[inline(always)] + fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shl_i64x4(a0, shift), self.shl_i64x4(a1, shift)) + } + #[inline(always)] + fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shlv_i64x4(a0, b0), self.shlv_i64x4(a1, b1)) + } + #[inline(always)] + fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shr_i64x4(a0, shift), self.shr_i64x4(a1, shift)) + } + #[inline(always)] + fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shrv_i64x4(a0, b0), self.shrv_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_eq_i64x4(a0, b0), self.simd_eq_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_lt_i64x4(a0, b0), self.simd_lt_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_le_i64x4(a0, b0), self.simd_le_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_ge_i64x4(a0, b0), self.simd_ge_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_gt_i64x4(a0, b0), self.simd_gt_i64x4(a1, b1)) + } + #[inline(always)] + fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, _) = self.split_i64x8(a); + let (b0, _) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a0, b0), self.zip_high_i64x4(a0, b0)) + } + #[inline(always)] + fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (_, a1) = self.split_i64x8(a); + let (_, b1) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a1, b1), self.zip_high_i64x4(a1, b1)) + } + #[inline(always)] + fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_low_i64x4(a0, a1), self.unzip_low_i64x4(b0, b1)) + } + #[inline(always)] + fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); self.combine_i64x4(self.unzip_high_i64x4(a0, a1), self.unzip_high_i64x4(b0, b1)) } #[inline(always)] @@ -18975,209 +19655,473 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn select_i64x8(self, a: mask64x8, b: i64x8, c: i64x8) -> i64x8 { - let (a0, a1) = self.split_mask64x8(a); - let (b0, b1) = self.split_i64x8(b); - let (c0, c1) = self.split_i64x8(c); - self.combine_i64x4(self.select_i64x4(a0, b0, c0), self.select_i64x4(a1, b1, c1)) + fn select_i64x8(self, a: mask64x8, b: i64x8, c: i64x8) -> i64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_i64x8(b); + let (c0, c1) = self.split_i64x8(c); + self.combine_i64x4(self.select_i64x4(a0, b0, c0), self.select_i64x4(a1, b1, c1)) + } + #[inline(always)] + fn min_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.min_i64x4(a0, b0), self.min_i64x4(a1, b1)) + } + #[inline(always)] + fn max_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.max_i64x4(a0, b0), self.max_i64x4(a1, b1)) + } + #[inline(always)] + fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { + ( + i64x4 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + i64x4 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn neg_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) + } + #[inline(always)] + fn reinterpret_u8_i64x8(self, a: i64x8) -> u8x64 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u8x32(self.reinterpret_u8_i64x4(a0), self.reinterpret_u8_i64x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_i64x8(self, a: i64x8) -> u32x16 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u32x8( + self.reinterpret_u32_i64x4(a0), + self.reinterpret_u32_i64x4(a1), + ) + } + #[inline(always)] + fn splat_u64x8(self, val: u64) -> u64x8 { + let half = self.splat_u64x4(val); + self.combine_u64x4(half, half) + } + #[inline(always)] + fn load_array_u64x8(self, val: [u64; 8usize]) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u64x8(self, val: &[u64; 8usize]) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u64x8(self, a: u64x8) -> [u64; 8usize] { + crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [u64; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u64x8(self, a: &u64x8) -> &[u64; 8usize] { + crate::transmute::checked_cast_ref::<[__m128i; 4usize], [u64; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u64x8(self, a: &mut u64x8) -> &mut [u64; 8usize] { + crate::transmute::checked_cast_mut::<[__m128i; 4usize], [u64; 8usize]>(&mut a.val.0) + } + #[inline(always)] + fn store_array_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { + crate::transmute::checked_transmute_store(a.val.0, dest); + } + #[inline(always)] + fn cvt_from_bytes_u64x8(self, a: u8x64) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn cvt_to_bytes_u64x8(self, a: u64x8) -> u8x64 { + u8x64 { + val: crate::transmute::checked_transmute_copy(&a.val), + simd: self, + } + } + #[inline(always)] + fn slide_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { + if SHIFT >= 8usize { + return b; + } + let result = cross_block_alignr_128x4( + self, + self.cvt_to_bytes_u64x8(b).val.0, + self.cvt_to_bytes_u64x8(a).val.0, + SHIFT * 8usize, + ); + self.cvt_from_bytes_u64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn slide_within_blocks_u64x8( + self, + a: u64x8, + b: u64x8, + ) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (b0, b1) = self.split_u64x8(b); + self.combine_u64x4( + self.slide_within_blocks_u64x4::(a0, b0), + self.slide_within_blocks_u64x4::(a1, b1), + ) + } + #[inline(always)] + fn rotate_elements_left_u64x8(self, a: u64x8) -> u64x8 { + match OFFSET % 8 { + 0 => self.slide_u64x8::<0>(a, a), + 1 => self.slide_u64x8::<1>(a, a), + 2 => self.slide_u64x8::<2>(a, a), + 3 => self.slide_u64x8::<3>(a, a), + 4 => self.slide_u64x8::<4>(a, a), + 5 => self.slide_u64x8::<5>(a, a), + 6 => self.slide_u64x8::<6>(a, a), + 7 => self.slide_u64x8::<7>(a, a), + _ => unreachable!(), + } + } + #[inline(always)] + fn rotate_elements_right_u64x8(self, a: u64x8) -> u64x8 { + match OFFSET % 8 { + 0 => self.slide_u64x8::<8>(a, a), + 1 => self.slide_u64x8::<7>(a, a), + 2 => self.slide_u64x8::<6>(a, a), + 3 => self.slide_u64x8::<5>(a, a), + 4 => self.slide_u64x8::<4>(a, a), + 5 => self.slide_u64x8::<3>(a, a), + 6 => self.slide_u64x8::<2>(a, a), + 7 => self.slide_u64x8::<1>(a, a), + _ => unreachable!(), + } } #[inline(always)] - fn min_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.min_i64x4(a0, b0), self.min_i64x4(a1, b1)) + fn shift_elements_left_u64x8( + self, + a: u64x8, + padding: u64, + ) -> u64x8 { + let padding = self.splat_u64x8(padding); + match OFFSET { + 0 => self.slide_u64x8::<0>(a, padding), + 1 => self.slide_u64x8::<1>(a, padding), + 2 => self.slide_u64x8::<2>(a, padding), + 3 => self.slide_u64x8::<3>(a, padding), + 4 => self.slide_u64x8::<4>(a, padding), + 5 => self.slide_u64x8::<5>(a, padding), + 6 => self.slide_u64x8::<6>(a, padding), + 7 => self.slide_u64x8::<7>(a, padding), + 8 => self.slide_u64x8::<8>(a, padding), + _ => self.slide_u64x8::<8>(a, padding), + } } #[inline(always)] - fn max_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.max_i64x4(a0, b0), self.max_i64x4(a1, b1)) + fn shift_elements_right_u64x8( + self, + a: u64x8, + padding: u64, + ) -> u64x8 { + let padding = self.splat_u64x8(padding); + match OFFSET { + 0 => self.slide_u64x8::<8>(padding, a), + 1 => self.slide_u64x8::<7>(padding, a), + 2 => self.slide_u64x8::<6>(padding, a), + 3 => self.slide_u64x8::<5>(padding, a), + 4 => self.slide_u64x8::<4>(padding, a), + 5 => self.slide_u64x8::<3>(padding, a), + 6 => self.slide_u64x8::<2>(padding, a), + 7 => self.slide_u64x8::<1>(padding, a), + 8 => self.slide_u64x8::<0>(padding, a), + _ => self.slide_u64x8::<0>(padding, a), + } } #[inline(always)] - fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { - ( - i64x4 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), - simd: self, + fn swizzle_dyn_within_blocks_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + let (a0, a1) = self.split_u64x8(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u64x4( + self.swizzle_dyn_within_blocks_u64x4(a0, indices0), + self.swizzle_dyn_within_blocks_u64x4(a1, indices1), + ) + } + #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + let bytes = self.cvt_to_bytes_u64x8(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) }, - i64x4 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), - simd: self, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) }, - ) - } - #[inline(always)] - fn neg_i64x8(self, a: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) - } - #[inline(always)] - fn reinterpret_u8_i64x8(self, a: i64x8) -> u8x64 { - let (a0, a1) = self.split_i64x8(a); - self.combine_u8x32(self.reinterpret_u8_i64x4(a0), self.reinterpret_u8_i64x4(a1)) - } - #[inline(always)] - fn reinterpret_u32_i64x8(self, a: i64x8) -> u32x16 { - let (a0, a1) = self.split_i64x8(a); - self.combine_u32x8( - self.reinterpret_u32_i64x4(a0), - self.reinterpret_u32_i64x4(a1), - ) - } - #[inline(always)] - fn splat_u64x8(self, val: u64) -> u64x8 { - let half = self.splat_u64x4(val); - self.combine_u64x4(half, half) - } - #[inline(always)] - fn load_array_u64x8(self, val: [u64; 8usize]) -> u64x8 { - u64x8 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } - } - #[inline(always)] - fn load_array_ref_u64x8(self, val: &[u64; 8usize]) -> u64x8 { - u64x8 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } - } - #[inline(always)] - fn as_array_u64x8(self, a: u64x8) -> [u64; 8usize] { - crate::transmute::checked_transmute_copy::<[__m128i; 4usize], [u64; 8usize]>(&a.val.0) - } - #[inline(always)] - fn as_array_ref_u64x8(self, a: &u64x8) -> &[u64; 8usize] { - crate::transmute::checked_cast_ref::<[__m128i; 4usize], [u64; 8usize]>(&a.val.0) - } - #[inline(always)] - fn as_array_mut_u64x8(self, a: &mut u64x8) -> &mut [u64; 8usize] { - crate::transmute::checked_cast_mut::<[__m128i; 4usize], [u64; 8usize]>(&mut a.val.0) - } - #[inline(always)] - fn store_array_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - crate::transmute::checked_transmute_store(a.val.0, dest); - } - #[inline(always)] - fn cvt_from_bytes_u64x8(self, a: u8x64) -> u64x8 { - u64x8 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } - } - #[inline(always)] - fn cvt_to_bytes_u64x8(self, a: u64x8) -> u8x64 { - u8x64 { - val: crate::transmute::checked_transmute_copy(&a.val), - simd: self, - } - } - #[inline(always)] - fn slide_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { - if SHIFT >= 8usize { - return b; - } - let result = cross_block_alignr_128x4( - self, - self.cvt_to_bytes_u64x8(b).val.0, - self.cvt_to_bytes_u64x8(a).val.0, - SHIFT * 8usize, - ); - self.cvt_from_bytes_u64x8(u8x64 { - val: crate::support::Aligned512(result), - simd: self, - }) - } - #[inline(always)] - fn slide_within_blocks_u64x8( - self, - a: u64x8, - b: u64x8, - ) -> u64x8 { - let (a0, a1) = self.split_u64x8(a); - let (b0, b1) = self.split_u64x8(b); - self.combine_u64x4( - self.slide_within_blocks_u64x4::(a0, b0), - self.slide_within_blocks_u64x4::(a1, b1), - ) - } - #[inline(always)] - fn rotate_elements_left_u64x8(self, a: u64x8) -> u64x8 { - match OFFSET % 8 { - 0 => self.slide_u64x8::<0>(a, a), - 1 => self.slide_u64x8::<1>(a, a), - 2 => self.slide_u64x8::<2>(a, a), - 3 => self.slide_u64x8::<3>(a, a), - 4 => self.slide_u64x8::<4>(a, a), - 5 => self.slide_u64x8::<5>(a, a), - 6 => self.slide_u64x8::<6>(a, a), - 7 => self.slide_u64x8::<7>(a, a), - _ => unreachable!(), - } - } - #[inline(always)] - fn rotate_elements_right_u64x8(self, a: u64x8) -> u64x8 { - match OFFSET % 8 { - 0 => self.slide_u64x8::<8>(a, a), - 1 => self.slide_u64x8::<7>(a, a), - 2 => self.slide_u64x8::<6>(a, a), - 3 => self.slide_u64x8::<5>(a, a), - 4 => self.slide_u64x8::<4>(a, a), - 5 => self.slide_u64x8::<3>(a, a), - 6 => self.slide_u64x8::<2>(a, a), - 7 => self.slide_u64x8::<1>(a, a), - _ => unreachable!(), - } - } - #[inline(always)] - fn shift_elements_left_u64x8( - self, - a: u64x8, - padding: u64, - ) -> u64x8 { - let padding = self.splat_u64x8(padding); - match OFFSET { - 0 => self.slide_u64x8::<0>(a, padding), - 1 => self.slide_u64x8::<1>(a, padding), - 2 => self.slide_u64x8::<2>(a, padding), - 3 => self.slide_u64x8::<3>(a, padding), - 4 => self.slide_u64x8::<4>(a, padding), - 5 => self.slide_u64x8::<5>(a, padding), - 6 => self.slide_u64x8::<6>(a, padding), - 7 => self.slide_u64x8::<7>(a, padding), - 8 => self.slide_u64x8::<8>(a, padding), - _ => self.slide_u64x8::<8>(a, padding), - } - } - #[inline(always)] - fn shift_elements_right_u64x8( - self, - a: u64x8, - padding: u64, - ) -> u64x8 { - let padding = self.splat_u64x8(padding); - match OFFSET { - 0 => self.slide_u64x8::<8>(padding, a), - 1 => self.slide_u64x8::<7>(padding, a), - 2 => self.slide_u64x8::<6>(padding, a), - 3 => self.slide_u64x8::<5>(padding, a), - 4 => self.slide_u64x8::<4>(padding, a), - 5 => self.slide_u64x8::<3>(padding, a), - 6 => self.slide_u64x8::<2>(padding, a), - 7 => self.slide_u64x8::<1>(padding, a), - 8 => self.slide_u64x8::<0>(padding, a), - _ => self.slide_u64x8::<0>(padding, a), - } - } - #[inline(always)] - fn swizzle_dyn_within_blocks_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { - let (a0, a1) = self.split_u64x8(a); - let (indices0, indices1) = self.split_u8x64(indices); - self.combine_u64x4( - self.swizzle_dyn_within_blocks_u64x4(a0, indices0), - self.swizzle_dyn_within_blocks_u64x4(a1, indices1), - ) + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u64x8(result) } #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 26e30dd6..8c02cd9a 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -4723,6 +4723,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i64x2, indices: u8x16) -> i64x2 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i64x2(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_i64x2(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -5109,6 +5127,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, indices: u8x16) -> u64x2 { + let indices = indices.into(); + let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u64x2(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + token.cvt_from_bytes_u64x2(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -9486,6 +9522,33 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i64x4, indices: u8x32) -> i64x4 { + let bytes = token.cvt_to_bytes_i64x4(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + token.cvt_from_bytes_i64x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { let (a0, a1) = self.split_i64x4(a); let (b0, b1) = self.split_i64x4(b); @@ -9825,6 +9888,33 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x4, indices: u8x32) -> u64x4 { + let bytes = token.cvt_to_bytes_u64x4(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + token.cvt_from_bytes_u64x4(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { let (a0, a1) = self.split_u64x4(a); let (b0, b1) = self.split_u64x4(b); @@ -16819,189 +16909,453 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.add_i64x4(a0, b0), self.add_i64x4(a1, b1)) - } - #[inline(always)] - fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.sub_i64x4(a0, b0), self.sub_i64x4(a1, b1)) - } - #[inline(always)] - fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1)) - } - #[inline(always)] - fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.and_i64x4(a0, b0), self.and_i64x4(a1, b1)) - } - #[inline(always)] - fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.or_i64x4(a0, b0), self.or_i64x4(a1, b1)) - } - #[inline(always)] - fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.xor_i64x4(a0, b0), self.xor_i64x4(a1, b1)) - } - #[inline(always)] - fn not_i64x8(self, a: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.not_i64x4(a0), self.not_i64x4(a1)) - } - #[inline(always)] - fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.shl_i64x4(a0, shift), self.shl_i64x4(a1, shift)) - } - #[inline(always)] - fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.shlv_i64x4(a0, b0), self.shlv_i64x4(a1, b1)) - } - #[inline(always)] - fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.shr_i64x4(a0, shift), self.shr_i64x4(a1, shift)) - } - #[inline(always)] - fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.shrv_i64x4(a0, b0), self.shrv_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_eq_i64x4(a0, b0), self.simd_eq_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_lt_i64x4(a0, b0), self.simd_lt_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_le_i64x4(a0, b0), self.simd_le_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_ge_i64x4(a0, b0), self.simd_ge_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_gt_i64x4(a0, b0), self.simd_gt_i64x4(a1, b1)) - } - #[inline(always)] - fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, _) = self.split_i64x8(a); - let (b0, _) = self.split_i64x8(b); - self.combine_i64x4(self.zip_low_i64x4(a0, b0), self.zip_high_i64x4(a0, b0)) - } - #[inline(always)] - fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (_, a1) = self.split_i64x8(a); - let (_, b1) = self.split_i64x8(b); - self.combine_i64x4(self.zip_low_i64x4(a1, b1), self.zip_high_i64x4(a1, b1)) - } - #[inline(always)] - fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.unzip_low_i64x4(a0, a1), self.unzip_low_i64x4(b0, b1)) - } - #[inline(always)] - fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.unzip_high_i64x4(a0, a1), self.unzip_high_i64x4(b0, b1)) - } - #[inline(always)] - fn interleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - let lo_lo = self.zip_low_i64x4(a0, b0); - let lo_hi = self.zip_high_i64x4(a0, b0); - let hi_lo = self.zip_low_i64x4(a1, b1); - let hi_hi = self.zip_high_i64x4(a1, b1); - ( - self.combine_i64x4(lo_lo, lo_hi), - self.combine_i64x4(hi_lo, hi_hi), - ) - } - #[inline(always)] - fn deinterleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - let lo_even = self.unzip_low_i64x4(a0, a1); - let lo_odd = self.unzip_high_i64x4(a0, a1); - let hi_even = self.unzip_low_i64x4(b0, b1); - let hi_odd = self.unzip_high_i64x4(b0, b1); - ( - self.combine_i64x4(lo_even, hi_even), - self.combine_i64x4(lo_odd, hi_odd), - ) - } - #[inline(always)] - fn select_i64x8(self, a: mask64x8, b: i64x8, c: i64x8) -> i64x8 { - let (a0, a1) = self.split_mask64x8(a); - let (b0, b1) = self.split_i64x8(b); - let (c0, c1) = self.split_i64x8(c); - self.combine_i64x4(self.select_i64x4(a0, b0, c0), self.select_i64x4(a1, b1, c1)) - } - #[inline(always)] - fn min_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.min_i64x4(a0, b0), self.min_i64x4(a1, b1)) - } - #[inline(always)] - fn max_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.max_i64x4(a0, b0), self.max_i64x4(a1, b1)) - } - #[inline(always)] - fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { - ( - i64x4 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), - simd: self, + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + let bytes = self.cvt_to_bytes_i64x8(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) }, - i64x4 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), - simd: self, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) }, - ) - } - #[inline(always)] - fn neg_i64x8(self, a: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i64x8(result) } #[inline(always)] - fn reinterpret_u8_i64x8(self, a: i64x8) -> u8x64 { + fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { let (a0, a1) = self.split_i64x8(a); - self.combine_u8x32(self.reinterpret_u8_i64x4(a0), self.reinterpret_u8_i64x4(a1)) + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.add_i64x4(a0, b0), self.add_i64x4(a1, b1)) + } + #[inline(always)] + fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.sub_i64x4(a0, b0), self.sub_i64x4(a1, b1)) + } + #[inline(always)] + fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1)) + } + #[inline(always)] + fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.and_i64x4(a0, b0), self.and_i64x4(a1, b1)) + } + #[inline(always)] + fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.or_i64x4(a0, b0), self.or_i64x4(a1, b1)) + } + #[inline(always)] + fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.xor_i64x4(a0, b0), self.xor_i64x4(a1, b1)) + } + #[inline(always)] + fn not_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.not_i64x4(a0), self.not_i64x4(a1)) + } + #[inline(always)] + fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shl_i64x4(a0, shift), self.shl_i64x4(a1, shift)) + } + #[inline(always)] + fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shlv_i64x4(a0, b0), self.shlv_i64x4(a1, b1)) + } + #[inline(always)] + fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shr_i64x4(a0, shift), self.shr_i64x4(a1, shift)) + } + #[inline(always)] + fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shrv_i64x4(a0, b0), self.shrv_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_eq_i64x4(a0, b0), self.simd_eq_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_lt_i64x4(a0, b0), self.simd_lt_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_le_i64x4(a0, b0), self.simd_le_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_ge_i64x4(a0, b0), self.simd_ge_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_gt_i64x4(a0, b0), self.simd_gt_i64x4(a1, b1)) + } + #[inline(always)] + fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, _) = self.split_i64x8(a); + let (b0, _) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a0, b0), self.zip_high_i64x4(a0, b0)) + } + #[inline(always)] + fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (_, a1) = self.split_i64x8(a); + let (_, b1) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a1, b1), self.zip_high_i64x4(a1, b1)) + } + #[inline(always)] + fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_low_i64x4(a0, a1), self.unzip_low_i64x4(b0, b1)) + } + #[inline(always)] + fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_high_i64x4(a0, a1), self.unzip_high_i64x4(b0, b1)) + } + #[inline(always)] + fn interleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_lo = self.zip_low_i64x4(a0, b0); + let lo_hi = self.zip_high_i64x4(a0, b0); + let hi_lo = self.zip_low_i64x4(a1, b1); + let hi_hi = self.zip_high_i64x4(a1, b1); + ( + self.combine_i64x4(lo_lo, lo_hi), + self.combine_i64x4(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_even = self.unzip_low_i64x4(a0, a1); + let lo_odd = self.unzip_high_i64x4(a0, a1); + let hi_even = self.unzip_low_i64x4(b0, b1); + let hi_odd = self.unzip_high_i64x4(b0, b1); + ( + self.combine_i64x4(lo_even, hi_even), + self.combine_i64x4(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_i64x8(self, a: mask64x8, b: i64x8, c: i64x8) -> i64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_i64x8(b); + let (c0, c1) = self.split_i64x8(c); + self.combine_i64x4(self.select_i64x4(a0, b0, c0), self.select_i64x4(a1, b1, c1)) + } + #[inline(always)] + fn min_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.min_i64x4(a0, b0), self.min_i64x4(a1, b1)) + } + #[inline(always)] + fn max_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.max_i64x4(a0, b0), self.max_i64x4(a1, b1)) + } + #[inline(always)] + fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { + ( + i64x4 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + i64x4 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn neg_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) + } + #[inline(always)] + fn reinterpret_u8_i64x8(self, a: i64x8) -> u8x64 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u8x32(self.reinterpret_u8_i64x4(a0), self.reinterpret_u8_i64x4(a1)) } #[inline(always)] fn reinterpret_u32_i64x8(self, a: i64x8) -> u32x16 { @@ -17167,6 +17521,270 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + let bytes = self.cvt_to_bytes_u64x8(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u64x8(result) + } + #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { let (a0, a1) = self.split_u64x8(a); let (b0, b1) = self.split_u64x8(b); diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index 3b0642c4..e83b038b 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -3065,6 +3065,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + let result = u8x16_swizzle(self.cvt_to_bytes_i64x2(a).val.0, indices.into()); + self.cvt_from_bytes_i64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { i64x2_add(a.into(), b.into()).simd_into(self) } @@ -3330,6 +3338,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + let result = u8x16_swizzle(self.cvt_to_bytes_u64x2(a).val.0, indices.into()); + self.cvt_from_bytes_u64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { u64x2_add(a.into(), b.into()).simd_into(self) } @@ -7446,6 +7462,23 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + let bytes = self.cvt_to_bytes_i64x4(a); + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + self.cvt_from_bytes_i64x4(result_bytes) + } + #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { let (a0, a1) = self.split_i64x4(a); let (b0, b1) = self.split_i64x4(b); @@ -7784,6 +7817,23 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + let bytes = self.cvt_to_bytes_u64x4(a); + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + self.cvt_from_bytes_u64x4(result_bytes) + } + #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { let (a0, a1) = self.split_u64x4(a); let (b0, b1) = self.split_u64x4(b); @@ -14607,223 +14657,487 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.add_i64x4(a0, b0), self.add_i64x4(a1, b1)) - } - #[inline(always)] - fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.sub_i64x4(a0, b0), self.sub_i64x4(a1, b1)) - } - #[inline(always)] - fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1)) - } - #[inline(always)] - fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.and_i64x4(a0, b0), self.and_i64x4(a1, b1)) - } - #[inline(always)] - fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.or_i64x4(a0, b0), self.or_i64x4(a1, b1)) - } - #[inline(always)] - fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.xor_i64x4(a0, b0), self.xor_i64x4(a1, b1)) - } - #[inline(always)] - fn not_i64x8(self, a: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.not_i64x4(a0), self.not_i64x4(a1)) - } - #[inline(always)] - fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.shl_i64x4(a0, shift), self.shl_i64x4(a1, shift)) - } - #[inline(always)] - fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.shlv_i64x4(a0, b0), self.shlv_i64x4(a1, b1)) - } - #[inline(always)] - fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.shr_i64x4(a0, shift), self.shr_i64x4(a1, shift)) - } - #[inline(always)] - fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.shrv_i64x4(a0, b0), self.shrv_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_eq_i64x4(a0, b0), self.simd_eq_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_lt_i64x4(a0, b0), self.simd_lt_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_le_i64x4(a0, b0), self.simd_le_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_ge_i64x4(a0, b0), self.simd_ge_i64x4(a1, b1)) - } - #[inline(always)] - fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_mask64x4(self.simd_gt_i64x4(a0, b0), self.simd_gt_i64x4(a1, b1)) - } - #[inline(always)] - fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, _) = self.split_i64x8(a); - let (b0, _) = self.split_i64x8(b); - self.combine_i64x4(self.zip_low_i64x4(a0, b0), self.zip_high_i64x4(a0, b0)) - } - #[inline(always)] - fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (_, a1) = self.split_i64x8(a); - let (_, b1) = self.split_i64x8(b); - self.combine_i64x4(self.zip_low_i64x4(a1, b1), self.zip_high_i64x4(a1, b1)) - } - #[inline(always)] - fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.unzip_low_i64x4(a0, a1), self.unzip_low_i64x4(b0, b1)) - } - #[inline(always)] - fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.unzip_high_i64x4(a0, a1), self.unzip_high_i64x4(b0, b1)) - } - #[inline(always)] - fn interleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - let lo_lo = self.zip_low_i64x4(a0, b0); - let lo_hi = self.zip_high_i64x4(a0, b0); - let hi_lo = self.zip_low_i64x4(a1, b1); - let hi_hi = self.zip_high_i64x4(a1, b1); - ( - self.combine_i64x4(lo_lo, lo_hi), - self.combine_i64x4(hi_lo, hi_hi), - ) - } - #[inline(always)] - fn deinterleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - let lo_even = self.unzip_low_i64x4(a0, a1); - let lo_odd = self.unzip_high_i64x4(a0, a1); - let hi_even = self.unzip_low_i64x4(b0, b1); - let hi_odd = self.unzip_high_i64x4(b0, b1); - ( - self.combine_i64x4(lo_even, hi_even), - self.combine_i64x4(lo_odd, hi_odd), - ) - } - #[inline(always)] - fn select_i64x8(self, a: mask64x8, b: i64x8, c: i64x8) -> i64x8 { - let (a0, a1) = self.split_mask64x8(a); - let (b0, b1) = self.split_i64x8(b); - let (c0, c1) = self.split_i64x8(c); - self.combine_i64x4(self.select_i64x4(a0, b0, c0), self.select_i64x4(a1, b1, c1)) - } - #[inline(always)] - fn min_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.min_i64x4(a0, b0), self.min_i64x4(a1, b1)) - } - #[inline(always)] - fn max_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - let (b0, b1) = self.split_i64x8(b); - self.combine_i64x4(self.max_i64x4(a0, b0), self.max_i64x4(a1, b1)) - } - #[inline(always)] - fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { - ( - i64x4 { - val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), - simd: self, + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + let bytes = self.cvt_to_bytes_i64x8(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) }, - i64x4 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), - simd: self, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) }, - ) - } - #[inline(always)] - fn neg_i64x8(self, a: i64x8) -> i64x8 { - let (a0, a1) = self.split_i64x8(a); - self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) - } - #[inline(always)] - fn reinterpret_u8_i64x8(self, a: i64x8) -> u8x64 { - let (a0, a1) = self.split_i64x8(a); - self.combine_u8x32(self.reinterpret_u8_i64x4(a0), self.reinterpret_u8_i64x4(a1)) - } - #[inline(always)] - fn reinterpret_u32_i64x8(self, a: i64x8) -> u32x16 { - let (a0, a1) = self.split_i64x8(a); - self.combine_u32x8( - self.reinterpret_u32_i64x4(a0), - self.reinterpret_u32_i64x4(a1), - ) - } - #[inline(always)] - fn splat_u64x8(self, val: u64) -> u64x8 { - let half = self.splat_u64x4(val); - self.combine_u64x4(half, half) - } - #[inline(always)] - fn load_array_u64x8(self, val: [u64; 8usize]) -> u64x8 { - u64x8 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } - } - #[inline(always)] - fn load_array_ref_u64x8(self, val: &[u64; 8usize]) -> u64x8 { - u64x8 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } - } - #[inline(always)] - fn as_array_u64x8(self, a: u64x8) -> [u64; 8usize] { - crate::transmute::checked_transmute_copy::<[v128; 4usize], [u64; 8usize]>(&a.val.0) + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_i64x8(result) } #[inline(always)] - fn as_array_ref_u64x8(self, a: &u64x8) -> &[u64; 8usize] { + fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.add_i64x4(a0, b0), self.add_i64x4(a1, b1)) + } + #[inline(always)] + fn sub_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.sub_i64x4(a0, b0), self.sub_i64x4(a1, b1)) + } + #[inline(always)] + fn mul_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.mul_i64x4(a0, b0), self.mul_i64x4(a1, b1)) + } + #[inline(always)] + fn and_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.and_i64x4(a0, b0), self.and_i64x4(a1, b1)) + } + #[inline(always)] + fn or_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.or_i64x4(a0, b0), self.or_i64x4(a1, b1)) + } + #[inline(always)] + fn xor_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.xor_i64x4(a0, b0), self.xor_i64x4(a1, b1)) + } + #[inline(always)] + fn not_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.not_i64x4(a0), self.not_i64x4(a1)) + } + #[inline(always)] + fn shl_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shl_i64x4(a0, shift), self.shl_i64x4(a1, shift)) + } + #[inline(always)] + fn shlv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shlv_i64x4(a0, b0), self.shlv_i64x4(a1, b1)) + } + #[inline(always)] + fn shr_i64x8(self, a: i64x8, shift: u32) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.shr_i64x4(a0, shift), self.shr_i64x4(a1, shift)) + } + #[inline(always)] + fn shrv_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.shrv_i64x4(a0, b0), self.shrv_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_eq_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_eq_i64x4(a0, b0), self.simd_eq_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_lt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_lt_i64x4(a0, b0), self.simd_lt_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_le_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_le_i64x4(a0, b0), self.simd_le_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_ge_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_ge_i64x4(a0, b0), self.simd_ge_i64x4(a1, b1)) + } + #[inline(always)] + fn simd_gt_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_mask64x4(self.simd_gt_i64x4(a0, b0), self.simd_gt_i64x4(a1, b1)) + } + #[inline(always)] + fn zip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, _) = self.split_i64x8(a); + let (b0, _) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a0, b0), self.zip_high_i64x4(a0, b0)) + } + #[inline(always)] + fn zip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (_, a1) = self.split_i64x8(a); + let (_, b1) = self.split_i64x8(b); + self.combine_i64x4(self.zip_low_i64x4(a1, b1), self.zip_high_i64x4(a1, b1)) + } + #[inline(always)] + fn unzip_low_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_low_i64x4(a0, a1), self.unzip_low_i64x4(b0, b1)) + } + #[inline(always)] + fn unzip_high_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.unzip_high_i64x4(a0, a1), self.unzip_high_i64x4(b0, b1)) + } + #[inline(always)] + fn interleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_lo = self.zip_low_i64x4(a0, b0); + let lo_hi = self.zip_high_i64x4(a0, b0); + let hi_lo = self.zip_low_i64x4(a1, b1); + let hi_hi = self.zip_high_i64x4(a1, b1); + ( + self.combine_i64x4(lo_lo, lo_hi), + self.combine_i64x4(hi_lo, hi_hi), + ) + } + #[inline(always)] + fn deinterleave_i64x8(self, a: i64x8, b: i64x8) -> (i64x8, i64x8) { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + let lo_even = self.unzip_low_i64x4(a0, a1); + let lo_odd = self.unzip_high_i64x4(a0, a1); + let hi_even = self.unzip_low_i64x4(b0, b1); + let hi_odd = self.unzip_high_i64x4(b0, b1); + ( + self.combine_i64x4(lo_even, hi_even), + self.combine_i64x4(lo_odd, hi_odd), + ) + } + #[inline(always)] + fn select_i64x8(self, a: mask64x8, b: i64x8, c: i64x8) -> i64x8 { + let (a0, a1) = self.split_mask64x8(a); + let (b0, b1) = self.split_i64x8(b); + let (c0, c1) = self.split_i64x8(c); + self.combine_i64x4(self.select_i64x4(a0, b0, c0), self.select_i64x4(a1, b1, c1)) + } + #[inline(always)] + fn min_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.min_i64x4(a0, b0), self.min_i64x4(a1, b1)) + } + #[inline(always)] + fn max_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + let (b0, b1) = self.split_i64x8(b); + self.combine_i64x4(self.max_i64x4(a0, b0), self.max_i64x4(a1, b1)) + } + #[inline(always)] + fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { + ( + i64x4 { + val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), + simd: self, + }, + i64x4 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) + } + #[inline(always)] + fn neg_i64x8(self, a: i64x8) -> i64x8 { + let (a0, a1) = self.split_i64x8(a); + self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) + } + #[inline(always)] + fn reinterpret_u8_i64x8(self, a: i64x8) -> u8x64 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u8x32(self.reinterpret_u8_i64x4(a0), self.reinterpret_u8_i64x4(a1)) + } + #[inline(always)] + fn reinterpret_u32_i64x8(self, a: i64x8) -> u32x16 { + let (a0, a1) = self.split_i64x8(a); + self.combine_u32x8( + self.reinterpret_u32_i64x4(a0), + self.reinterpret_u32_i64x4(a1), + ) + } + #[inline(always)] + fn splat_u64x8(self, val: u64) -> u64x8 { + let half = self.splat_u64x4(val); + self.combine_u64x4(half, half) + } + #[inline(always)] + fn load_array_u64x8(self, val: [u64; 8usize]) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u64x8(self, val: &[u64; 8usize]) -> u64x8 { + u64x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u64x8(self, a: u64x8) -> [u64; 8usize] { + crate::transmute::checked_transmute_copy::<[v128; 4usize], [u64; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u64x8(self, a: &u64x8) -> &[u64; 8usize] { crate::transmute::checked_cast_ref::<[v128; 4usize], [u64; 8usize]>(&a.val.0) } #[inline(always)] @@ -14954,6 +15268,270 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + let bytes = self.cvt_to_bytes_u64x8(a); + let result: u8x64 = [ + { + let index = indices[0usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[1usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[2usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[3usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[4usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[5usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[6usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[7usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[8usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[9usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[10usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[11usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[12usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[13usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[14usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[15usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[16usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[17usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[18usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[19usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[20usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[21usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[22usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[23usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[24usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[25usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[26usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[27usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[28usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[29usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[30usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[31usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[32usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[33usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[34usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[35usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[36usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[37usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[38usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[39usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[40usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[41usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[42usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[43usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[44usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[45usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[46usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[47usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[48usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[49usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[50usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[51usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[52usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[53usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[54usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[55usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[56usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[57usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[58usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[59usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[60usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[61usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[62usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + { + let index = indices[63usize] as usize; + bytes.get(index).copied().unwrap_or(0) + }, + ] + .simd_into(self); + self.cvt_from_bytes_u64x8(result) + } + #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { let (a0, a1) = self.split_u64x8(a); let (b0, b1) = self.split_u64x8(b); diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 9ba62f57..6a7bce6c 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2784,26 +2784,22 @@ impl X86 { recursive_swizzle_dyn_precise_body(vec_ty, token) } (Self::Avx2, 256) => quote! { - // carefully tuned implementation reused from std::simd: - // https://github.com/rust-lang/portable-simd/blob/7d497cca160ae6062acc1a2db838667f83c0b58e/crates/core_simd/src/swizzle_dyn.rs#L205-L224 let bytes = #token.#to_bytes(a); let idxs = indices; - let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); - let hi_shuf = #bytes { - val: #wrapper(_mm256_shuffle_epi8(hihi, idxs.into())), - simd: #token, - }; - let result_bytes = idxs - .simd_lt(#bytes::splat(#token, 32)) - .select(hi_shuf, #bytes::splat(#token, 0)); let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); - let lo_shuf = #bytes { - val: #wrapper(_mm256_shuffle_epi8(lolo, idxs.into())), - simd: #token, - }; - let result_bytes = idxs - .simd_lt(#bytes::splat(#token, 16)) - .select(lo_shuf, result_bytes); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + + // Adding 0x60 preserves the low nibble and bit 4 for valid + // indices 0..=31. Larger indices get their high bit set, so + // VPSHUFB supplies the required out-of-bounds zeroing. + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + + // Move index bit 4 into each byte's sign bit for VPBLENDVB. + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = #bytes { val: #wrapper(result), simd: #token }; }, (Self::Avx512, 128 | 256 | 512) => { let cmp = intrinsic_ident("cmp", "epu8_mask", vec_ty.n_bits()); diff --git a/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs b/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs index 5dd8933d..36113588 100644 --- a/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs +++ b/fearless_simd_tests/tests/harness/lm_generated/swizzle_dyn_precise.rs @@ -40,6 +40,22 @@ fn swizzle_dyn_precise_u8x32_crosses_blocks(simd: S) { assert_eq!(*result, expected); } +#[simd_test] +fn swizzle_dyn_precise_u8x32_all_indices(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| u8::try_from(i + 1).unwrap()); + + for start in (0..=224).step_by(32) { + let indices = core::array::from_fn(|i| u8::try_from(start + i).unwrap()); + let expected = expected_swizzle_precise(bytes, indices); + + let value = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let result = value.swizzle_dyn_precise(index_vec); + + assert_eq!(*result, expected); + } +} + #[simd_test] fn swizzle_dyn_precise_u8x64_crosses_blocks(simd: S) { let bytes: [u8; 64] = core::array::from_fn(|i| u8::try_from(i + 1).unwrap()); From ab3fd43092e319be094621bbe4dc5423cd0a4e22 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Thu, 23 Jul 2026 13:51:44 +0100 Subject: [PATCH 15/23] Empty commit to re-run stuck CI From 3e3a00cc0efe1cf7c62e2e7d0ebc7c9d1a0c4587 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sat, 25 Jul 2026 09:30:49 +0100 Subject: [PATCH 16/23] Optimize SSE4.2 as suggested by axnsan12. Changes the control sequence from vpcmpgtb + vpblendvb to vpaddb + vpor --- fearless_simd/src/generated/avx2.rs | 20 ++++++++++---------- fearless_simd/src/generated/sse4_2.rs | 20 ++++++++++---------- fearless_simd_gen/src/mk_x86.rs | 4 +++- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 8ec5648f..f1e2c9f2 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -257,7 +257,7 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: f32x4, indices: u8x16) -> f32x4 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_f32x4(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -911,7 +911,7 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: i8x16, indices: u8x16) -> i8x16 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_i8x16(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -1447,7 +1447,7 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: u8x16, indices: u8x16) -> u8x16 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_u8x16(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -2130,7 +2130,7 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: i16x8, indices: u8x16) -> i16x8 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_i16x8(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -2599,7 +2599,7 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: u16x8, indices: u8x16) -> u16x8 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_u16x8(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -3229,7 +3229,7 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: i32x4, indices: u8x16) -> i32x4 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_i32x4(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -3682,7 +3682,7 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: u32x4, indices: u8x16) -> u32x4 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_u32x4(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -4300,7 +4300,7 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: f64x2, indices: u8x16) -> f64x2 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_f64x2(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -4785,7 +4785,7 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: i64x2, indices: u8x16) -> i64x2 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_i64x2(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -5194,7 +5194,7 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: u64x2, indices: u8x16) -> u64x2 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_u64x2(a).val.0, zeroing_indices); let result_bytes = u8x16 { diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 8c02cd9a..dd574793 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -254,7 +254,7 @@ impl Simd for Sse4_2 { #[inline(always)] fn kernel(token: Sse4_2, a: f32x4, indices: u8x16) -> f32x4 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_f32x4(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -893,7 +893,7 @@ impl Simd for Sse4_2 { #[inline(always)] fn kernel(token: Sse4_2, a: i8x16, indices: u8x16) -> i8x16 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_i8x16(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -1426,7 +1426,7 @@ impl Simd for Sse4_2 { #[inline(always)] fn kernel(token: Sse4_2, a: u8x16, indices: u8x16) -> u8x16 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_u8x16(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -2106,7 +2106,7 @@ impl Simd for Sse4_2 { #[inline(always)] fn kernel(token: Sse4_2, a: i16x8, indices: u8x16) -> i16x8 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_i16x8(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -2572,7 +2572,7 @@ impl Simd for Sse4_2 { #[inline(always)] fn kernel(token: Sse4_2, a: u16x8, indices: u8x16) -> u16x8 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_u16x8(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -3196,7 +3196,7 @@ impl Simd for Sse4_2 { #[inline(always)] fn kernel(token: Sse4_2, a: i32x4, indices: u8x16) -> i32x4 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_i32x4(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -3646,7 +3646,7 @@ impl Simd for Sse4_2 { #[inline(always)] fn kernel(token: Sse4_2, a: u32x4, indices: u8x16) -> u32x4 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_u32x4(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -4258,7 +4258,7 @@ impl Simd for Sse4_2 { #[inline(always)] fn kernel(token: Sse4_2, a: f64x2, indices: u8x16) -> f64x2 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_f64x2(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -4728,7 +4728,7 @@ impl Simd for Sse4_2 { #[inline(always)] fn kernel(token: Sse4_2, a: i64x2, indices: u8x16) -> i64x2 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_i64x2(a).val.0, zeroing_indices); let result_bytes = u8x16 { @@ -5132,7 +5132,7 @@ impl Simd for Sse4_2 { #[inline(always)] fn kernel(token: Sse4_2, a: u64x2, indices: u8x16) -> u64x2 { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(token.cvt_to_bytes_u64x2(a).val.0, zeroing_indices); let result_bytes = u8x16 { diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 6a7bce6c..143f1bbf 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2775,7 +2775,9 @@ impl X86 { let body = match (*self, vec_ty.n_bits()) { (Self::Sse4_2 | Self::Avx2, 128) => quote! { let indices = indices.into(); - let index_out_of_range = _mm_cmpgt_epi8(indices, _mm_set1_epi8(15)); + // Preserve the original high bit, and set it for indices 16..=127. + // The added value only changes bits that PSHUFB ignores for valid indices. + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); let result = _mm_shuffle_epi8(#token.#to_bytes(a).val.0, zeroing_indices); let result_bytes = #bytes { val: #wrapper(result), simd: #token }; From 2d4449fb1c90e9010aed21e1bf08d93701c555a9 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Tue, 28 Jul 2026 20:39:52 +0100 Subject: [PATCH 17/23] Resolve main merge conflicts --- fearless_simd/src/generated/avx2.rs | 120 ++++++++++++------------ fearless_simd/src/generated/avx512.rs | 120 ++++++++++++------------ fearless_simd/src/generated/fallback.rs | 120 ++++++++++++------------ fearless_simd/src/generated/neon.rs | 120 ++++++++++++------------ fearless_simd/src/generated/sse2.rs | 120 ++++++++++++------------ fearless_simd/src/generated/sse4_2.rs | 120 ++++++++++++------------ fearless_simd/src/generated/wasm.rs | 120 ++++++++++++------------ fearless_simd_gen/src/generic.rs | 3 +- fearless_simd_gen/src/mk_fallback.rs | 6 +- fearless_simd_gen/src/mk_neon.rs | 10 +- fearless_simd_gen/src/mk_wasm.rs | 19 +--- fearless_simd_gen/src/mk_x86.rs | 21 +---- fearless_simd_gen/src/ops.rs | 26 +---- 13 files changed, 438 insertions(+), 487 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 4f3421ef..d425a224 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -245,12 +245,12 @@ impl Simd for Avx2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_f32x4(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_f32x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -845,12 +845,12 @@ impl Simd for Avx2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_i8x16(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_i8x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -1348,12 +1348,12 @@ impl Simd for Avx2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_u8x16(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_u8x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -2008,12 +2008,12 @@ impl Simd for Avx2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_i16x8(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_i16x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -2443,12 +2443,12 @@ impl Simd for Avx2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_u16x8(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_u16x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -3039,12 +3039,12 @@ impl Simd for Avx2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_i32x4(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_i32x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -3458,12 +3458,12 @@ impl Simd for Avx2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_u32x4(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_u32x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -4052,12 +4052,12 @@ impl Simd for Avx2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_f64x2(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_f64x2(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -4513,12 +4513,12 @@ impl Simd for Avx2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_i64x2(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_i64x2(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -4899,12 +4899,12 @@ impl Simd for Avx2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_u64x2(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_u64x2(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -5477,7 +5477,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: f32x8, indices: u8x32) -> f32x8 { - let bytes = token.cvt_to_bytes_f32x8(a); + let bytes = Bytes::to_bytes(a); let idxs = indices; let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); @@ -5490,7 +5490,7 @@ impl Simd for Avx2 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_f32x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -6209,7 +6209,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: i8x32, indices: u8x32) -> i8x32 { - let bytes = token.cvt_to_bytes_i8x32(a); + let bytes = Bytes::to_bytes(a); let idxs = indices; let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); @@ -6222,7 +6222,7 @@ impl Simd for Avx2 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_i8x32(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -6899,7 +6899,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: u8x32, indices: u8x32) -> u8x32 { - let bytes = token.cvt_to_bytes_u8x32(a); + let bytes = Bytes::to_bytes(a); let idxs = indices; let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); @@ -6912,7 +6912,7 @@ impl Simd for Avx2 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_u8x32(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -7734,7 +7734,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: i16x16, indices: u8x32) -> i16x16 { - let bytes = token.cvt_to_bytes_i16x16(a); + let bytes = Bytes::to_bytes(a); let idxs = indices; let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); @@ -7747,7 +7747,7 @@ impl Simd for Avx2 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_i16x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -8316,7 +8316,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: u16x16, indices: u8x32) -> u16x16 { - let bytes = token.cvt_to_bytes_u16x16(a); + let bytes = Bytes::to_bytes(a); let idxs = indices; let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); @@ -8329,7 +8329,7 @@ impl Simd for Avx2 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_u16x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -9070,7 +9070,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: i32x8, indices: u8x32) -> i32x8 { - let bytes = token.cvt_to_bytes_i32x8(a); + let bytes = Bytes::to_bytes(a); let idxs = indices; let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); @@ -9083,7 +9083,7 @@ impl Simd for Avx2 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_i32x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -9576,7 +9576,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: u32x8, indices: u8x32) -> u32x8 { - let bytes = token.cvt_to_bytes_u32x8(a); + let bytes = Bytes::to_bytes(a); let idxs = indices; let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); @@ -9589,7 +9589,7 @@ impl Simd for Avx2 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_u32x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -10261,7 +10261,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: f64x4, indices: u8x32) -> f64x4 { - let bytes = token.cvt_to_bytes_f64x4(a); + let bytes = Bytes::to_bytes(a); let idxs = indices; let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); @@ -10274,7 +10274,7 @@ impl Simd for Avx2 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_f64x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -10790,7 +10790,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: i64x4, indices: u8x32) -> i64x4 { - let bytes = token.cvt_to_bytes_i64x4(a); + let bytes = Bytes::to_bytes(a); let idxs = indices; let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); @@ -10803,7 +10803,7 @@ impl Simd for Avx2 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_i64x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -11258,7 +11258,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: u64x4, indices: u8x32) -> u64x4 { - let bytes = token.cvt_to_bytes_u64x4(a); + let bytes = Bytes::to_bytes(a); let idxs = indices; let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); @@ -11271,7 +11271,7 @@ impl Simd for Avx2 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_u64x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -11927,7 +11927,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: f32x16, indices: u8x64) -> f32x16 { - let bytes = token.cvt_to_bytes_f32x16(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x64(bytes); let (indices_low, indices_high) = token.split_u8x64(indices); let high_table_offset = token.splat_u8x32(32); @@ -11944,7 +11944,7 @@ impl Simd for Avx2 { ); let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x32(output_low, output_high); - token.cvt_from_bytes_f32x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -12651,7 +12651,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: i8x64, indices: u8x64) -> i8x64 { - let bytes = token.cvt_to_bytes_i8x64(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x64(bytes); let (indices_low, indices_high) = token.split_u8x64(indices); let high_table_offset = token.splat_u8x32(32); @@ -12668,7 +12668,7 @@ impl Simd for Avx2 { ); let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x32(output_low, output_high); - token.cvt_from_bytes_i8x64(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -13223,7 +13223,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: u8x64, indices: u8x64) -> u8x64 { - let bytes = token.cvt_to_bytes_u8x64(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x64(bytes); let (indices_low, indices_high) = token.split_u8x64(indices); let high_table_offset = token.splat_u8x32(32); @@ -13240,7 +13240,7 @@ impl Simd for Avx2 { ); let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x32(output_low, output_high); - token.cvt_from_bytes_u8x64(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -13903,7 +13903,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: i16x32, indices: u8x64) -> i16x32 { - let bytes = token.cvt_to_bytes_i16x32(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x64(bytes); let (indices_low, indices_high) = token.split_u8x64(indices); let high_table_offset = token.splat_u8x32(32); @@ -13920,7 +13920,7 @@ impl Simd for Avx2 { ); let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x32(output_low, output_high); - token.cvt_from_bytes_i16x32(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -14357,7 +14357,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: u16x32, indices: u8x64) -> u16x32 { - let bytes = token.cvt_to_bytes_u16x32(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x64(bytes); let (indices_low, indices_high) = token.split_u8x64(indices); let high_table_offset = token.splat_u8x32(32); @@ -14374,7 +14374,7 @@ impl Simd for Avx2 { ); let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x32(output_low, output_high); - token.cvt_from_bytes_u16x32(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -14973,7 +14973,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: i32x16, indices: u8x64) -> i32x16 { - let bytes = token.cvt_to_bytes_i32x16(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x64(bytes); let (indices_low, indices_high) = token.split_u8x64(indices); let high_table_offset = token.splat_u8x32(32); @@ -14990,7 +14990,7 @@ impl Simd for Avx2 { ); let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x32(output_low, output_high); - token.cvt_from_bytes_i32x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -15362,7 +15362,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: u32x16, indices: u8x64) -> u32x16 { - let bytes = token.cvt_to_bytes_u32x16(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x64(bytes); let (indices_low, indices_high) = token.split_u8x64(indices); let high_table_offset = token.splat_u8x32(32); @@ -15379,7 +15379,7 @@ impl Simd for Avx2 { ); let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x32(output_low, output_high); - token.cvt_from_bytes_u32x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -15923,7 +15923,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: f64x8, indices: u8x64) -> f64x8 { - let bytes = token.cvt_to_bytes_f64x8(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x64(bytes); let (indices_low, indices_high) = token.split_u8x64(indices); let high_table_offset = token.splat_u8x32(32); @@ -15940,7 +15940,7 @@ impl Simd for Avx2 { ); let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x32(output_low, output_high); - token.cvt_from_bytes_f64x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -16322,7 +16322,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: i64x8, indices: u8x64) -> i64x8 { - let bytes = token.cvt_to_bytes_i64x8(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x64(bytes); let (indices_low, indices_high) = token.split_u8x64(indices); let high_table_offset = token.splat_u8x32(32); @@ -16339,7 +16339,7 @@ impl Simd for Avx2 { ); let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x32(output_low, output_high); - token.cvt_from_bytes_i64x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -16670,7 +16670,7 @@ impl Simd for Avx2 { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: u64x8, indices: u8x64) -> u64x8 { - let bytes = token.cvt_to_bytes_u64x8(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x64(bytes); let (indices_low, indices_high) = token.split_u8x64(indices); let high_table_offset = token.splat_u8x32(32); @@ -16687,7 +16687,7 @@ impl Simd for Avx2 { ); let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x32(output_low, output_high); - token.cvt_from_bytes_u64x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index a02587d3..bded67fe 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -250,7 +250,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: f32x4, indices: u8x16) -> f32x4 { - let bytes = token.cvt_to_bytes_f32x4(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); @@ -258,7 +258,7 @@ impl Simd for Avx512 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_f32x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -848,7 +848,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: i8x16, indices: u8x16) -> i8x16 { - let bytes = token.cvt_to_bytes_i8x16(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); @@ -856,7 +856,7 @@ impl Simd for Avx512 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_i8x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -1394,7 +1394,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: u8x16, indices: u8x16) -> u8x16 { - let bytes = token.cvt_to_bytes_u8x16(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); @@ -1402,7 +1402,7 @@ impl Simd for Avx512 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_u8x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -2043,7 +2043,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: i16x8, indices: u8x16) -> i16x8 { - let bytes = token.cvt_to_bytes_i16x8(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); @@ -2051,7 +2051,7 @@ impl Simd for Avx512 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_i16x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -2507,7 +2507,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: u16x8, indices: u8x16) -> u16x8 { - let bytes = token.cvt_to_bytes_u16x8(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); @@ -2515,7 +2515,7 @@ impl Simd for Avx512 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_u16x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -3080,7 +3080,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: i32x4, indices: u8x16) -> i32x4 { - let bytes = token.cvt_to_bytes_i32x4(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); @@ -3088,7 +3088,7 @@ impl Simd for Avx512 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_i32x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -3528,7 +3528,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: u32x4, indices: u8x16) -> u32x4 { - let bytes = token.cvt_to_bytes_u32x4(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); @@ -3536,7 +3536,7 @@ impl Simd for Avx512 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_u32x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -4094,7 +4094,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: f64x2, indices: u8x16) -> f64x2 { - let bytes = token.cvt_to_bytes_f64x2(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); @@ -4102,7 +4102,7 @@ impl Simd for Avx512 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_f64x2(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -4583,7 +4583,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: i64x2, indices: u8x16) -> i64x2 { - let bytes = token.cvt_to_bytes_i64x2(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); @@ -4591,7 +4591,7 @@ impl Simd for Avx512 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_i64x2(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -5011,7 +5011,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: u64x2, indices: u8x16) -> u64x2 { - let bytes = token.cvt_to_bytes_u64x2(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); @@ -5019,7 +5019,7 @@ impl Simd for Avx512 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_u64x2(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -5617,7 +5617,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: f32x8, indices: u8x32) -> f32x8 { - let bytes = token.cvt_to_bytes_f32x8(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); @@ -5626,7 +5626,7 @@ impl Simd for Avx512 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_f32x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -6381,7 +6381,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: i8x32, indices: u8x32) -> i8x32 { - let bytes = token.cvt_to_bytes_i8x32(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); @@ -6390,7 +6390,7 @@ impl Simd for Avx512 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_i8x32(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -7101,7 +7101,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: u8x32, indices: u8x32) -> u8x32 { - let bytes = token.cvt_to_bytes_u8x32(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); @@ -7110,7 +7110,7 @@ impl Simd for Avx512 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_u8x32(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -7908,7 +7908,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: i16x16, indices: u8x32) -> i16x16 { - let bytes = token.cvt_to_bytes_i16x16(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); @@ -7917,7 +7917,7 @@ impl Simd for Avx512 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_i16x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -8500,7 +8500,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: u16x16, indices: u8x32) -> u16x16 { - let bytes = token.cvt_to_bytes_u16x16(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); @@ -8509,7 +8509,7 @@ impl Simd for Avx512 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_u16x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -9205,7 +9205,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: i32x8, indices: u8x32) -> i32x8 { - let bytes = token.cvt_to_bytes_i32x8(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); @@ -9214,7 +9214,7 @@ impl Simd for Avx512 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_i32x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -9749,7 +9749,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: u32x8, indices: u8x32) -> u32x8 { - let bytes = token.cvt_to_bytes_u32x8(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); @@ -9758,7 +9758,7 @@ impl Simd for Avx512 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_u32x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -10417,7 +10417,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: f64x4, indices: u8x32) -> f64x4 { - let bytes = token.cvt_to_bytes_f64x4(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); @@ -10426,7 +10426,7 @@ impl Simd for Avx512 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_f64x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -10994,7 +10994,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: i64x4, indices: u8x32) -> i64x4 { - let bytes = token.cvt_to_bytes_i64x4(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); @@ -11003,7 +11003,7 @@ impl Simd for Avx512 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_i64x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -11496,7 +11496,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: u64x4, indices: u8x32) -> u64x4 { - let bytes = token.cvt_to_bytes_u64x4(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); @@ -11505,7 +11505,7 @@ impl Simd for Avx512 { val: crate::support::Aligned256(result), simd: token, }; - token.cvt_from_bytes_u64x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -12190,7 +12190,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: f32x16, indices: u8x64) -> f32x16 { - let bytes = token.cvt_to_bytes_f32x16(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); @@ -12199,7 +12199,7 @@ impl Simd for Avx512 { val: crate::support::Aligned512(result), simd: token, }; - token.cvt_from_bytes_f32x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -13125,7 +13125,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: i8x64, indices: u8x64) -> i8x64 { - let bytes = token.cvt_to_bytes_i8x64(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); @@ -13134,7 +13134,7 @@ impl Simd for Avx512 { val: crate::support::Aligned512(result), simd: token, }; - token.cvt_from_bytes_i8x64(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -13984,7 +13984,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: u8x64, indices: u8x64) -> u8x64 { - let bytes = token.cvt_to_bytes_u8x64(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); @@ -13993,7 +13993,7 @@ impl Simd for Avx512 { val: crate::support::Aligned512(result), simd: token, }; - token.cvt_from_bytes_u8x64(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -14884,7 +14884,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: i16x32, indices: u8x64) -> i16x32 { - let bytes = token.cvt_to_bytes_i16x32(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); @@ -14893,7 +14893,7 @@ impl Simd for Avx512 { val: crate::support::Aligned512(result), simd: token, }; - token.cvt_from_bytes_i16x32(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -15549,7 +15549,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: u16x32, indices: u8x64) -> u16x32 { - let bytes = token.cvt_to_bytes_u16x32(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); @@ -15558,7 +15558,7 @@ impl Simd for Avx512 { val: crate::support::Aligned512(result), simd: token, }; - token.cvt_from_bytes_u16x32(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -16327,7 +16327,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: i32x16, indices: u8x64) -> i32x16 { - let bytes = token.cvt_to_bytes_i32x16(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); @@ -16336,7 +16336,7 @@ impl Simd for Avx512 { val: crate::support::Aligned512(result), simd: token, }; - token.cvt_from_bytes_i32x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -16920,7 +16920,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: u32x16, indices: u8x64) -> u32x16 { - let bytes = token.cvt_to_bytes_u32x16(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); @@ -16929,7 +16929,7 @@ impl Simd for Avx512 { val: crate::support::Aligned512(result), simd: token, }; - token.cvt_from_bytes_u32x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -17638,7 +17638,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: f64x8, indices: u8x64) -> f64x8 { - let bytes = token.cvt_to_bytes_f64x8(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); @@ -17647,7 +17647,7 @@ impl Simd for Avx512 { val: crate::support::Aligned512(result), simd: token, }; - token.cvt_from_bytes_f64x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -18242,7 +18242,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: i64x8, indices: u8x64) -> i64x8 { - let bytes = token.cvt_to_bytes_i64x8(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); @@ -18251,7 +18251,7 @@ impl Simd for Avx512 { val: crate::support::Aligned512(result), simd: token, }; - token.cvt_from_bytes_i64x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -18767,7 +18767,7 @@ impl Simd for Avx512 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: u64x8, indices: u8x64) -> u64x8 { - let bytes = token.cvt_to_bytes_u64x8(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); @@ -18776,7 +18776,7 @@ impl Simd for Avx512 { val: crate::support::Aligned512(result), simd: token, }; - token.cvt_from_bytes_u64x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index 714ed8dd..31735197 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -338,7 +338,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { - let bytes = self.cvt_to_bytes_f32x4(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -406,7 +406,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_f32x4(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { @@ -960,7 +960,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { - let bytes = self.cvt_to_bytes_i8x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -1028,7 +1028,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_i8x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { @@ -1807,7 +1807,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { - let bytes = self.cvt_to_bytes_u8x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -1875,7 +1875,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_u8x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { @@ -2968,7 +2968,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { - let bytes = self.cvt_to_bytes_i16x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -3036,7 +3036,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_i16x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { @@ -3584,7 +3584,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { - let bytes = self.cvt_to_bytes_u16x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -3652,7 +3652,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_u16x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { @@ -4396,7 +4396,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { - let bytes = self.cvt_to_bytes_i32x4(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -4464,7 +4464,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_i32x4(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { @@ -4898,7 +4898,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { - let bytes = self.cvt_to_bytes_u32x4(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -4966,7 +4966,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_u32x4(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { @@ -5536,7 +5536,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { - let bytes = self.cvt_to_bytes_f64x2(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -5604,7 +5604,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_f64x2(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { @@ -5980,7 +5980,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { - let bytes = self.cvt_to_bytes_i64x2(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -6048,7 +6048,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_i64x2(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { @@ -6408,7 +6408,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { - let bytes = self.cvt_to_bytes_u64x2(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -6476,7 +6476,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_u64x2(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { @@ -6927,7 +6927,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { - let bytes = self.cvt_to_bytes_f32x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -7059,7 +7059,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_f32x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { @@ -7550,7 +7550,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { - let bytes = self.cvt_to_bytes_i8x32(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -7682,7 +7682,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_i8x32(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { @@ -8096,7 +8096,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { - let bytes = self.cvt_to_bytes_u8x32(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -8228,7 +8228,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_u8x32(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { @@ -8701,7 +8701,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { - let bytes = self.cvt_to_bytes_i16x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -8833,7 +8833,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_i16x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { @@ -9187,7 +9187,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { - let bytes = self.cvt_to_bytes_u16x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -9319,7 +9319,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_u16x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { @@ -9773,7 +9773,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { - let bytes = self.cvt_to_bytes_i32x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -9905,7 +9905,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_i32x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { @@ -10228,7 +10228,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { - let bytes = self.cvt_to_bytes_u32x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -10360,7 +10360,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_u32x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { @@ -10781,7 +10781,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { - let bytes = self.cvt_to_bytes_f64x4(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -10913,7 +10913,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_f64x4(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { @@ -11266,7 +11266,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { - let bytes = self.cvt_to_bytes_i64x4(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -11398,7 +11398,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_i64x4(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { @@ -11700,7 +11700,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { - let bytes = self.cvt_to_bytes_u64x4(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -11832,7 +11832,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_u64x4(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { @@ -12300,7 +12300,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { - let bytes = self.cvt_to_bytes_f32x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -12560,7 +12560,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_f32x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { @@ -13202,7 +13202,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { - let bytes = self.cvt_to_bytes_i8x64(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -13462,7 +13462,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_i8x64(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { @@ -13997,7 +13997,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { - let bytes = self.cvt_to_bytes_u8x64(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -14257,7 +14257,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_u8x64(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { @@ -14860,7 +14860,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { - let bytes = self.cvt_to_bytes_i16x32(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -15120,7 +15120,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_i16x32(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { @@ -15537,7 +15537,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { - let bytes = self.cvt_to_bytes_u16x32(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -15797,7 +15797,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_u16x32(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { @@ -16313,7 +16313,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { - let bytes = self.cvt_to_bytes_i32x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -16573,7 +16573,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_i32x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { @@ -16925,7 +16925,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { - let bytes = self.cvt_to_bytes_u32x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -17185,7 +17185,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_u32x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { @@ -17638,7 +17638,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { - let bytes = self.cvt_to_bytes_f64x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -17898,7 +17898,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_f64x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { @@ -18260,7 +18260,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { - let bytes = self.cvt_to_bytes_i64x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -18520,7 +18520,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_i64x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { @@ -18831,7 +18831,7 @@ impl Simd for Fallback { } #[inline(always)] fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { - let bytes = self.cvt_to_bytes_u64x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -19091,7 +19091,7 @@ impl Simd for Fallback { }, ] .simd_into(self); - self.cvt_from_bytes_u64x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index df974181..3cb8d5ef 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -229,8 +229,8 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: f32x4, indices: u8x16) -> f32x4 { - let result = vqtbl1q_u8(token.cvt_to_bytes_f32x4(a).val.0, indices.into()); - token.cvt_from_bytes_f32x4(u8x16 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: token, }) @@ -780,8 +780,8 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: i8x16, indices: u8x16) -> i8x16 { - let result = vqtbl1q_u8(token.cvt_to_bytes_i8x16(a).val.0, indices.into()); - token.cvt_from_bytes_i8x16(u8x16 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: token, }) @@ -1240,8 +1240,8 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: u8x16, indices: u8x16) -> u8x16 { - let result = vqtbl1q_u8(token.cvt_to_bytes_u8x16(a).val.0, indices.into()); - token.cvt_from_bytes_u8x16(u8x16 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: token, }) @@ -1857,8 +1857,8 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: i16x8, indices: u8x16) -> i16x8 { - let result = vqtbl1q_u8(token.cvt_to_bytes_i16x8(a).val.0, indices.into()); - token.cvt_from_bytes_i16x8(u8x16 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: token, }) @@ -2285,8 +2285,8 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: u16x8, indices: u8x16) -> u16x8 { - let result = vqtbl1q_u8(token.cvt_to_bytes_u16x8(a).val.0, indices.into()); - token.cvt_from_bytes_u16x8(u8x16 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: token, }) @@ -2866,8 +2866,8 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: i32x4, indices: u8x16) -> i32x4 { - let result = vqtbl1q_u8(token.cvt_to_bytes_i32x4(a).val.0, indices.into()); - token.cvt_from_bytes_i32x4(u8x16 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: token, }) @@ -3288,8 +3288,8 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: u32x4, indices: u8x16) -> u32x4 { - let result = vqtbl1q_u8(token.cvt_to_bytes_u32x4(a).val.0, indices.into()); - token.cvt_from_bytes_u32x4(u8x16 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: token, }) @@ -3870,8 +3870,8 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: f64x2, indices: u8x16) -> f64x2 { - let result = vqtbl1q_u8(token.cvt_to_bytes_f64x2(a).val.0, indices.into()); - token.cvt_from_bytes_f64x2(u8x16 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: token, }) @@ -4337,8 +4337,8 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: i64x2, indices: u8x16) -> i64x2 { - let result = vqtbl1q_u8(token.cvt_to_bytes_i64x2(a).val.0, indices.into()); - token.cvt_from_bytes_i64x2(u8x16 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: token, }) @@ -4735,8 +4735,8 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: u64x2, indices: u8x16) -> u64x2 { - let result = vqtbl1q_u8(token.cvt_to_bytes_u64x2(a).val.0, indices.into()); - token.cvt_from_bytes_u64x2(u8x16 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: token, }) @@ -5340,11 +5340,11 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: f32x8, indices: u8x32) -> f32x8 { - let table = token.cvt_to_bytes_f32x8(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x2_t = indices.into(); let result = uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); - token.cvt_from_bytes_f32x8(u8x32 { + Bytes::from_bytes(u8x32 { val: crate::support::Aligned256(result), simd: token, }) @@ -5881,11 +5881,11 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: i8x32, indices: u8x32) -> i8x32 { - let table = token.cvt_to_bytes_i8x32(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x2_t = indices.into(); let result = uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); - token.cvt_from_bytes_i8x32(u8x32 { + Bytes::from_bytes(u8x32 { val: crate::support::Aligned256(result), simd: token, }) @@ -6345,11 +6345,11 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: u8x32, indices: u8x32) -> u8x32 { - let table = token.cvt_to_bytes_u8x32(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x2_t = indices.into(); let result = uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); - token.cvt_from_bytes_u8x32(u8x32 { + Bytes::from_bytes(u8x32 { val: crate::support::Aligned256(result), simd: token, }) @@ -6875,11 +6875,11 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: i16x16, indices: u8x32) -> i16x16 { - let table = token.cvt_to_bytes_i16x16(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x2_t = indices.into(); let result = uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); - token.cvt_from_bytes_i16x16(u8x32 { + Bytes::from_bytes(u8x32 { val: crate::support::Aligned256(result), simd: token, }) @@ -7279,11 +7279,11 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: u16x16, indices: u8x32) -> u16x16 { - let table = token.cvt_to_bytes_u16x16(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x2_t = indices.into(); let result = uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); - token.cvt_from_bytes_u16x16(u8x32 { + Bytes::from_bytes(u8x32 { val: crate::support::Aligned256(result), simd: token, }) @@ -7781,11 +7781,11 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: i32x8, indices: u8x32) -> i32x8 { - let table = token.cvt_to_bytes_i32x8(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x2_t = indices.into(); let result = uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); - token.cvt_from_bytes_i32x8(u8x32 { + Bytes::from_bytes(u8x32 { val: crate::support::Aligned256(result), simd: token, }) @@ -8154,11 +8154,11 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: u32x8, indices: u8x32) -> u32x8 { - let table = token.cvt_to_bytes_u32x8(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x2_t = indices.into(); let result = uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); - token.cvt_from_bytes_u32x8(u8x32 { + Bytes::from_bytes(u8x32 { val: crate::support::Aligned256(result), simd: token, }) @@ -8632,11 +8632,11 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: f64x4, indices: u8x32) -> f64x4 { - let table = token.cvt_to_bytes_f64x4(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x2_t = indices.into(); let result = uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); - token.cvt_from_bytes_f64x4(u8x32 { + Bytes::from_bytes(u8x32 { val: crate::support::Aligned256(result), simd: token, }) @@ -9035,11 +9035,11 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: i64x4, indices: u8x32) -> i64x4 { - let table = token.cvt_to_bytes_i64x4(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x2_t = indices.into(); let result = uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); - token.cvt_from_bytes_i64x4(u8x32 { + Bytes::from_bytes(u8x32 { val: crate::support::Aligned256(result), simd: token, }) @@ -9387,11 +9387,11 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: u64x4, indices: u8x32) -> u64x4 { - let table = token.cvt_to_bytes_u64x4(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x2_t = indices.into(); let result = uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); - token.cvt_from_bytes_u64x4(u8x32 { + Bytes::from_bytes(u8x32 { val: crate::support::Aligned256(result), simd: token, }) @@ -9930,7 +9930,7 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: f32x16, indices: u8x64) -> f32x16 { - let table = token.cvt_to_bytes_f32x16(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x4_t = indices.into(); let result = uint8x16x4_t( vqtbl4q_u8(table, indices.0), @@ -9938,7 +9938,7 @@ impl Simd for Neon { vqtbl4q_u8(table, indices.2), vqtbl4q_u8(table, indices.3), ); - token.cvt_from_bytes_f32x16(u8x64 { + Bytes::from_bytes(u8x64 { val: crate::support::Aligned512(result), simd: token, }) @@ -10620,7 +10620,7 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: i8x64, indices: u8x64) -> i8x64 { - let table = token.cvt_to_bytes_i8x64(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x4_t = indices.into(); let result = uint8x16x4_t( vqtbl4q_u8(table, indices.0), @@ -10628,7 +10628,7 @@ impl Simd for Neon { vqtbl4q_u8(table, indices.2), vqtbl4q_u8(table, indices.3), ); - token.cvt_from_bytes_i8x64(u8x64 { + Bytes::from_bytes(u8x64 { val: crate::support::Aligned512(result), simd: token, }) @@ -11225,7 +11225,7 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: u8x64, indices: u8x64) -> u8x64 { - let table = token.cvt_to_bytes_u8x64(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x4_t = indices.into(); let result = uint8x16x4_t( vqtbl4q_u8(table, indices.0), @@ -11233,7 +11233,7 @@ impl Simd for Neon { vqtbl4q_u8(table, indices.2), vqtbl4q_u8(table, indices.3), ); - token.cvt_from_bytes_u8x64(u8x64 { + Bytes::from_bytes(u8x64 { val: crate::support::Aligned512(result), simd: token, }) @@ -11826,7 +11826,7 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: i16x32, indices: u8x64) -> i16x32 { - let table = token.cvt_to_bytes_i16x32(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x4_t = indices.into(); let result = uint8x16x4_t( vqtbl4q_u8(table, indices.0), @@ -11834,7 +11834,7 @@ impl Simd for Neon { vqtbl4q_u8(table, indices.2), vqtbl4q_u8(table, indices.3), ); - token.cvt_from_bytes_i16x32(u8x64 { + Bytes::from_bytes(u8x64 { val: crate::support::Aligned512(result), simd: token, }) @@ -12313,7 +12313,7 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: u16x32, indices: u8x64) -> u16x32 { - let table = token.cvt_to_bytes_u16x32(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x4_t = indices.into(); let result = uint8x16x4_t( vqtbl4q_u8(table, indices.0), @@ -12321,7 +12321,7 @@ impl Simd for Neon { vqtbl4q_u8(table, indices.2), vqtbl4q_u8(table, indices.3), ); - token.cvt_from_bytes_u16x32(u8x64 { + Bytes::from_bytes(u8x64 { val: crate::support::Aligned512(result), simd: token, }) @@ -12864,7 +12864,7 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: i32x16, indices: u8x64) -> i32x16 { - let table = token.cvt_to_bytes_i32x16(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x4_t = indices.into(); let result = uint8x16x4_t( vqtbl4q_u8(table, indices.0), @@ -12872,7 +12872,7 @@ impl Simd for Neon { vqtbl4q_u8(table, indices.2), vqtbl4q_u8(table, indices.3), ); - token.cvt_from_bytes_i32x16(u8x64 { + Bytes::from_bytes(u8x64 { val: crate::support::Aligned512(result), simd: token, }) @@ -13286,7 +13286,7 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: u32x16, indices: u8x64) -> u32x16 { - let table = token.cvt_to_bytes_u32x16(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x4_t = indices.into(); let result = uint8x16x4_t( vqtbl4q_u8(table, indices.0), @@ -13294,7 +13294,7 @@ impl Simd for Neon { vqtbl4q_u8(table, indices.2), vqtbl4q_u8(table, indices.3), ); - token.cvt_from_bytes_u32x16(u8x64 { + Bytes::from_bytes(u8x64 { val: crate::support::Aligned512(result), simd: token, }) @@ -13792,7 +13792,7 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: f64x8, indices: u8x64) -> f64x8 { - let table = token.cvt_to_bytes_f64x8(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x4_t = indices.into(); let result = uint8x16x4_t( vqtbl4q_u8(table, indices.0), @@ -13800,7 +13800,7 @@ impl Simd for Neon { vqtbl4q_u8(table, indices.2), vqtbl4q_u8(table, indices.3), ); - token.cvt_from_bytes_f64x8(u8x64 { + Bytes::from_bytes(u8x64 { val: crate::support::Aligned512(result), simd: token, }) @@ -14224,7 +14224,7 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: i64x8, indices: u8x64) -> i64x8 { - let table = token.cvt_to_bytes_i64x8(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x4_t = indices.into(); let result = uint8x16x4_t( vqtbl4q_u8(table, indices.0), @@ -14232,7 +14232,7 @@ impl Simd for Neon { vqtbl4q_u8(table, indices.2), vqtbl4q_u8(table, indices.3), ); - token.cvt_from_bytes_i64x8(u8x64 { + Bytes::from_bytes(u8x64 { val: crate::support::Aligned512(result), simd: token, }) @@ -14605,7 +14605,7 @@ impl Simd for Neon { crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: u64x8, indices: u8x64) -> u64x8 { - let table = token.cvt_to_bytes_u64x8(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x4_t = indices.into(); let result = uint8x16x4_t( vqtbl4q_u8(table, indices.0), @@ -14613,7 +14613,7 @@ impl Simd for Neon { vqtbl4q_u8(table, indices.2), vqtbl4q_u8(table, indices.3), ); - token.cvt_from_bytes_u64x8(u8x64 { + Bytes::from_bytes(u8x64 { val: crate::support::Aligned512(result), simd: token, }) diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index 57f30bc0..9e04c1f4 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -360,7 +360,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { - let bytes = self.cvt_to_bytes_f32x4(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -428,7 +428,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_f32x4(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { @@ -1032,7 +1032,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { - let bytes = self.cvt_to_bytes_i8x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -1100,7 +1100,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_i8x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { @@ -1660,7 +1660,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { - let bytes = self.cvt_to_bytes_u8x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -1728,7 +1728,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_u8x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { @@ -2456,7 +2456,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { - let bytes = self.cvt_to_bytes_i16x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -2524,7 +2524,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_i16x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { @@ -3000,7 +3000,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { - let bytes = self.cvt_to_bytes_u16x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -3068,7 +3068,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_u16x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { @@ -3744,7 +3744,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { - let bytes = self.cvt_to_bytes_i32x4(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -3812,7 +3812,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_i32x4(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { @@ -4296,7 +4296,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { - let bytes = self.cvt_to_bytes_u32x4(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -4364,7 +4364,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_u32x4(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { @@ -5041,7 +5041,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { - let bytes = self.cvt_to_bytes_f64x2(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -5109,7 +5109,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_f64x2(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { @@ -5591,7 +5591,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { - let bytes = self.cvt_to_bytes_i64x2(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -5659,7 +5659,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_i64x2(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { @@ -6086,7 +6086,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { - let bytes = self.cvt_to_bytes_u64x2(a); + let bytes = Bytes::to_bytes(a); let result: u8x16 = [ { let index = indices[0usize] as usize; @@ -6154,7 +6154,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_u64x2(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { @@ -6705,7 +6705,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { - let bytes = self.cvt_to_bytes_f32x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -6837,7 +6837,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_f32x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { @@ -7342,7 +7342,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { - let bytes = self.cvt_to_bytes_i8x32(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -7474,7 +7474,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_i8x32(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { @@ -7902,7 +7902,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { - let bytes = self.cvt_to_bytes_u8x32(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -8034,7 +8034,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_u8x32(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { @@ -8526,7 +8526,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { - let bytes = self.cvt_to_bytes_i16x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -8658,7 +8658,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_i16x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { @@ -9026,7 +9026,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { - let bytes = self.cvt_to_bytes_u16x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -9158,7 +9158,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_u16x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { @@ -9630,7 +9630,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { - let bytes = self.cvt_to_bytes_i32x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -9762,7 +9762,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_i32x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { @@ -10099,7 +10099,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { - let bytes = self.cvt_to_bytes_u32x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -10231,7 +10231,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_u32x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { @@ -10671,7 +10671,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { - let bytes = self.cvt_to_bytes_f64x4(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -10803,7 +10803,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_f64x4(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { @@ -11170,7 +11170,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { - let bytes = self.cvt_to_bytes_i64x4(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -11302,7 +11302,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_i64x4(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { @@ -11618,7 +11618,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { - let bytes = self.cvt_to_bytes_u64x4(a); + let bytes = Bytes::to_bytes(a); let result: u8x32 = [ { let index = indices[0usize] as usize; @@ -11750,7 +11750,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_u64x4(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { @@ -12237,7 +12237,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { - let bytes = self.cvt_to_bytes_f32x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -12497,7 +12497,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_f32x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { @@ -13198,7 +13198,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { - let bytes = self.cvt_to_bytes_i8x64(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -13458,7 +13458,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_i8x64(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { @@ -14007,7 +14007,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { - let bytes = self.cvt_to_bytes_u8x64(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -14267,7 +14267,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_u8x64(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { @@ -14889,7 +14889,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { - let bytes = self.cvt_to_bytes_i16x32(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -15149,7 +15149,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_i16x32(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { @@ -15580,7 +15580,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { - let bytes = self.cvt_to_bytes_u16x32(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -15840,7 +15840,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_u16x32(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { @@ -16384,7 +16384,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { - let bytes = self.cvt_to_bytes_i32x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -16644,7 +16644,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_i32x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { @@ -17010,7 +17010,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { - let bytes = self.cvt_to_bytes_u32x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -17270,7 +17270,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_u32x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { @@ -17787,7 +17787,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { - let bytes = self.cvt_to_bytes_f64x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -18047,7 +18047,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_f64x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { @@ -18423,7 +18423,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { - let bytes = self.cvt_to_bytes_i64x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -18683,7 +18683,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_i64x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { @@ -19008,7 +19008,7 @@ impl Simd for Sse2 { } #[inline(always)] fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { - let bytes = self.cvt_to_bytes_u64x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -19268,7 +19268,7 @@ impl Simd for Sse2 { }, ] .simd_into(self); - self.cvt_from_bytes_u64x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 3397a3db..224e3566 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -242,12 +242,12 @@ impl Simd for Sse4_2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_f32x4(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_f32x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -827,12 +827,12 @@ impl Simd for Sse4_2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_i8x16(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_i8x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -1327,12 +1327,12 @@ impl Simd for Sse4_2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_u8x16(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_u8x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -1984,12 +1984,12 @@ impl Simd for Sse4_2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_i16x8(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_i16x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -2416,12 +2416,12 @@ impl Simd for Sse4_2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_u16x8(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_u16x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -3006,12 +3006,12 @@ impl Simd for Sse4_2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_i32x4(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_i32x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -3422,12 +3422,12 @@ impl Simd for Sse4_2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_u32x4(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_u32x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -4010,12 +4010,12 @@ impl Simd for Sse4_2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_f64x2(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_f64x2(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -4456,12 +4456,12 @@ impl Simd for Sse4_2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_i64x2(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_i64x2(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -4830,12 +4830,12 @@ impl Simd for Sse4_2 { let indices = indices.into(); let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(token.cvt_to_bytes_u64x2(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, }; - token.cvt_from_bytes_u64x2(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -5381,7 +5381,7 @@ impl Simd for Sse4_2 { crate::kernel!( #[inline(always)] fn kernel(token: Sse4_2, a: f32x8, indices: u8x32) -> f32x8 { - let bytes = token.cvt_to_bytes_f32x8(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x32(bytes); let (indices_low, indices_high) = token.split_u8x32(indices); let high_table_offset = token.splat_u8x16(16); @@ -5398,7 +5398,7 @@ impl Simd for Sse4_2 { ); let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x16(output_low, output_high); - token.cvt_from_bytes_f32x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -5909,7 +5909,7 @@ impl Simd for Sse4_2 { crate::kernel!( #[inline(always)] fn kernel(token: Sse4_2, a: i8x32, indices: u8x32) -> i8x32 { - let bytes = token.cvt_to_bytes_i8x32(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x32(bytes); let (indices_low, indices_high) = token.split_u8x32(indices); let high_table_offset = token.splat_u8x16(16); @@ -5926,7 +5926,7 @@ impl Simd for Sse4_2 { ); let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x16(output_low, output_high); - token.cvt_from_bytes_i8x32(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -6360,7 +6360,7 @@ impl Simd for Sse4_2 { crate::kernel!( #[inline(always)] fn kernel(token: Sse4_2, a: u8x32, indices: u8x32) -> u8x32 { - let bytes = token.cvt_to_bytes_u8x32(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x32(bytes); let (indices_low, indices_high) = token.split_u8x32(indices); let high_table_offset = token.splat_u8x16(16); @@ -6377,7 +6377,7 @@ impl Simd for Sse4_2 { ); let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x16(output_low, output_high); - token.cvt_from_bytes_u8x32(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -6875,7 +6875,7 @@ impl Simd for Sse4_2 { crate::kernel!( #[inline(always)] fn kernel(token: Sse4_2, a: i16x16, indices: u8x32) -> i16x16 { - let bytes = token.cvt_to_bytes_i16x16(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x32(bytes); let (indices_low, indices_high) = token.split_u8x32(indices); let high_table_offset = token.splat_u8x16(16); @@ -6892,7 +6892,7 @@ impl Simd for Sse4_2 { ); let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x16(output_low, output_high); - token.cvt_from_bytes_i16x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -7266,7 +7266,7 @@ impl Simd for Sse4_2 { crate::kernel!( #[inline(always)] fn kernel(token: Sse4_2, a: u16x16, indices: u8x32) -> u16x16 { - let bytes = token.cvt_to_bytes_u16x16(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x32(bytes); let (indices_low, indices_high) = token.split_u8x32(indices); let high_table_offset = token.splat_u8x16(16); @@ -7283,7 +7283,7 @@ impl Simd for Sse4_2 { ); let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x16(output_low, output_high); - token.cvt_from_bytes_u16x16(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -7761,7 +7761,7 @@ impl Simd for Sse4_2 { crate::kernel!( #[inline(always)] fn kernel(token: Sse4_2, a: i32x8, indices: u8x32) -> i32x8 { - let bytes = token.cvt_to_bytes_i32x8(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x32(bytes); let (indices_low, indices_high) = token.split_u8x32(indices); let high_table_offset = token.splat_u8x16(16); @@ -7778,7 +7778,7 @@ impl Simd for Sse4_2 { ); let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x16(output_low, output_high); - token.cvt_from_bytes_i32x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -8121,7 +8121,7 @@ impl Simd for Sse4_2 { crate::kernel!( #[inline(always)] fn kernel(token: Sse4_2, a: u32x8, indices: u8x32) -> u32x8 { - let bytes = token.cvt_to_bytes_u32x8(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x32(bytes); let (indices_low, indices_high) = token.split_u8x32(indices); let high_table_offset = token.splat_u8x16(16); @@ -8138,7 +8138,7 @@ impl Simd for Sse4_2 { ); let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x16(output_low, output_high); - token.cvt_from_bytes_u32x8(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -8584,7 +8584,7 @@ impl Simd for Sse4_2 { crate::kernel!( #[inline(always)] fn kernel(token: Sse4_2, a: f64x4, indices: u8x32) -> f64x4 { - let bytes = token.cvt_to_bytes_f64x4(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x32(bytes); let (indices_low, indices_high) = token.split_u8x32(indices); let high_table_offset = token.splat_u8x16(16); @@ -8601,7 +8601,7 @@ impl Simd for Sse4_2 { ); let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x16(output_low, output_high); - token.cvt_from_bytes_f64x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -8974,7 +8974,7 @@ impl Simd for Sse4_2 { crate::kernel!( #[inline(always)] fn kernel(token: Sse4_2, a: i64x4, indices: u8x32) -> i64x4 { - let bytes = token.cvt_to_bytes_i64x4(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x32(bytes); let (indices_low, indices_high) = token.split_u8x32(indices); let high_table_offset = token.splat_u8x16(16); @@ -8991,7 +8991,7 @@ impl Simd for Sse4_2 { ); let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x16(output_low, output_high); - token.cvt_from_bytes_i64x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -9313,7 +9313,7 @@ impl Simd for Sse4_2 { crate::kernel!( #[inline(always)] fn kernel(token: Sse4_2, a: u64x4, indices: u8x32) -> u64x4 { - let bytes = token.cvt_to_bytes_u64x4(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = token.split_u8x32(bytes); let (indices_low, indices_high) = token.split_u8x32(indices); let high_table_offset = token.splat_u8x16(16); @@ -9330,7 +9330,7 @@ impl Simd for Sse4_2 { ); let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = token.combine_u8x16(output_low, output_high); - token.cvt_from_bytes_u64x4(result_bytes) + Bytes::from_bytes(result_bytes) } ); kernel(self, a, indices) @@ -9820,7 +9820,7 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { - let bytes = self.cvt_to_bytes_f32x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -10080,7 +10080,7 @@ impl Simd for Sse4_2 { }, ] .simd_into(self); - self.cvt_from_bytes_f32x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { @@ -10781,7 +10781,7 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { - let bytes = self.cvt_to_bytes_i8x64(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -11041,7 +11041,7 @@ impl Simd for Sse4_2 { }, ] .simd_into(self); - self.cvt_from_bytes_i8x64(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { @@ -11590,7 +11590,7 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { - let bytes = self.cvt_to_bytes_u8x64(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -11850,7 +11850,7 @@ impl Simd for Sse4_2 { }, ] .simd_into(self); - self.cvt_from_bytes_u8x64(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { @@ -12513,7 +12513,7 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { - let bytes = self.cvt_to_bytes_i16x32(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -12773,7 +12773,7 @@ impl Simd for Sse4_2 { }, ] .simd_into(self); - self.cvt_from_bytes_i16x32(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { @@ -13204,7 +13204,7 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { - let bytes = self.cvt_to_bytes_u16x32(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -13464,7 +13464,7 @@ impl Simd for Sse4_2 { }, ] .simd_into(self); - self.cvt_from_bytes_u16x32(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { @@ -14045,7 +14045,7 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { - let bytes = self.cvt_to_bytes_i32x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -14305,7 +14305,7 @@ impl Simd for Sse4_2 { }, ] .simd_into(self); - self.cvt_from_bytes_i32x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { @@ -14671,7 +14671,7 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { - let bytes = self.cvt_to_bytes_u32x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -14931,7 +14931,7 @@ impl Simd for Sse4_2 { }, ] .simd_into(self); - self.cvt_from_bytes_u32x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { @@ -15448,7 +15448,7 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { - let bytes = self.cvt_to_bytes_f64x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -15708,7 +15708,7 @@ impl Simd for Sse4_2 { }, ] .simd_into(self); - self.cvt_from_bytes_f64x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { @@ -16084,7 +16084,7 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { - let bytes = self.cvt_to_bytes_i64x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -16344,7 +16344,7 @@ impl Simd for Sse4_2 { }, ] .simd_into(self); - self.cvt_from_bytes_i64x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { @@ -16669,7 +16669,7 @@ impl Simd for Sse4_2 { } #[inline(always)] fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { - let bytes = self.cvt_to_bytes_u64x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -16929,7 +16929,7 @@ impl Simd for Sse4_2 { }, ] .simd_into(self); - self.cvt_from_bytes_u64x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index eac34947..ffdc1357 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -213,8 +213,8 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { - let result = u8x16_swizzle(self.cvt_to_bytes_f32x4(a).val.0, indices.into()); - self.cvt_from_bytes_f32x4(u8x16 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: self, }) @@ -589,8 +589,8 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { - let result = u8x16_swizzle(self.cvt_to_bytes_i8x16(a).val.0, indices.into()); - self.cvt_from_bytes_i8x16(u8x16 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: self, }) @@ -928,8 +928,8 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { - let result = u8x16_swizzle(self.cvt_to_bytes_u8x16(a).val.0, indices.into()); - self.cvt_from_bytes_u8x16(u8x16 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: self, }) @@ -1340,8 +1340,8 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { - let result = u8x16_swizzle(self.cvt_to_bytes_i16x8(a).val.0, indices.into()); - self.cvt_from_bytes_i16x8(u8x16 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: self, }) @@ -1619,8 +1619,8 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { - let result = u8x16_swizzle(self.cvt_to_bytes_u16x8(a).val.0, indices.into()); - self.cvt_from_bytes_u16x8(u8x16 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: self, }) @@ -1975,8 +1975,8 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { - let result = u8x16_swizzle(self.cvt_to_bytes_i32x4(a).val.0, indices.into()); - self.cvt_from_bytes_i32x4(u8x16 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: self, }) @@ -2234,8 +2234,8 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { - let result = u8x16_swizzle(self.cvt_to_bytes_u32x4(a).val.0, indices.into()); - self.cvt_from_bytes_u32x4(u8x16 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: self, }) @@ -2578,8 +2578,8 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { - let result = u8x16_swizzle(self.cvt_to_bytes_f64x2(a).val.0, indices.into()); - self.cvt_from_bytes_f64x2(u8x16 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: self, }) @@ -2872,8 +2872,8 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { - let result = u8x16_swizzle(self.cvt_to_bytes_i64x2(a).val.0, indices.into()); - self.cvt_from_bytes_i64x2(u8x16 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: self, }) @@ -3123,8 +3123,8 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { - let result = u8x16_swizzle(self.cvt_to_bytes_u64x2(a).val.0, indices.into()); - self.cvt_from_bytes_u64x2(u8x16 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { val: crate::support::Aligned128(result), simd: self, }) @@ -3518,7 +3518,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { - let bytes = self.cvt_to_bytes_f32x8(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = self.split_u8x32(bytes); let (indices_low, indices_high) = self.split_u8x32(indices); let high_table_offset = self.splat_u8x16(16); @@ -3531,7 +3531,7 @@ impl Simd for WasmSimd128 { .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = self.combine_u8x16(output_low, output_high); - self.cvt_from_bytes_f32x8(result_bytes) + Bytes::from_bytes(result_bytes) } #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { @@ -4032,7 +4032,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { - let bytes = self.cvt_to_bytes_i8x32(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = self.split_u8x32(bytes); let (indices_low, indices_high) = self.split_u8x32(indices); let high_table_offset = self.splat_u8x16(16); @@ -4045,7 +4045,7 @@ impl Simd for WasmSimd128 { .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = self.combine_u8x16(output_low, output_high); - self.cvt_from_bytes_i8x32(result_bytes) + Bytes::from_bytes(result_bytes) } #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { @@ -4469,7 +4469,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { - let bytes = self.cvt_to_bytes_u8x32(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = self.split_u8x32(bytes); let (indices_low, indices_high) = self.split_u8x32(indices); let high_table_offset = self.splat_u8x16(16); @@ -4482,7 +4482,7 @@ impl Simd for WasmSimd128 { .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = self.combine_u8x16(output_low, output_high); - self.cvt_from_bytes_u8x32(result_bytes) + Bytes::from_bytes(result_bytes) } #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { @@ -4973,7 +4973,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { - let bytes = self.cvt_to_bytes_i16x16(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = self.split_u8x32(bytes); let (indices_low, indices_high) = self.split_u8x32(indices); let high_table_offset = self.splat_u8x16(16); @@ -4986,7 +4986,7 @@ impl Simd for WasmSimd128 { .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = self.combine_u8x16(output_low, output_high); - self.cvt_from_bytes_i16x16(result_bytes) + Bytes::from_bytes(result_bytes) } #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { @@ -5353,7 +5353,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { - let bytes = self.cvt_to_bytes_u16x16(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = self.split_u8x32(bytes); let (indices_low, indices_high) = self.split_u8x32(indices); let high_table_offset = self.splat_u8x16(16); @@ -5366,7 +5366,7 @@ impl Simd for WasmSimd128 { .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = self.combine_u8x16(output_low, output_high); - self.cvt_from_bytes_u16x16(result_bytes) + Bytes::from_bytes(result_bytes) } #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { @@ -5825,7 +5825,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { - let bytes = self.cvt_to_bytes_i32x8(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = self.split_u8x32(bytes); let (indices_low, indices_high) = self.split_u8x32(indices); let high_table_offset = self.splat_u8x16(16); @@ -5838,7 +5838,7 @@ impl Simd for WasmSimd128 { .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = self.combine_u8x16(output_low, output_high); - self.cvt_from_bytes_i32x8(result_bytes) + Bytes::from_bytes(result_bytes) } #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { @@ -6174,7 +6174,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { - let bytes = self.cvt_to_bytes_u32x8(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = self.split_u8x32(bytes); let (indices_low, indices_high) = self.split_u8x32(indices); let high_table_offset = self.splat_u8x16(16); @@ -6187,7 +6187,7 @@ impl Simd for WasmSimd128 { .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = self.combine_u8x16(output_low, output_high); - self.cvt_from_bytes_u32x8(result_bytes) + Bytes::from_bytes(result_bytes) } #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { @@ -6626,7 +6626,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { - let bytes = self.cvt_to_bytes_f64x4(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = self.split_u8x32(bytes); let (indices_low, indices_high) = self.split_u8x32(indices); let high_table_offset = self.splat_u8x16(16); @@ -6639,7 +6639,7 @@ impl Simd for WasmSimd128 { .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = self.combine_u8x16(output_low, output_high); - self.cvt_from_bytes_f64x4(result_bytes) + Bytes::from_bytes(result_bytes) } #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { @@ -7005,7 +7005,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { - let bytes = self.cvt_to_bytes_i64x4(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = self.split_u8x32(bytes); let (indices_low, indices_high) = self.split_u8x32(indices); let high_table_offset = self.splat_u8x16(16); @@ -7018,7 +7018,7 @@ impl Simd for WasmSimd128 { .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = self.combine_u8x16(output_low, output_high); - self.cvt_from_bytes_i64x4(result_bytes) + Bytes::from_bytes(result_bytes) } #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { @@ -7333,7 +7333,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { - let bytes = self.cvt_to_bytes_u64x4(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = self.split_u8x32(bytes); let (indices_low, indices_high) = self.split_u8x32(indices); let high_table_offset = self.splat_u8x16(16); @@ -7346,7 +7346,7 @@ impl Simd for WasmSimd128 { .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); let result_bytes = self.combine_u8x16(output_low, output_high); - self.cvt_from_bytes_u64x4(result_bytes) + Bytes::from_bytes(result_bytes) } #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { @@ -7832,7 +7832,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { - let bytes = self.cvt_to_bytes_f32x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -8092,7 +8092,7 @@ impl Simd for WasmSimd128 { }, ] .simd_into(self); - self.cvt_from_bytes_f32x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { @@ -8760,7 +8760,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { - let bytes = self.cvt_to_bytes_i8x64(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -9020,7 +9020,7 @@ impl Simd for WasmSimd128 { }, ] .simd_into(self); - self.cvt_from_bytes_i8x64(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { @@ -9565,7 +9565,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { - let bytes = self.cvt_to_bytes_u8x64(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -9825,7 +9825,7 @@ impl Simd for WasmSimd128 { }, ] .simd_into(self); - self.cvt_from_bytes_u8x64(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { @@ -10431,7 +10431,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { - let bytes = self.cvt_to_bytes_i16x32(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -10691,7 +10691,7 @@ impl Simd for WasmSimd128 { }, ] .simd_into(self); - self.cvt_from_bytes_i16x32(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { @@ -11121,7 +11121,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { - let bytes = self.cvt_to_bytes_u16x32(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -11381,7 +11381,7 @@ impl Simd for WasmSimd128 { }, ] .simd_into(self); - self.cvt_from_bytes_u16x32(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { @@ -11913,7 +11913,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { - let bytes = self.cvt_to_bytes_i32x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -12173,7 +12173,7 @@ impl Simd for WasmSimd128 { }, ] .simd_into(self); - self.cvt_from_bytes_i32x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { @@ -12538,7 +12538,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { - let bytes = self.cvt_to_bytes_u32x16(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -12798,7 +12798,7 @@ impl Simd for WasmSimd128 { }, ] .simd_into(self); - self.cvt_from_bytes_u32x16(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { @@ -13285,7 +13285,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { - let bytes = self.cvt_to_bytes_f64x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -13545,7 +13545,7 @@ impl Simd for WasmSimd128 { }, ] .simd_into(self); - self.cvt_from_bytes_f64x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { @@ -13920,7 +13920,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { - let bytes = self.cvt_to_bytes_i64x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -14180,7 +14180,7 @@ impl Simd for WasmSimd128 { }, ] .simd_into(self); - self.cvt_from_bytes_i64x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { @@ -14504,7 +14504,7 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { - let bytes = self.cvt_to_bytes_u64x8(a); + let bytes = Bytes::to_bytes(a); let result: u8x64 = [ { let index = indices[0usize] as usize; @@ -14764,7 +14764,7 @@ impl Simd for WasmSimd128 { }, ] .simd_into(self); - self.cvt_from_bytes_u64x8(result) + Bytes::from_bytes(result) } #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index de321fda..89090a62 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -34,7 +34,6 @@ pub(crate) fn recursive_swizzle_dyn_precise_body( // the final combine is very cheap: a bitwise or. let bytes_ty = vec_ty.bytes_ty(); let half_bytes_ty = VecType::new(ScalarType::Unsigned, 8, bytes_ty.len / 2); - let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); let split_bytes = generic_op_name("split", &bytes_ty); let combine_half_bytes = generic_op_name("combine", &half_bytes_ty); let swizzle_half = generic_op_name("swizzle_dyn_precise", &half_bytes_ty); @@ -44,7 +43,7 @@ pub(crate) fn recursive_swizzle_dyn_precise_body( let half_len = Literal::u8_unsuffixed(u8::try_from(bytes_ty.len / 2).unwrap()); quote! { - let bytes = #token.#to_bytes(a); + let bytes = Bytes::to_bytes(a); let (table_low, table_high) = #token.#split_bytes(bytes); let (indices_low, indices_high) = #token.#split_bytes(indices); let high_table_offset = #token.#splat_half(#half_len); diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index 70df6527..b0593ed8 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -442,8 +442,6 @@ impl Level for Fallback { OpSig::SwizzleDynPrecise => { let bytes_ty = vec_ty.bytes_ty(); let bytes_rust = bytes_ty.rust(); - let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); - let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); let byte_count = bytes_ty.len; let items = make_list( (0..byte_count) @@ -460,9 +458,9 @@ impl Level for Fallback { quote! { #method_sig { - let bytes = self.#to_bytes(a); + let bytes = Bytes::to_bytes(a); let result: #bytes_rust = #items.simd_into(self); - self.#from_bytes(result) + Bytes::from_bytes(result) } } } diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index 7a01f09c..6bab6ea2 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -468,16 +468,14 @@ impl Level for Neon { let bytes_ty = vec_ty.bytes_ty(); let bytes = bytes_ty.rust(); let wrapper = bytes_ty.aligned_wrapper(); - let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); - let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); self.kernel_method(op, vec_ty, |token| { let body = match vec_ty.n_bits() { 128 => quote! { - let result = vqtbl1q_u8(#token.#to_bytes(a).val.0, indices.into()); + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); }, 256 => quote! { - let table = #token.#to_bytes(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x2_t = indices.into(); let result = uint8x16x2_t( vqtbl2q_u8(table, indices.0), @@ -485,7 +483,7 @@ impl Level for Neon { ); }, 512 => quote! { - let table = #token.#to_bytes(a).val.0; + let table = Bytes::to_bytes(a).val.0; let indices: uint8x16x4_t = indices.into(); let result = uint8x16x4_t( vqtbl4q_u8(table, indices.0), @@ -499,7 +497,7 @@ impl Level for Neon { quote! { #body - #token.#from_bytes(#bytes { val: #wrapper(result), simd: #token }) + Bytes::from_bytes(#bytes { val: #wrapper(result), simd: #token }) } }) } diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index 132c3ae8..f6745c8d 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -7,16 +7,8 @@ use quote::{format_ident, quote}; use crate::arch::wasm::{arch_prefix, v128_intrinsic}; use crate::generic::{ fallback_method, generic_as_array, generic_block_combine, generic_block_split, -<<<<<<< HEAD - generic_from_array, generic_from_bytes, generic_mask_set, generic_op_name, generic_store_array, - generic_to_bytes, integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, -||||||| bd94894 - generic_from_array, generic_from_bytes, generic_mask_set, generic_op_name, generic_store_array, - generic_to_bytes, integer_lane_mask_splat_arg, -======= generic_from_array, generic_mask_set, generic_op_name, generic_store_array, - integer_lane_mask_splat_arg, ->>>>>>> main + integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, }; use crate::level::Level; use crate::ops::{Op, Quantifier, SlideGranularity}; @@ -522,24 +514,21 @@ impl Level for WasmSimd128 { let bytes_ty = vec_ty.bytes_ty(); let bytes = bytes_ty.rust(); let wrapper = bytes_ty.aligned_wrapper(); - let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); - let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); quote! { #method_sig { - let result = u8x16_swizzle(self.#to_bytes(a).val.0, indices.into()); - self.#from_bytes(#bytes { val: #wrapper(result), simd: self }) + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(#bytes { val: #wrapper(result), simd: self }) } } } 256 => { - let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); let body = recursive_swizzle_dyn_precise_body(vec_ty, "e! { self }); quote! { #method_sig { #body - self.#from_bytes(result_bytes) + Bytes::from_bytes(result_bytes) } } } diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index a2f3f5eb..8e063dc6 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -8,17 +8,8 @@ use crate::arch::x86::{ }; use crate::generic::{ fallback_method, generic_as_array, generic_block_combine, generic_block_split, -<<<<<<< HEAD - generic_from_array, generic_from_bytes, generic_mask_from_bitmask, generic_mask_set, - generic_op_name, generic_store_array, generic_to_bytes, integer_lane_mask_splat_arg, - recursive_swizzle_dyn_precise_body, -||||||| bd94894 - generic_from_array, generic_from_bytes, generic_mask_from_bitmask, generic_mask_set, - generic_op_name, generic_store_array, generic_to_bytes, integer_lane_mask_splat_arg, -======= generic_from_array, generic_mask_from_bitmask, generic_mask_set, generic_op_name, - generic_store_array, integer_lane_mask_splat_arg, ->>>>>>> main + generic_store_array, integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, }; use crate::level::Level; use crate::ops::{Op, OpSig, Quantifier, SlideGranularity}; @@ -2842,8 +2833,6 @@ impl X86 { let bytes_ty = vec_ty.bytes_ty(); let bytes = bytes_ty.rust(); let wrapper = bytes_ty.aligned_wrapper(); - let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); - let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); if *self == Self::Sse2 || (*self == Self::Sse4_2 && vec_ty.n_bits() == 512) { return fallback_method(op, vec_ty); @@ -2857,14 +2846,14 @@ impl X86 { // The added value only changes bits that PSHUFB ignores for valid indices. let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); let zeroing_indices = _mm_or_si128(indices, index_out_of_range); - let result = _mm_shuffle_epi8(#token.#to_bytes(a).val.0, zeroing_indices); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); let result_bytes = #bytes { val: #wrapper(result), simd: #token }; }, (Self::Sse4_2, 256) | (Self::Avx2, 512) => { recursive_swizzle_dyn_precise_body(vec_ty, token) } (Self::Avx2, 256) => quote! { - let bytes = #token.#to_bytes(a); + let bytes = Bytes::to_bytes(a); let idxs = indices; let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); @@ -2888,7 +2877,7 @@ impl X86 { let set1 = set1_intrinsic(&bytes_ty); let byte_count = signed_literal(bytes_ty.len as u64, 8); quote! { - let bytes = #token.#to_bytes(a).val.0; + let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); let in_range = #cmp::<{ _MM_CMPINT_LT }>(indices, #set1(#byte_count)); let result = #maskz_permute(in_range, indices, bytes); @@ -2900,7 +2889,7 @@ impl X86 { quote! { #body - #token.#from_bytes(result_bytes) + Bytes::from_bytes(result_bytes) } }) } diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index 3c8a439d..af3e37d5 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -1607,20 +1607,8 @@ impl OpSig { | Self::WidenNarrow { .. } | Self::MaskReduce { .. } | Self::MaskToBitmask -<<<<<<< HEAD - | Self::AsArray { .. } - | Self::FromBytes - | Self::ToBytes => &["a"], - Self::SwizzleDynWithinBlocks | Self::SwizzleDynPrecise => &["a", "indices"], -||||||| bd94894 - | Self::AsArray { .. } - | Self::FromBytes - | Self::ToBytes => &["a"], - Self::SwizzleDynWithinBlocks => &["a", "indices"], -======= | Self::AsArray { .. } => &["a"], - Self::SwizzleDynWithinBlocks => &["a", "indices"], ->>>>>>> main + Self::SwizzleDynWithinBlocks | Self::SwizzleDynPrecise => &["a", "indices"], Self::Binary | Self::Compare | Self::Combine { .. } @@ -1651,18 +1639,8 @@ impl OpSig { | Self::Cvt { .. } | Self::WidenNarrow { .. } | Self::MaskReduce { .. } -<<<<<<< HEAD - | Self::AsArray { .. } - | Self::ToBytes => &["self"], - Self::SwizzleDynWithinBlocks | Self::SwizzleDynPrecise => &["self", "indices"], -||||||| bd94894 - | Self::AsArray { .. } - | Self::ToBytes => &["self"], - Self::SwizzleDynWithinBlocks => &["self", "indices"], -======= | Self::AsArray { .. } => &["self"], - Self::SwizzleDynWithinBlocks => &["self", "indices"], ->>>>>>> main + Self::SwizzleDynWithinBlocks | Self::SwizzleDynPrecise => &["self", "indices"], Self::Binary | Self::Compare | Self::Zip { .. } From 896f20908a10d97199ddfea43d63c6976f7c4b7d Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Tue, 28 Jul 2026 20:48:44 +0100 Subject: [PATCH 18/23] Reintroduce swizzle_dyn_precise random test behind #[ignore] --- .../tests/harness/ops/swizzle_dyn_precise.rs | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/fearless_simd_tests/tests/harness/ops/swizzle_dyn_precise.rs b/fearless_simd_tests/tests/harness/ops/swizzle_dyn_precise.rs index 36113588..123e7012 100644 --- a/fearless_simd_tests/tests/harness/ops/swizzle_dyn_precise.rs +++ b/fearless_simd_tests/tests/harness/ops/swizzle_dyn_precise.rs @@ -110,3 +110,50 @@ fn swizzle_dyn_precise_generic_indices(simd: S) { assert_eq!(*result, expected); } + +#[simd_test] +#[ignore = "this test is slow"] +// run with: cargo test --release swizzle_dyn_precise_random_u8_all_widths -- --ignored +fn swizzle_dyn_precise_random_u8_all_widths(simd: S) { + let mut rng = fastrand::Rng::with_seed(0x5eed_5eed_cafe_f00d); + + for iteration in 0..100_000 { + let mut bytes = [0u8; 64]; + let mut indices = [0u8; 64]; + rng.fill(&mut bytes); + rng.fill(&mut indices); + + let bytes16 = &bytes[..16]; + let indices16 = &indices[..16]; + let expected16 = expected_swizzle_precise_from_slice::<16>(bytes16, indices16); + let value16 = u8x16::from_slice(simd, bytes16); + let index_vec16 = u8x16::from_slice(simd, indices16); + let result16 = value16.swizzle_dyn_precise(index_vec16); + assert_eq!(*result16, expected16, "iteration {iteration}, width 16"); + + let bytes32 = &bytes[..32]; + let indices32 = &indices[..32]; + let expected32 = expected_swizzle_precise_from_slice::<32>(bytes32, indices32); + let value32 = u8x32::from_slice(simd, bytes32); + let index_vec32 = u8x32::from_slice(simd, indices32); + let result32 = value32.swizzle_dyn_precise(index_vec32); + assert_eq!(*result32, expected32, "iteration {iteration}, width 32"); + + let bytes64 = &bytes[..64]; + let indices64 = &indices[..64]; + let expected64 = expected_swizzle_precise_from_slice::<64>(bytes64, indices64); + let value64 = u8x64::from_slice(simd, bytes64); + let index_vec64 = u8x64::from_slice(simd, indices64); + let result64 = value64.swizzle_dyn_precise(index_vec64); + assert_eq!(*result64, expected64, "iteration {iteration}, width 64"); + } +} + +fn expected_swizzle_precise_from_slice(bytes: &[u8], indices: &[u8]) -> [u8; N] { + assert_eq!(bytes.len(), N); + assert_eq!(indices.len(), N); + core::array::from_fn(|i| { + let index = indices[i] as usize; + bytes.get(index).copied().unwrap_or(0) + }) +} From 27089185c8a640b226aa5f4c2f75586c5ed13758 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Tue, 28 Jul 2026 20:52:18 +0100 Subject: [PATCH 19/23] placate clippy --- fearless_simd_tests/tests/harness/ops/swizzle_dyn_precise.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fearless_simd_tests/tests/harness/ops/swizzle_dyn_precise.rs b/fearless_simd_tests/tests/harness/ops/swizzle_dyn_precise.rs index 123e7012..df7b05b8 100644 --- a/fearless_simd_tests/tests/harness/ops/swizzle_dyn_precise.rs +++ b/fearless_simd_tests/tests/harness/ops/swizzle_dyn_precise.rs @@ -118,8 +118,8 @@ fn swizzle_dyn_precise_random_u8_all_widths(simd: S) { let mut rng = fastrand::Rng::with_seed(0x5eed_5eed_cafe_f00d); for iteration in 0..100_000 { - let mut bytes = [0u8; 64]; - let mut indices = [0u8; 64]; + let mut bytes: [u8; 64] = [0; 64]; + let mut indices: [u8; 64] = [0; 64]; rng.fill(&mut bytes); rng.fill(&mut indices); From af454fe8a0acaf4d4321483870f01b224ec173fd Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Thu, 30 Jul 2026 15:52:55 +0100 Subject: [PATCH 20/23] Use a faster formulation of swizzle_dyn_precise on AVX-512 VBMI. Doubles the throughput on both Ice Lake and Zen4. Improves latency on Ice Lake from 23 to 13 cycles, on Zen4 from 17 to 15 cycles. Improves encoding performance of a toy base64 implementation on Zen4 by -20% (time) +25% (throughput), no change on decoding benchmark --- fearless_simd/src/generated/avx512.rs | 140 +++++++++++--------------- fearless_simd_gen/src/mk_x86.rs | 13 ++- 2 files changed, 68 insertions(+), 85 deletions(-) diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index 172dd642..7c44687a 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -252,8 +252,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: f32x4, indices: u8x16) -> f32x4 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); - let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, @@ -850,8 +850,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: i8x16, indices: u8x16) -> i8x16 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); - let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, @@ -1396,8 +1396,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: u8x16, indices: u8x16) -> u8x16 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); - let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, @@ -2042,8 +2042,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: i16x8, indices: u8x16) -> i16x8 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); - let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, @@ -2506,8 +2506,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: u16x8, indices: u8x16) -> u16x8 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); - let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, @@ -3079,8 +3079,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: i32x4, indices: u8x16) -> i32x4 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); - let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, @@ -3527,8 +3527,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: u32x4, indices: u8x16) -> u32x4 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); - let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, @@ -4093,8 +4093,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: f64x2, indices: u8x16) -> f64x2 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); - let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, @@ -4582,8 +4582,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: i64x2, indices: u8x16) -> i64x2 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); - let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, @@ -5010,8 +5010,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: u64x2, indices: u8x16) -> u64x2 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = _mm_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm_set1_epi8(16)); - let result = _mm_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); let result_bytes = u8x16 { val: crate::support::Aligned128(result), simd: token, @@ -5616,9 +5616,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: f32x8, indices: u8x32) -> f32x8 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); - let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); let result_bytes = u8x32 { val: crate::support::Aligned256(result), simd: token, @@ -6380,9 +6379,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: i8x32, indices: u8x32) -> i8x32 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); - let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); let result_bytes = u8x32 { val: crate::support::Aligned256(result), simd: token, @@ -7098,9 +7096,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: u8x32, indices: u8x32) -> u8x32 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); - let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); let result_bytes = u8x32 { val: crate::support::Aligned256(result), simd: token, @@ -7902,9 +7899,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: i16x16, indices: u8x32) -> i16x16 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); - let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); let result_bytes = u8x32 { val: crate::support::Aligned256(result), simd: token, @@ -8494,9 +8490,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: u16x16, indices: u8x32) -> u16x16 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); - let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); let result_bytes = u8x32 { val: crate::support::Aligned256(result), simd: token, @@ -9199,9 +9194,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: i32x8, indices: u8x32) -> i32x8 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); - let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); let result_bytes = u8x32 { val: crate::support::Aligned256(result), simd: token, @@ -9743,9 +9737,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: u32x8, indices: u8x32) -> u32x8 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); - let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); let result_bytes = u8x32 { val: crate::support::Aligned256(result), simd: token, @@ -10411,9 +10404,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: f64x4, indices: u8x32) -> f64x4 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); - let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); let result_bytes = u8x32 { val: crate::support::Aligned256(result), simd: token, @@ -10988,9 +10980,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: i64x4, indices: u8x32) -> i64x4 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); - let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); let result_bytes = u8x32 { val: crate::support::Aligned256(result), simd: token, @@ -11490,9 +11481,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: u64x4, indices: u8x32) -> u64x4 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm256_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm256_set1_epi8(32)); - let result = _mm256_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); let result_bytes = u8x32 { val: crate::support::Aligned256(result), simd: token, @@ -12184,9 +12174,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: f32x16, indices: u8x64) -> f32x16 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); - let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); let result_bytes = u8x64 { val: crate::support::Aligned512(result), simd: token, @@ -13119,9 +13108,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: i8x64, indices: u8x64) -> i8x64 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); - let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); let result_bytes = u8x64 { val: crate::support::Aligned512(result), simd: token, @@ -13973,9 +13961,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: u8x64, indices: u8x64) -> u8x64 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); - let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); let result_bytes = u8x64 { val: crate::support::Aligned512(result), simd: token, @@ -14870,9 +14857,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: i16x32, indices: u8x64) -> i16x32 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); - let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); let result_bytes = u8x64 { val: crate::support::Aligned512(result), simd: token, @@ -15535,9 +15521,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: u16x32, indices: u8x64) -> u16x32 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); - let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); let result_bytes = u8x64 { val: crate::support::Aligned512(result), simd: token, @@ -16313,9 +16298,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: i32x16, indices: u8x64) -> i32x16 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); - let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); let result_bytes = u8x64 { val: crate::support::Aligned512(result), simd: token, @@ -16906,9 +16890,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: u32x16, indices: u8x64) -> u32x16 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); - let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); let result_bytes = u8x64 { val: crate::support::Aligned512(result), simd: token, @@ -17624,9 +17607,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: f64x8, indices: u8x64) -> f64x8 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); - let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); let result_bytes = u8x64 { val: crate::support::Aligned512(result), simd: token, @@ -18228,9 +18210,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: i64x8, indices: u8x64) -> i64x8 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); - let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); let result_bytes = u8x64 { val: crate::support::Aligned512(result), simd: token, @@ -18753,9 +18734,8 @@ impl Simd for Avx512 { fn kernel(token: Avx512, a: u64x8, indices: u8x64) -> u64x8 { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = - _mm512_cmp_epu8_mask::<{ _MM_CMPINT_LT }>(indices, _mm512_set1_epi8(64)); - let result = _mm512_maskz_permutexvar_epi8(in_range, indices, bytes); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); let result_bytes = u8x64 { val: crate::support::Aligned512(result), simd: token, diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 51fdbca6..815e0d0c 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2873,16 +2873,19 @@ impl X86 { let result_bytes = #bytes { val: #wrapper(result), simd: #token }; }, (Self::Avx512, 128 | 256 | 512) => { - let cmp = intrinsic_ident("cmp", "epu8_mask", vec_ty.n_bits()); - let maskz_permute = - intrinsic_ident("maskz_permutexvar", "epi8", vec_ty.n_bits()); + let min = intrinsic_ident("min", "epu8", vec_ty.n_bits()); + let permute = intrinsic_ident("permutex2var", "epi8", vec_ty.n_bits()); let set1 = set1_intrinsic(&bytes_ty); + let setzero = + intrinsic_ident("setzero", coarse_type(&bytes_ty), vec_ty.n_bits()); let byte_count = signed_literal(bytes_ty.len as u64, 8); quote! { let bytes = Bytes::to_bytes(a).val.0; let indices = indices.into(); - let in_range = #cmp::<{ _MM_CMPINT_LT }>(indices, #set1(#byte_count)); - let result = #maskz_permute(in_range, indices, bytes); + // Clamp out-of-range indices to the first byte of a + // second, all-zero table. + let indices = #min(indices, #set1(#byte_count)); + let result = #permute(bytes, indices, #setzero()); let result_bytes = #bytes { val: #wrapper(result), simd: #token }; } } From b5832c66532da2193a46915a83ce438a2da42831 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Tue, 4 Aug 2026 11:29:46 +0100 Subject: [PATCH 21/23] Use a better formulation for fallback swizzle_dyn_precise that lowers into one cmov per element instead of branches on SSE2 and SSE4.2 and autovectorizes on RISC-V --- fearless_simd/src/generated/fallback.rs | 4780 +---------------------- fearless_simd/src/generated/sse2.rs | 4780 +---------------------- fearless_simd/src/generated/sse4_2.rs | 2660 +------------ fearless_simd/src/generated/wasm.rs | 2660 +------------ fearless_simd_gen/src/mk_fallback.rs | 25 +- 5 files changed, 572 insertions(+), 14333 deletions(-) diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index 31735197..64744c6a 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -339,73 +339,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -961,73 +901,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -1808,73 +1688,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -2969,73 +2789,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -3585,73 +3345,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -4397,73 +4097,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -4899,73 +4539,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -5537,73 +5117,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -5981,73 +5501,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -6409,73 +5869,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -6928,137 +6328,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -7551,137 +6827,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -8097,137 +7249,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -8702,137 +7730,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -9188,137 +8092,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -9774,137 +8554,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -10229,137 +8885,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -10782,137 +9314,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -11267,137 +9675,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -11701,137 +9985,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -12301,265 +10461,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -13203,265 +11111,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -13998,265 +11654,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -14861,265 +12265,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -15538,265 +12690,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -16314,265 +13214,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -16926,265 +13574,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -17639,265 +14035,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -18261,265 +14405,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -18832,265 +14724,13 @@ impl Simd for Fallback { #[inline(always)] fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index c951bed8..303b7951 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -361,73 +361,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -1033,73 +973,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -1664,73 +1544,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -2457,73 +2277,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -3001,73 +2761,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -3745,73 +3445,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -4297,73 +3937,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -5042,73 +4622,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -5592,73 +5112,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -6087,73 +5547,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { let bytes = Bytes::to_bytes(a); - let result: u8x16 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); + output[lane] = bytes[index & (16usize - 1)] & in_range; + } + let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -6706,137 +6106,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -7343,137 +6619,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -7903,137 +7055,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -8527,137 +7555,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -9027,137 +7931,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -9631,137 +8411,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -10100,137 +8756,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -10672,137 +9204,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -11171,137 +9579,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -11619,137 +9903,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { let bytes = Bytes::to_bytes(a); - let result: u8x32 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); + output[lane] = bytes[index & (32usize - 1)] & in_range; + } + let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -12238,265 +10398,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -13199,265 +11107,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -14008,265 +11664,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -14890,265 +12294,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -15581,265 +12733,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -16385,265 +13285,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -17011,265 +13659,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -17788,265 +14184,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -18424,265 +14568,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -19009,265 +14901,13 @@ impl Simd for Sse2 { #[inline(always)] fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index db6656fb..ee7756c5 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -9821,265 +9821,13 @@ impl Simd for Sse4_2 { #[inline(always)] fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -10782,265 +10530,13 @@ impl Simd for Sse4_2 { #[inline(always)] fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -11591,265 +11087,13 @@ impl Simd for Sse4_2 { #[inline(always)] fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -12514,265 +11758,13 @@ impl Simd for Sse4_2 { #[inline(always)] fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -13205,265 +12197,13 @@ impl Simd for Sse4_2 { #[inline(always)] fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -14046,265 +12786,13 @@ impl Simd for Sse4_2 { #[inline(always)] fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -14672,265 +13160,13 @@ impl Simd for Sse4_2 { #[inline(always)] fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -15449,265 +13685,13 @@ impl Simd for Sse4_2 { #[inline(always)] fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -16085,265 +14069,13 @@ impl Simd for Sse4_2 { #[inline(always)] fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -16670,265 +14402,13 @@ impl Simd for Sse4_2 { #[inline(always)] fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index ffdc1357..3616cfae 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -7833,265 +7833,13 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -8761,265 +8509,13 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -9566,265 +9062,13 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -10432,265 +9676,13 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -11122,265 +10114,13 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -11914,265 +10654,13 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -12539,265 +11027,13 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -13286,265 +11522,13 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -13921,265 +11905,13 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] @@ -14505,265 +12237,13 @@ impl Simd for WasmSimd128 { #[inline(always)] fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { let bytes = Bytes::to_bytes(a); - let result: u8x64 = [ - { - let index = indices[0usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[1usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[2usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[3usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[4usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[5usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[6usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[7usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[8usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[9usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[10usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[11usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[12usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[13usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[14usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[15usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[16usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[17usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[18usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[19usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[20usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[21usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[22usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[23usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[24usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[25usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[26usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[27usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[28usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[29usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[30usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[31usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[32usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[33usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[34usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[35usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[36usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[37usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[38usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[39usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[40usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[41usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[42usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[43usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[44usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[45usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[46usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[47usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[48usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[49usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[50usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[51usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[52usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[53usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[54usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[55usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[56usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[57usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[58usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[59usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[60usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[61usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[62usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - { - let index = indices[63usize] as usize; - bytes.get(index).copied().unwrap_or(0) - }, - ] - .simd_into(self); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); + output[lane] = bytes[index & (64usize - 1)] & in_range; + } + let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) } #[inline(always)] diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index b0593ed8..26767c31 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -443,23 +443,22 @@ impl Level for Fallback { let bytes_ty = vec_ty.bytes_ty(); let bytes_rust = bytes_ty.rust(); let byte_count = bytes_ty.len; - let items = make_list( - (0..byte_count) - .map(|idx| { - quote! { - { - let index = indices[#idx] as usize; - bytes.get(index).copied().unwrap_or(0) - } - } - }) - .collect::>(), - ); + // Keep the load unconditionally in bounds so LLVM can select zero without a branch. + // This formulation lowers into one cmov per element on SSE2/SSE4.2 + // and autovectorizes on RISC-V. + // Every byte-vector width is a power of two, so masking preserves all valid indices. + assert!(byte_count.is_power_of_two()); quote! { #method_sig { let bytes = Bytes::to_bytes(a); - let result: #bytes_rust = #items.simd_into(self); + let mut output = [0u8; #byte_count]; + for lane in 0..#byte_count { + let index = indices[lane] as usize; + let in_range = 0u8.wrapping_sub(u8::from(index < #byte_count)); + output[lane] = bytes[index & (#byte_count - 1)] & in_range; + } + let result: #bytes_rust = output.simd_into(self); Bytes::from_bytes(result) } } From dc70e3f5e7846c742ee9e2e4ba7699bef1908880 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Tue, 4 Aug 2026 11:32:57 +0100 Subject: [PATCH 22/23] Placate clippy --- fearless_simd_gen/src/mk_fallback.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index 26767c31..62ddbc4d 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -447,7 +447,10 @@ impl Level for Fallback { // This formulation lowers into one cmov per element on SSE2/SSE4.2 // and autovectorizes on RISC-V. // Every byte-vector width is a power of two, so masking preserves all valid indices. - assert!(byte_count.is_power_of_two()); + assert!( + byte_count.is_power_of_two(), + "What have you done to your vector types?!" + ); quote! { #method_sig { From 685d429864ef57f88fd835af34d0eb3cc496f240 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Tue, 4 Aug 2026 11:42:43 +0100 Subject: [PATCH 23/23] Simplify the fallback formulation for swizzle_dyn_precise --- fearless_simd/src/generated/fallback.rs | 120 ++++++++++++------------ fearless_simd/src/generated/sse2.rs | 120 ++++++++++++------------ fearless_simd/src/generated/sse4_2.rs | 40 ++++---- fearless_simd/src/generated/wasm.rs | 40 ++++---- fearless_simd_gen/src/mk_fallback.rs | 13 +-- 5 files changed, 164 insertions(+), 169 deletions(-) diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index 64744c6a..98a47e13 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -342,8 +342,8 @@ impl Simd for Fallback { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -904,8 +904,8 @@ impl Simd for Fallback { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -1691,8 +1691,8 @@ impl Simd for Fallback { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -2792,8 +2792,8 @@ impl Simd for Fallback { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -3348,8 +3348,8 @@ impl Simd for Fallback { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -4100,8 +4100,8 @@ impl Simd for Fallback { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -4542,8 +4542,8 @@ impl Simd for Fallback { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -5120,8 +5120,8 @@ impl Simd for Fallback { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -5504,8 +5504,8 @@ impl Simd for Fallback { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -5872,8 +5872,8 @@ impl Simd for Fallback { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -6331,8 +6331,8 @@ impl Simd for Fallback { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -6830,8 +6830,8 @@ impl Simd for Fallback { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -7252,8 +7252,8 @@ impl Simd for Fallback { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -7733,8 +7733,8 @@ impl Simd for Fallback { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -8095,8 +8095,8 @@ impl Simd for Fallback { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -8557,8 +8557,8 @@ impl Simd for Fallback { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -8888,8 +8888,8 @@ impl Simd for Fallback { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -9317,8 +9317,8 @@ impl Simd for Fallback { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -9678,8 +9678,8 @@ impl Simd for Fallback { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -9988,8 +9988,8 @@ impl Simd for Fallback { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -10464,8 +10464,8 @@ impl Simd for Fallback { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -11114,8 +11114,8 @@ impl Simd for Fallback { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -11657,8 +11657,8 @@ impl Simd for Fallback { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -12268,8 +12268,8 @@ impl Simd for Fallback { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -12693,8 +12693,8 @@ impl Simd for Fallback { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -13217,8 +13217,8 @@ impl Simd for Fallback { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -13577,8 +13577,8 @@ impl Simd for Fallback { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -14038,8 +14038,8 @@ impl Simd for Fallback { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -14408,8 +14408,8 @@ impl Simd for Fallback { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -14727,8 +14727,8 @@ impl Simd for Fallback { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index 303b7951..5fbc1dbd 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -364,8 +364,8 @@ impl Simd for Sse2 { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -976,8 +976,8 @@ impl Simd for Sse2 { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -1547,8 +1547,8 @@ impl Simd for Sse2 { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -2280,8 +2280,8 @@ impl Simd for Sse2 { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -2764,8 +2764,8 @@ impl Simd for Sse2 { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -3448,8 +3448,8 @@ impl Simd for Sse2 { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -3940,8 +3940,8 @@ impl Simd for Sse2 { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -4625,8 +4625,8 @@ impl Simd for Sse2 { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -5115,8 +5115,8 @@ impl Simd for Sse2 { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -5550,8 +5550,8 @@ impl Simd for Sse2 { let mut output = [0u8; 16usize]; for lane in 0..16usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 16usize)); - output[lane] = bytes[index & (16usize - 1)] & in_range; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; } let result: u8x16 = output.simd_into(self); Bytes::from_bytes(result) @@ -6109,8 +6109,8 @@ impl Simd for Sse2 { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -6622,8 +6622,8 @@ impl Simd for Sse2 { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -7058,8 +7058,8 @@ impl Simd for Sse2 { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -7558,8 +7558,8 @@ impl Simd for Sse2 { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -7934,8 +7934,8 @@ impl Simd for Sse2 { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -8414,8 +8414,8 @@ impl Simd for Sse2 { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -8759,8 +8759,8 @@ impl Simd for Sse2 { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -9207,8 +9207,8 @@ impl Simd for Sse2 { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -9582,8 +9582,8 @@ impl Simd for Sse2 { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -9906,8 +9906,8 @@ impl Simd for Sse2 { let mut output = [0u8; 32usize]; for lane in 0..32usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 32usize)); - output[lane] = bytes[index & (32usize - 1)] & in_range; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; } let result: u8x32 = output.simd_into(self); Bytes::from_bytes(result) @@ -10401,8 +10401,8 @@ impl Simd for Sse2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -11110,8 +11110,8 @@ impl Simd for Sse2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -11667,8 +11667,8 @@ impl Simd for Sse2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -12297,8 +12297,8 @@ impl Simd for Sse2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -12736,8 +12736,8 @@ impl Simd for Sse2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -13288,8 +13288,8 @@ impl Simd for Sse2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -13662,8 +13662,8 @@ impl Simd for Sse2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -14187,8 +14187,8 @@ impl Simd for Sse2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -14571,8 +14571,8 @@ impl Simd for Sse2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -14904,8 +14904,8 @@ impl Simd for Sse2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index ee7756c5..c3139d09 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -9824,8 +9824,8 @@ impl Simd for Sse4_2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -10533,8 +10533,8 @@ impl Simd for Sse4_2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -11090,8 +11090,8 @@ impl Simd for Sse4_2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -11761,8 +11761,8 @@ impl Simd for Sse4_2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -12200,8 +12200,8 @@ impl Simd for Sse4_2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -12789,8 +12789,8 @@ impl Simd for Sse4_2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -13163,8 +13163,8 @@ impl Simd for Sse4_2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -13688,8 +13688,8 @@ impl Simd for Sse4_2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -14072,8 +14072,8 @@ impl Simd for Sse4_2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -14405,8 +14405,8 @@ impl Simd for Sse4_2 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index 3616cfae..b7828ec3 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -7836,8 +7836,8 @@ impl Simd for WasmSimd128 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -8512,8 +8512,8 @@ impl Simd for WasmSimd128 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -9065,8 +9065,8 @@ impl Simd for WasmSimd128 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -9679,8 +9679,8 @@ impl Simd for WasmSimd128 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -10117,8 +10117,8 @@ impl Simd for WasmSimd128 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -10657,8 +10657,8 @@ impl Simd for WasmSimd128 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -11030,8 +11030,8 @@ impl Simd for WasmSimd128 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -11525,8 +11525,8 @@ impl Simd for WasmSimd128 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -11908,8 +11908,8 @@ impl Simd for WasmSimd128 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) @@ -12240,8 +12240,8 @@ impl Simd for WasmSimd128 { let mut output = [0u8; 64usize]; for lane in 0..64usize { let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < 64usize)); - output[lane] = bytes[index & (64usize - 1)] & in_range; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; } let result: u8x64 = output.simd_into(self); Bytes::from_bytes(result) diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index 62ddbc4d..4e5451d5 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -443,23 +443,18 @@ impl Level for Fallback { let bytes_ty = vec_ty.bytes_ty(); let bytes_rust = bytes_ty.rust(); let byte_count = bytes_ty.len; - // Keep the load unconditionally in bounds so LLVM can select zero without a branch. // This formulation lowers into one cmov per element on SSE2/SSE4.2 // and autovectorizes on RISC-V. - // Every byte-vector width is a power of two, so masking preserves all valid indices. - assert!( - byte_count.is_power_of_two(), - "What have you done to your vector types?!" - ); - quote! { #method_sig { let bytes = Bytes::to_bytes(a); let mut output = [0u8; #byte_count]; for lane in 0..#byte_count { + // Keep the load unconditionally in bounds so LLVM can always execute it, + // and select zero afterwards. This avoids a branch that could be mispredicted. let index = indices[lane] as usize; - let in_range = 0u8.wrapping_sub(u8::from(index < #byte_count)); - output[lane] = bytes[index & (#byte_count - 1)] & in_range; + let value = bytes[index % #byte_count]; + output[lane] = if index < #byte_count { value } else { 0 }; } let result: #bytes_rust = output.simd_into(self); Bytes::from_bytes(result)