diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 0a3ecd73..63c2087d 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -238,6 +238,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x4, indices: u8x16) -> f32x4 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -820,6 +838,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i8x16, indices: u8x16) -> i8x16 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1308,6 +1344,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x16, indices: u8x16) -> u8x16 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1947,6 +2001,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i16x8, indices: u8x16) -> i16x8 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2364,6 +2436,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u16x8, indices: u8x16) -> u16x8 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2942,6 +3032,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i32x4, indices: u8x16) -> i32x4 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -3343,6 +3451,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u32x4, indices: u8x16) -> u32x4 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -3919,6 +4045,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x2, indices: u8x16) -> f64x2 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -4362,6 +4506,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x2, indices: u8x16) -> i64x2 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -4730,6 +4892,24 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x2, indices: u8x16) -> u64x2 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5293,6 +5473,29 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = Bytes::to_bytes(a); + let idxs = indices; + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] @@ -6002,6 +6205,29 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = Bytes::to_bytes(a); + let idxs = indices; + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] @@ -6670,6 +6896,29 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = Bytes::to_bytes(a); + let idxs = indices; + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] @@ -7479,6 +7728,29 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = Bytes::to_bytes(a); + let idxs = indices; + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] @@ -8038,6 +8310,29 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = Bytes::to_bytes(a); + let idxs = indices; + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -8769,6 +9064,29 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = Bytes::to_bytes(a); + let idxs = indices; + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] @@ -9252,6 +9570,29 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = Bytes::to_bytes(a); + let idxs = indices; + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] @@ -9914,6 +10255,29 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = Bytes::to_bytes(a); + let idxs = indices; + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { crate::kernel!( #[inline(always)] @@ -10420,6 +10784,29 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4, indices: u8x32) -> i64x4 { + let bytes = Bytes::to_bytes(a); + let idxs = indices; + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { crate::kernel!( #[inline(always)] @@ -10865,6 +11252,29 @@ impl Simd for Avx2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4, indices: u8x32) -> u64x4 { + let bytes = Bytes::to_bytes(a); + let idxs = indices; + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { crate::kernel!( #[inline(always)] @@ -11511,6 +11921,33 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x16, indices: u8x64) -> f32x16 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -12208,6 +12645,33 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i8x64, indices: u8x64) -> i8x64 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -12753,6 +13217,33 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x64, indices: u8x64) -> u8x64 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -13406,6 +13897,33 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i16x32, indices: u8x64) -> i16x32 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -13833,6 +14351,33 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u16x32, indices: u8x64) -> u16x32 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -14422,6 +14967,33 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i32x16, indices: u8x64) -> i32x16 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -14784,6 +15356,33 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u32x16, indices: u8x64) -> u32x16 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -15318,6 +15917,33 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x8, indices: u8x64) -> f64x8 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) @@ -15690,6 +16316,33 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x8, indices: u8x64) -> i64x8 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { let (a0, a1) = self.split_i64x8(a); let (b0, b1) = self.split_i64x8(b); @@ -16011,6 +16664,33 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x8, indices: u8x64) -> u64x8 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x64(bytes); + let (indices_low, indices_high) = token.split_u8x64(indices); + let high_table_offset = token.splat_u8x32(32); + let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_low, high_table_offset), + ); + let output_low = token.or_u8x32(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x32( + table_high, + token.sub_u8x32(indices_high, high_table_offset), + ); + let output_high = token.or_u8x32(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x32(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { let (a0, a1) = self.split_u64x8(a); let (b0, b1) = self.split_u64x8(b); diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index d26117f1..7c44687a 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -246,6 +246,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, indices: u8x16) -> f32x4 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -826,6 +844,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, indices: u8x16) -> i8x16 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1354,6 +1390,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, indices: u8x16) -> u8x16 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1982,6 +2036,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, indices: u8x16) -> i16x8 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2428,6 +2500,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, indices: u8x16) -> u16x8 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2983,6 +3073,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, indices: u8x16) -> i32x4 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -3413,6 +3521,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, indices: u8x16) -> u32x4 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -3961,6 +4087,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, indices: u8x16) -> f64x2 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -4432,6 +4576,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, indices: u8x16) -> i64x2 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -4842,6 +5004,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x2, indices: u8x16) -> u64x2 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm_min_epu8(indices, _mm_set1_epi8(16)); + let result = _mm_permutex2var_epi8(bytes, indices, _mm_setzero_si128()); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5430,6 +5610,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] @@ -6175,6 +6373,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] @@ -6874,6 +7090,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] @@ -7659,6 +7893,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] @@ -8232,6 +8484,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -8918,6 +9188,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] @@ -9443,6 +9731,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] @@ -10092,6 +10398,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { crate::kernel!( #[inline(always)] @@ -10650,6 +10974,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x4, indices: u8x32) -> i64x4 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { crate::kernel!( #[inline(always)] @@ -11133,6 +11475,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x4, indices: u8x32) -> u64x4 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm256_min_epu8(indices, _mm256_set1_epi8(32)); + let result = _mm256_permutex2var_epi8(bytes, indices, _mm256_setzero_si256()); + let result_bytes = u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { crate::kernel!( #[inline(always)] @@ -11808,6 +12168,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, indices: u8x64) -> f32x16 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { crate::kernel!( #[inline(always)] @@ -12724,6 +13102,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, indices: u8x64) -> i8x64 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { crate::kernel!( #[inline(always)] @@ -13559,6 +13955,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, indices: u8x64) -> u8x64 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { crate::kernel!( #[inline(always)] @@ -14437,6 +14851,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, indices: u8x64) -> i16x32 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { crate::kernel!( #[inline(always)] @@ -15083,6 +15515,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, indices: u8x64) -> u16x32 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { crate::kernel!( #[inline(always)] @@ -15842,6 +16292,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, indices: u8x64) -> i32x16 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { crate::kernel!( #[inline(always)] @@ -16416,6 +16884,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, indices: u8x64) -> u32x16 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { crate::kernel!( #[inline(always)] @@ -17115,6 +17601,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, indices: u8x64) -> f64x8 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { crate::kernel!( #[inline(always)] @@ -17700,6 +18204,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, indices: u8x64) -> i64x8 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { crate::kernel!( #[inline(always)] @@ -18206,6 +18728,24 @@ impl Simd for Avx512 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u64x8, indices: u8x64) -> u64x8 { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + let indices = _mm512_min_epu8(indices, _mm512_set1_epi8(64)); + let result = _mm512_permutex2var_epi8(bytes, indices, _mm512_setzero_si512()); + let result_bytes = u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { crate::kernel!( #[inline(always)] diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index 9f682274..98a47e13 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -337,6 +337,18 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { [ f32::abs(a[0usize]), @@ -887,6 +899,18 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { [ i8::wrapping_add(a[0usize], b[0usize]), @@ -1662,6 +1686,18 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { [ u8::wrapping_add(a[0usize], b[0usize]), @@ -2751,6 +2787,18 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { [ i16::wrapping_add(a[0usize], b[0usize]), @@ -3295,6 +3343,18 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { [ u16::wrapping_add(a[0usize], b[0usize]), @@ -4035,6 +4095,18 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { [ i32::wrapping_add(a[0usize], b[0usize]), @@ -4465,6 +4537,18 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { [ u32::wrapping_add(a[0usize], b[0usize]), @@ -5031,6 +5115,18 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { [f64::abs(a[0usize]), f64::abs(a[1usize])].simd_into(self) } @@ -5403,6 +5499,18 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { [ i64::wrapping_add(a[0usize], b[0usize]), @@ -5759,6 +5867,18 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { [ u64::wrapping_add(a[0usize], b[0usize]), @@ -6206,6 +6326,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -6693,6 +6825,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -7103,6 +7247,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -7572,6 +7728,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -7922,6 +8090,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -8372,6 +8552,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -8691,6 +8883,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -9108,6 +9312,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -9457,6 +9673,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { let (a0, a1) = self.split_i64x4(a); let (b0, b1) = self.split_i64x4(b); @@ -9755,6 +9983,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { let (a0, a1) = self.split_u64x4(a); let (b0, b1) = self.split_u64x4(b); @@ -10219,6 +10459,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -10857,6 +11109,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -11388,6 +11652,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -11987,6 +12263,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -12400,6 +12688,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -12912,6 +13212,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -13260,6 +13572,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -13709,6 +14033,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) @@ -14067,6 +14403,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { let (a0, a1) = self.split_i64x8(a); let (b0, b1) = self.split_i64x8(b); @@ -14374,6 +14722,18 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { let (a0, a1) = self.split_u64x8(a); let (b0, b1) = self.split_u64x8(b); diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index 5c9d1882..3cb8d5ef 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -225,6 +225,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f32x4, indices: u8x16) -> f32x4 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -762,6 +776,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i8x16, indices: u8x16) -> i8x16 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1208,6 +1236,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u8x16, indices: u8x16) -> u8x16 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1811,6 +1853,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i16x8, indices: u8x16) -> i16x8 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2225,6 +2281,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u16x8, indices: u8x16) -> u16x8 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2792,6 +2862,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i32x4, indices: u8x16) -> i32x4 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -3200,6 +3284,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u32x4, indices: u8x16) -> u32x4 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -3768,6 +3866,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f64x2, indices: u8x16) -> f64x2 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -4221,6 +4333,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x2, indices: u8x16) -> i64x2 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -4605,6 +4731,20 @@ impl Simd for Neon { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x2, indices: u8x16) -> u64x2 { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5196,6 +5336,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f32x8, indices: u8x32) -> f32x8 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + Bytes::from_bytes(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -5720,6 +5877,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i8x32, indices: u8x32) -> i8x32 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + Bytes::from_bytes(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -6167,6 +6341,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u8x32, indices: u8x32) -> u8x32 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + Bytes::from_bytes(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -6680,6 +6871,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i16x16, indices: u8x32) -> i16x16 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + Bytes::from_bytes(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -7067,6 +7275,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u16x16, indices: u8x32) -> u16x16 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + Bytes::from_bytes(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -7552,6 +7777,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i32x8, indices: u8x32) -> i32x8 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + Bytes::from_bytes(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -7908,6 +8150,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u32x8, indices: u8x32) -> u32x8 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + Bytes::from_bytes(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -8369,6 +8628,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f64x4, indices: u8x32) -> f64x4 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + Bytes::from_bytes(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -8755,6 +9031,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x4, indices: u8x32) -> i64x4 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + Bytes::from_bytes(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { let (a0, a1) = self.split_i64x4(a); let (b0, b1) = self.split_i64x4(b); @@ -9090,6 +9383,23 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x4, indices: u8x32) -> u64x4 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = + uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1)); + Bytes::from_bytes(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { let (a0, a1) = self.split_u64x4(a); let (b0, b1) = self.split_u64x4(b); @@ -9616,6 +9926,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f32x16, indices: u8x64) -> f32x16 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + Bytes::from_bytes(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -10285,6 +10616,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i8x64, indices: u8x64) -> i8x64 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + Bytes::from_bytes(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -10869,6 +11221,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u8x64, indices: u8x64) -> u8x64 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + Bytes::from_bytes(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -11449,6 +11822,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i16x32, indices: u8x64) -> i16x32 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + Bytes::from_bytes(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -11915,6 +12309,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u16x32, indices: u8x64) -> u16x32 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + Bytes::from_bytes(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -12445,6 +12860,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i32x16, indices: u8x64) -> i32x16 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + Bytes::from_bytes(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -12846,6 +13282,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u32x16, indices: u8x64) -> u32x16 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + Bytes::from_bytes(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -13331,6 +13788,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f64x8, indices: u8x64) -> f64x8 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + Bytes::from_bytes(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) @@ -13742,6 +14220,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i64x8, indices: u8x64) -> i64x8 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + Bytes::from_bytes(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { let (a0, a1) = self.split_i64x8(a); let (b0, b1) = self.split_i64x8(b); @@ -14102,6 +14601,27 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u64x8, indices: u8x64) -> u64x8 { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + Bytes::from_bytes(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { let (a0, a1) = self.split_u64x8(a); let (b0, b1) = self.split_u64x8(b); diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index a75874eb..6ce2f534 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -201,6 +201,8 @@ pub trait Simd: ) -> f32x4; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4; #[doc = "Compute the absolute value of each element."] fn abs_f32x4(self, a: f32x4) -> f32x4; #[doc = "Negate each element of the vector."] @@ -315,6 +317,8 @@ pub trait Simd: ) -> i8x16; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -409,6 +413,8 @@ pub trait Simd: ) -> u8x16; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -542,6 +548,8 @@ pub trait Simd: ) -> i16x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -636,6 +644,8 @@ pub trait Simd: ) -> u16x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -767,6 +777,8 @@ pub trait Simd: ) -> i32x4; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -863,6 +875,8 @@ pub trait Simd: ) -> u32x4; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -996,6 +1010,8 @@ pub trait Simd: ) -> f64x2; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2; #[doc = "Compute the absolute value of each element."] fn abs_f64x2(self, a: f64x2) -> f64x2; #[doc = "Negate each element of the vector."] @@ -1102,6 +1118,8 @@ pub trait Simd: ) -> i64x2; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i64x2(self, a: i64x2, indices: u8x16) -> i64x2; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1196,6 +1214,8 @@ pub trait Simd: ) -> u64x2; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u64x2(self, a: u64x2, indices: u8x16) -> u64x2; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1327,6 +1347,8 @@ pub trait Simd: ) -> f32x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8; #[doc = "Compute the absolute value of each element."] fn abs_f32x8(self, a: f32x8) -> f32x8; #[doc = "Negate each element of the vector."] @@ -1443,6 +1465,8 @@ pub trait Simd: ) -> i8x32; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1539,6 +1563,8 @@ pub trait Simd: ) -> u8x32; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1680,6 +1706,8 @@ pub trait Simd: a: i16x16, indices: u8x32, ) -> i16x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1780,6 +1808,8 @@ pub trait Simd: a: u16x16, indices: u8x32, ) -> u16x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1917,6 +1947,8 @@ pub trait Simd: ) -> i32x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2015,6 +2047,8 @@ pub trait Simd: ) -> u32x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2152,6 +2186,8 @@ pub trait Simd: ) -> f64x4; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4; #[doc = "Compute the absolute value of each element."] fn abs_f64x4(self, a: f64x4) -> f64x4; #[doc = "Negate each element of the vector."] @@ -2260,6 +2296,8 @@ pub trait Simd: ) -> i64x4; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i64x4(self, a: i64x4, indices: u8x32) -> i64x4; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2356,6 +2394,8 @@ pub trait Simd: ) -> u64x4; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u64x4(self, a: u64x4, indices: u8x32) -> u64x4; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2495,6 +2535,8 @@ pub trait Simd: a: f32x16, indices: u8x64, ) -> f32x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16; #[doc = "Compute the absolute value of each element."] fn abs_f32x16(self, a: f32x16) -> f32x16; #[doc = "Negate each element of the vector."] @@ -2613,6 +2655,8 @@ pub trait Simd: ) -> i8x64; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2707,6 +2751,8 @@ pub trait Simd: ) -> u8x64; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2846,6 +2892,8 @@ pub trait Simd: a: i16x32, indices: u8x64, ) -> i16x32; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2944,6 +2992,8 @@ pub trait Simd: a: u16x32, indices: u8x64, ) -> u16x32; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -3085,6 +3135,8 @@ pub trait Simd: a: i32x16, indices: u8x64, ) -> i32x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -3185,6 +3237,8 @@ pub trait Simd: a: u32x16, indices: u8x64, ) -> u32x16; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -3322,6 +3376,8 @@ pub trait Simd: ) -> f64x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8; #[doc = "Compute the absolute value of each element."] fn abs_f64x8(self, a: f64x8) -> f64x8; #[doc = "Negate each element of the vector."] @@ -3428,6 +3484,8 @@ pub trait Simd: ) -> i64x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i64x8(self, a: i64x8, indices: u8x64) -> i64x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -3522,6 +3580,8 @@ pub trait Simd: ) -> u64x8; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u64x8(self, a: u64x8, indices: u8x64) -> u64x8; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -3751,6 +3811,8 @@ pub trait SimdBase: fn shift_elements_right(self, padding: Self::Element) -> Self; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self; + #[doc = "Dynamically swizzle this vector's bytes across the whole vector.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes."] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self; } #[doc = r" Functionality implemented by floating-point SIMD vectors."] pub trait SimdFloat: diff --git a/fearless_simd/src/generated/simd_types.rs b/fearless_simd/src/generated/simd_types.rs index ea3ad41b..ce9cc6a6 100644 --- a/fearless_simd/src/generated/simd_types.rs +++ b/fearless_simd/src/generated/simd_types.rs @@ -159,6 +159,11 @@ impl SimdBase for f32x4 { self.simd .swizzle_dyn_within_blocks_f32x4(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_f32x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f32x4 { #[inline(always)] @@ -458,6 +463,11 @@ impl SimdBase for i8x16 { self.simd .swizzle_dyn_within_blocks_i8x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i8x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i8x16 { #[inline(always)] @@ -689,6 +699,11 @@ impl SimdBase for u8x16 { self.simd .swizzle_dyn_within_blocks_u8x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u8x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u8x16 { #[inline(always)] @@ -1003,6 +1018,11 @@ impl SimdBase for i16x8 { self.simd .swizzle_dyn_within_blocks_i16x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i16x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i16x8 { #[inline(always)] @@ -1226,6 +1246,11 @@ impl SimdBase for u16x8 { self.simd .swizzle_dyn_within_blocks_u16x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u16x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u16x8 { #[inline(always)] @@ -1531,6 +1556,11 @@ impl SimdBase for i32x4 { self.simd .swizzle_dyn_within_blocks_i32x4(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i32x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i32x4 { #[inline(always)] @@ -1757,6 +1787,11 @@ impl SimdBase for u32x4 { self.simd .swizzle_dyn_within_blocks_u32x4(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u32x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u32x4 { #[inline(always)] @@ -2074,6 +2109,11 @@ impl SimdBase for f64x2 { self.simd .swizzle_dyn_within_blocks_f64x2(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_f64x2(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f64x2 { #[inline(always)] @@ -2342,6 +2382,11 @@ impl SimdBase for i64x2 { self.simd .swizzle_dyn_within_blocks_i64x2(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i64x2(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i64x2 { #[inline(always)] @@ -2556,6 +2601,11 @@ impl SimdBase for u64x2 { self.simd .swizzle_dyn_within_blocks_u64x2(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u64x2(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u64x2 { #[inline(always)] @@ -2870,6 +2920,11 @@ impl SimdBase for f32x8 { self.simd .swizzle_dyn_within_blocks_f32x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_f32x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f32x8 { #[inline(always)] @@ -3192,6 +3247,11 @@ impl SimdBase for i8x32 { self.simd .swizzle_dyn_within_blocks_i8x32(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i8x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i8x32 { #[inline(always)] @@ -3446,6 +3506,11 @@ impl SimdBase for u8x32 { self.simd .swizzle_dyn_within_blocks_u8x32(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u8x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u8x32 { #[inline(always)] @@ -3781,6 +3846,11 @@ impl SimdBase for i16x16 { self.simd .swizzle_dyn_within_blocks_i16x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i16x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i16x16 { #[inline(always)] @@ -4026,6 +4096,11 @@ impl SimdBase for u16x16 { self.simd .swizzle_dyn_within_blocks_u16x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u16x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u16x16 { #[inline(always)] @@ -4348,6 +4423,11 @@ impl SimdBase for i32x8 { self.simd .swizzle_dyn_within_blocks_i32x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i32x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i32x8 { #[inline(always)] @@ -4590,6 +4670,11 @@ impl SimdBase for u32x8 { self.simd .swizzle_dyn_within_blocks_u32x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u32x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u32x8 { #[inline(always)] @@ -4914,6 +4999,11 @@ impl SimdBase for f64x4 { self.simd .swizzle_dyn_within_blocks_f64x4(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_f64x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f64x4 { #[inline(always)] @@ -5189,6 +5279,11 @@ impl SimdBase for i64x4 { self.simd .swizzle_dyn_within_blocks_i64x4(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i64x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i64x4 { #[inline(always)] @@ -5410,6 +5505,11 @@ impl SimdBase for u64x4 { self.simd .swizzle_dyn_within_blocks_u64x4(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u64x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u64x4 { #[inline(always)] @@ -5746,6 +5846,11 @@ impl SimdBase for f32x16 { self.simd .swizzle_dyn_within_blocks_f32x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_f32x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f32x16 { #[inline(always)] @@ -6095,6 +6200,11 @@ impl SimdBase for i8x64 { self.simd .swizzle_dyn_within_blocks_i8x64(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i8x64(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i8x64 { #[inline(always)] @@ -6375,6 +6485,11 @@ impl SimdBase for u8x64 { self.simd .swizzle_dyn_within_blocks_u8x64(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u8x64(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u8x64 { #[inline(always)] @@ -6720,6 +6835,11 @@ impl SimdBase for i16x32 { self.simd .swizzle_dyn_within_blocks_i16x32(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i16x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i16x32 { #[inline(always)] @@ -6975,6 +7095,11 @@ impl SimdBase for u16x32 { self.simd .swizzle_dyn_within_blocks_u16x32(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u16x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u16x32 { #[inline(always)] @@ -7305,6 +7430,11 @@ impl SimdBase for i32x16 { self.simd .swizzle_dyn_within_blocks_i32x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i32x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i32x16 { #[inline(always)] @@ -7556,6 +7686,11 @@ impl SimdBase for u32x16 { self.simd .swizzle_dyn_within_blocks_u32x16(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u32x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u32x16 { #[inline(always)] @@ -7884,6 +8019,11 @@ impl SimdBase for f64x8 { self.simd .swizzle_dyn_within_blocks_f64x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_f64x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f64x8 { #[inline(always)] @@ -8162,6 +8302,11 @@ impl SimdBase for i64x8 { self.simd .swizzle_dyn_within_blocks_i64x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_i64x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i64x8 { #[inline(always)] @@ -8386,6 +8531,11 @@ impl SimdBase for u64x8 { self.simd .swizzle_dyn_within_blocks_u64x8(self, indices.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_precise_u64x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u64x8 { #[inline(always)] diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index 9169137a..5fbc1dbd 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -359,6 +359,18 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -959,6 +971,18 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1518,6 +1542,18 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -2239,6 +2275,18 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2711,6 +2759,18 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -3383,6 +3443,18 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -3863,6 +3935,18 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -4536,6 +4620,18 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -5014,6 +5110,18 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -5437,6 +5545,18 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 16usize]; + for lane in 0..16usize { + let index = indices[lane] as usize; + let value = bytes[index % 16usize]; + output[lane] = if index < 16usize { value } else { 0 }; + } + let result: u8x16 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5984,6 +6104,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -6485,6 +6617,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -6909,6 +7053,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -7397,6 +7553,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -7761,6 +7929,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -8229,6 +8409,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -8562,6 +8754,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -8998,6 +9202,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -9361,6 +9577,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { let (a0, a1) = self.split_i64x4(a); let (b0, b1) = self.split_i64x4(b); @@ -9673,6 +9901,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 32usize]; + for lane in 0..32usize { + let index = indices[lane] as usize; + let value = bytes[index % 32usize]; + output[lane] = if index < 32usize { value } else { 0 }; + } + let result: u8x32 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { let (a0, a1) = self.split_u64x4(a); let (b0, b1) = self.split_u64x4(b); @@ -10156,6 +10396,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -10853,6 +11105,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -11398,6 +11662,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -12016,6 +12292,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -12443,6 +12731,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -12983,6 +13283,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -13345,6 +13657,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -13858,6 +14182,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) @@ -14230,6 +14566,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { let (a0, a1) = self.split_i64x8(a); let (b0, b1) = self.split_i64x8(b); @@ -14551,6 +14899,18 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { let (a0, a1) = self.split_u64x8(a); let (b0, b1) = self.split_u64x8(b); diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 60f7da23..c3139d09 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -235,6 +235,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f32x4, indices: u8x16) -> f32x4 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -802,6 +820,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i8x16, indices: u8x16) -> i8x16 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1287,6 +1323,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u8x16, indices: u8x16) -> u8x16 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1923,6 +1977,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i16x8, indices: u8x16) -> i16x8 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2337,6 +2409,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u16x8, indices: u8x16) -> u16x8 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2909,6 +2999,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i32x4, indices: u8x16) -> i32x4 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -3307,6 +3415,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u32x4, indices: u8x16) -> u32x4 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -3877,6 +4003,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f64x2, indices: u8x16) -> f64x2 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -4305,6 +4449,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i64x2, indices: u8x16) -> i64x2 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -4661,6 +4823,24 @@ impl Simd for Sse4_2 { kernel(self, a, indices) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2, indices: u8x16) -> u64x2 { + let indices = indices.into(); + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }; + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5197,6 +5377,33 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -5698,6 +5905,33 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -6122,6 +6356,33 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -6610,6 +6871,33 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -6974,6 +7262,33 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -7442,6 +7757,33 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -7775,6 +8117,33 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -8211,6 +8580,33 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -8574,6 +8970,33 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i64x4, indices: u8x32) -> i64x4 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { let (a0, a1) = self.split_i64x4(a); let (b0, b1) = self.split_i64x4(b); @@ -8886,6 +9309,33 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x4, indices: u8x32) -> u64x4 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = token.split_u8x32(bytes); + let (indices_low, indices_high) = token.split_u8x32(indices); + let high_table_offset = token.splat_u8x16(16); + let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_low, high_table_offset), + ); + let output_low = token.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = token.swizzle_dyn_precise_u8x16( + table_high, + token.sub_u8x16(indices_high, high_table_offset), + ); + let output_high = token.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = token.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { let (a0, a1) = self.split_u64x4(a); let (b0, b1) = self.split_u64x4(b); @@ -9369,6 +9819,18 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -10066,6 +10528,18 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -10611,6 +11085,18 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -11270,6 +11756,18 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -11697,6 +12195,18 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -12274,6 +12784,18 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -12636,6 +13158,18 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -13149,6 +13683,18 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) @@ -13521,6 +14067,18 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { let (a0, a1) = self.split_i64x8(a); let (b0, b1) = self.split_i64x8(b); @@ -13842,6 +14400,18 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { let (a0, a1) = self.split_u64x8(a); let (b0, b1) = self.split_u64x8(b); diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index 288be79c..b7828ec3 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -212,6 +212,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { f32x4_abs(a.into()).simd_into(self) } @@ -580,6 +588,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { i8x16_add(a.into(), b.into()).simd_into(self) } @@ -911,6 +927,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { u8x16_add(a.into(), b.into()).simd_into(self) } @@ -1315,6 +1339,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { i16x8_add(a.into(), b.into()).simd_into(self) } @@ -1586,6 +1618,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { u16x8_add(a.into(), b.into()).simd_into(self) } @@ -1934,6 +1974,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { i32x4_add(a.into(), b.into()).simd_into(self) } @@ -2185,6 +2233,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { u32x4_add(a.into(), b.into()).simd_into(self) } @@ -2521,6 +2577,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { f64x2_abs(a.into()).simd_into(self) } @@ -2807,6 +2871,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_i64x2(self, a: i64x2, indices: u8x16) -> i64x2 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { i64x2_add(a.into(), b.into()).simd_into(self) } @@ -3050,6 +3122,14 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn swizzle_dyn_precise_u64x2(self, a: u64x2, indices: u8x16) -> u64x2 { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { u64x2_add(a.into(), b.into()).simd_into(self) } @@ -3437,6 +3517,23 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -3934,6 +4031,23 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -4354,6 +4468,23 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -4841,6 +4972,23 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x16(self, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -5204,6 +5352,23 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x16(self, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -5659,6 +5824,23 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -5991,6 +6173,23 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -6426,6 +6625,23 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -6788,6 +7004,23 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x4(self, a: i64x4, indices: u8x32) -> i64x4 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + #[inline(always)] fn add_i64x4(self, a: i64x4, b: i64x4) -> i64x4 { let (a0, a1) = self.split_i64x4(a); let (b0, b1) = self.split_i64x4(b); @@ -7099,6 +7332,23 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x4(self, a: u64x4, indices: u8x32) -> u64x4 { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = self.split_u8x32(bytes); + let (indices_low, indices_high) = self.split_u8x32(indices); + let high_table_offset = self.splat_u8x16(16); + let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low); + let output_low_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset)); + let output_low = self.or_u8x16(output_low_from_low, output_low_from_high); + let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high); + let output_high_from_high = self + .swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset)); + let output_high = self.or_u8x16(output_high_from_low, output_high_from_high); + let result_bytes = self.combine_u8x16(output_low, output_high); + Bytes::from_bytes(result_bytes) + } + #[inline(always)] fn add_u64x4(self, a: u64x4, b: u64x4) -> u64x4 { let (a0, a1) = self.split_u64x4(a); let (b0, b1) = self.split_u64x4(b); @@ -7581,6 +7831,18 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_f32x16(self, a: f32x16, indices: u8x64) -> f32x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -8245,6 +8507,18 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -8786,6 +9060,18 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -9388,6 +9674,18 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_i16x32(self, a: i16x32, indices: u8x64) -> i16x32 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -9814,6 +10112,18 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u16x32(self, a: u16x32, indices: u8x64) -> u16x32 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -10342,6 +10652,18 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_i32x16(self, a: i32x16, indices: u8x64) -> i32x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -10703,6 +11025,18 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u32x16(self, a: u32x16, indices: u8x64) -> u32x16 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -11186,6 +11520,18 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) @@ -11557,6 +11903,18 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_i64x8(self, a: i64x8, indices: u8x64) -> i64x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_i64x8(self, a: i64x8, b: i64x8) -> i64x8 { let (a0, a1) = self.split_i64x8(a); let (b0, b1) = self.split_i64x8(b); @@ -11877,6 +12235,18 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_precise_u64x8(self, a: u64x8, indices: u8x64) -> u64x8 { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; 64usize]; + for lane in 0..64usize { + let index = indices[lane] as usize; + let value = bytes[index % 64usize]; + output[lane] = if index < 64usize { value } else { 0 }; + } + let result: u8x64 = output.simd_into(self); + Bytes::from_bytes(result) + } + #[inline(always)] fn add_u64x8(self, a: u64x8, b: u64x8) -> u64x8 { let (a0, a1) = self.split_u64x8(a); let (b0, b1) = self.split_u64x8(b); diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index b53de485..89090a62 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -18,6 +18,54 @@ pub(crate) fn fallback_method(op: Op, vec_ty: &VecType) -> TokenStream { crate::mk_fallback::Fallback.make_method(op, vec_ty) } +pub(crate) fn recursive_swizzle_dyn_precise_body( + vec_ty: &VecType, + token: &T, +) -> TokenStream { + // We can take advantage of the "precise" property (out-of-range is 0) + // to assemble a double-vector-width arbitrary shuffle. + // The trick is to split the input into two, then run for each half of output indices J: + // ``` + // from_low = swizzle_H(low_table, J); + // from_high = swizzle_H(high_table, J.wrapping_sub(H)); + // result = from_low | from_high; + // ``` + // Since each element is out-of-range for at least one half, + // the final combine is very cheap: a bitwise or. + let bytes_ty = vec_ty.bytes_ty(); + let half_bytes_ty = VecType::new(ScalarType::Unsigned, 8, bytes_ty.len / 2); + let split_bytes = generic_op_name("split", &bytes_ty); + let combine_half_bytes = generic_op_name("combine", &half_bytes_ty); + let swizzle_half = generic_op_name("swizzle_dyn_precise", &half_bytes_ty); + let splat_half = generic_op_name("splat", &half_bytes_ty); + let sub_half = generic_op_name("sub", &half_bytes_ty); + let or_half = generic_op_name("or", &half_bytes_ty); + let half_len = Literal::u8_unsuffixed(u8::try_from(bytes_ty.len / 2).unwrap()); + + quote! { + let bytes = Bytes::to_bytes(a); + let (table_low, table_high) = #token.#split_bytes(bytes); + let (indices_low, indices_high) = #token.#split_bytes(indices); + let high_table_offset = #token.#splat_half(#half_len); + + let output_low_from_low = #token.#swizzle_half(table_low, indices_low); + let output_low_from_high = #token.#swizzle_half( + table_high, + #token.#sub_half(indices_low, high_table_offset), + ); + let output_low = #token.#or_half(output_low_from_low, output_low_from_high); + + let output_high_from_low = #token.#swizzle_half(table_low, indices_high); + let output_high_from_high = #token.#swizzle_half( + table_high, + #token.#sub_half(indices_high, high_table_offset), + ); + let output_high = #token.#or_half(output_high_from_low, output_high_from_high); + + let result_bytes = #token.#combine_half_bytes(output_low, output_high); + } +} + /// For backends that store masks as all-zero/all-one integer lanes, convert the public /// `bool` mask splat argument into the backend's lane representation. pub(crate) fn integer_lane_mask_splat_arg(vec_ty: &VecType) -> TokenStream { @@ -88,6 +136,9 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { } } } + OpSig::SwizzleDynPrecise => { + panic!("whole-vector swizzles cannot be done via split/combine"); + } OpSig::Ternary => { quote! { #method_sig { diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index fe4b1d72..4e5451d5 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -439,6 +439,28 @@ impl Level for Fallback { } } } + OpSig::SwizzleDynPrecise => { + let bytes_ty = vec_ty.bytes_ty(); + let bytes_rust = bytes_ty.rust(); + let byte_count = bytes_ty.len; + // This formulation lowers into one cmov per element on SSE2/SSE4.2 + // and autovectorizes on RISC-V. + quote! { + #method_sig { + let bytes = Bytes::to_bytes(a); + let mut output = [0u8; #byte_count]; + for lane in 0..#byte_count { + // Keep the load unconditionally in bounds so LLVM can always execute it, + // and select zero afterwards. This avoids a branch that could be mispredicted. + let index = indices[lane] as usize; + let value = bytes[index % #byte_count]; + output[lane] = if index < #byte_count { value } else { 0 }; + } + let result: #bytes_rust = output.simd_into(self); + Bytes::from_bytes(result) + } + } + } OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index 5c418455..6bab6ea2 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -464,6 +464,43 @@ impl Level for Neon { } }) } + OpSig::SwizzleDynPrecise => { + let bytes_ty = vec_ty.bytes_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + + self.kernel_method(op, vec_ty, |token| { + let body = match vec_ty.n_bits() { + 128 => quote! { + let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into()); + }, + 256 => quote! { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x2_t = indices.into(); + let result = uint8x16x2_t( + vqtbl2q_u8(table, indices.0), + vqtbl2q_u8(table, indices.1), + ); + }, + 512 => quote! { + let table = Bytes::to_bytes(a).val.0; + let indices: uint8x16x4_t = indices.into(); + let result = uint8x16x4_t( + vqtbl4q_u8(table, indices.0), + vqtbl4q_u8(table, indices.1), + vqtbl4q_u8(table, indices.2), + vqtbl4q_u8(table, indices.3), + ); + }, + _ => unreachable!(), + }; + + quote! { + #body + Bytes::from_bytes(#bytes { val: #wrapper(result), simd: #token }) + } + }) + } OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_simd_types.rs b/fearless_simd_gen/src/mk_simd_types.rs index 46886d83..b313f790 100644 --- a/fearless_simd_gen/src/mk_simd_types.rs +++ b/fearless_simd_gen/src/mk_simd_types.rs @@ -440,6 +440,7 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { let shift_elements_left_op = generic_op_name("shift_elements_left", ty); let shift_elements_right_op = generic_op_name("shift_elements_right", ty); let swizzle_dyn_within_blocks_op = generic_op_name("swizzle_dyn_within_blocks", ty); + let swizzle_dyn_precise_op = generic_op_name("swizzle_dyn_precise", ty); quote! { impl SimdBase for #name { type Element = #scalar; @@ -523,6 +524,11 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd.#swizzle_dyn_within_blocks_op(self, indices.simd_into(self.simd)) } + + #[inline(always)] + fn swizzle_dyn_precise(self, indices: impl SimdInto) -> Self { + self.simd.#swizzle_dyn_precise_op(self, indices.simd_into(self.simd)) + } } impl crate::#vec_trait_id for #name { #( #methods )* diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index f1f58ee5..f6745c8d 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -8,7 +8,7 @@ use crate::arch::wasm::{arch_prefix, v128_intrinsic}; use crate::generic::{ fallback_method, generic_as_array, generic_block_combine, generic_block_split, generic_from_array, generic_mask_set, generic_op_name, generic_store_array, - integer_lane_mask_splat_arg, + integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, }; use crate::level::Level; use crate::ops::{Op, Quantifier, SlideGranularity}; @@ -509,6 +509,35 @@ impl Level for WasmSimd128 { } } } + OpSig::SwizzleDynPrecise => match vec_ty.n_bits() { + 128 => { + let bytes_ty = vec_ty.bytes_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + + quote! { + #method_sig { + let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into()); + Bytes::from_bytes(#bytes { val: #wrapper(result), simd: self }) + } + } + } + 256 => { + let body = recursive_swizzle_dyn_precise_body(vec_ty, "e! { self }); + + quote! { + #method_sig { + #body + Bytes::from_bytes(result_bytes) + } + } + } + // We don't use the recursive decomposition for the 512-bit case + // because register spills get way too bad. + // We can only hope that the compiler will recognize the shuffle operation + // and express it in terms of 256-bit or 512-bit vectors. + _ => crate::mk_fallback::Fallback.make_method(op, vec_ty), + }, OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index d3b6a81d..815e0d0c 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -9,7 +9,7 @@ use crate::arch::x86::{ use crate::generic::{ fallback_method, generic_as_array, generic_block_combine, generic_block_split, generic_from_array, generic_mask_from_bitmask, generic_mask_set, generic_op_name, - generic_store_array, integer_lane_mask_splat_arg, + generic_store_array, integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, }; use crate::level::Level; use crate::ops::{Op, OpSig, Quantifier, SlideGranularity}; @@ -313,6 +313,7 @@ impl Level for X86 { unreachable!("element moves use generic lowering") } OpSig::SwizzleDynWithinBlocks => self.handle_swizzle_dyn_within_blocks(op, vec_ty), + OpSig::SwizzleDynPrecise => self.handle_swizzle_dyn_precise(op, vec_ty), OpSig::Cvt { target_ty, scalar_bits, @@ -2830,6 +2831,74 @@ impl X86 { }) } + pub(crate) fn handle_swizzle_dyn_precise(&self, op: Op, vec_ty: &VecType) -> TokenStream { + let bytes_ty = vec_ty.bytes_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + + if *self == Self::Sse2 || (*self == Self::Sse4_2 && vec_ty.n_bits() == 512) { + return fallback_method(op, vec_ty); + } + + self.kernel_method(op, vec_ty, |token| { + let body = match (*self, vec_ty.n_bits()) { + (Self::Sse4_2 | Self::Avx2, 128) => quote! { + let indices = indices.into(); + // Preserve the original high bit, and set it for indices 16..=127. + // The added value only changes bits that PSHUFB ignores for valid indices. + let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112)); + let zeroing_indices = _mm_or_si128(indices, index_out_of_range); + let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices); + let result_bytes = #bytes { val: #wrapper(result), simd: #token }; + }, + (Self::Sse4_2, 256) | (Self::Avx2, 512) => { + recursive_swizzle_dyn_precise_body(vec_ty, token) + } + (Self::Avx2, 256) => quote! { + let bytes = Bytes::to_bytes(a); + let idxs = indices; + let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0); + let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0); + + // Adding 0x60 preserves the low nibble and bit 4 for valid + // indices 0..=31. Larger indices get their high bit set, so + // VPSHUFB supplies the required out-of-bounds zeroing. + let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60)); + + // Move index bit 4 into each byte's sign bit for VPBLENDVB. + let select_high = _mm256_slli_epi16::<3>(control); + let from_low = _mm256_shuffle_epi8(lolo, control); + let from_high = _mm256_shuffle_epi8(hihi, control); + let result = _mm256_blendv_epi8(from_low, from_high, select_high); + let result_bytes = #bytes { val: #wrapper(result), simd: #token }; + }, + (Self::Avx512, 128 | 256 | 512) => { + let min = intrinsic_ident("min", "epu8", vec_ty.n_bits()); + let permute = intrinsic_ident("permutex2var", "epi8", vec_ty.n_bits()); + let set1 = set1_intrinsic(&bytes_ty); + let setzero = + intrinsic_ident("setzero", coarse_type(&bytes_ty), vec_ty.n_bits()); + let byte_count = signed_literal(bytes_ty.len as u64, 8); + quote! { + let bytes = Bytes::to_bytes(a).val.0; + let indices = indices.into(); + // Clamp out-of-range indices to the first byte of a + // second, all-zero table. + let indices = #min(indices, #set1(#byte_count)); + let result = #permute(bytes, indices, #setzero()); + let result_bytes = #bytes { val: #wrapper(result), simd: #token }; + } + } + _ => unreachable!(), + }; + + quote! { + #body + Bytes::from_bytes(result_bytes) + } + }) + } + pub(crate) fn handle_cvt( &self, op: Op, diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index c04c87f1..af3e37d5 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -96,6 +96,9 @@ pub(crate) enum OpSig { /// Takes a vector and a same-width byte-index vector, and returns the original vector type with its bytes /// dynamically swizzled within each 128-bit block. SwizzleDynWithinBlocks, + /// Takes a vector and a same-width byte-index vector, and returns the original vector type with its bytes + /// dynamically swizzled across the whole vector. Out-of-range indices produce zero bytes. + SwizzleDynPrecise, /// Takes a single argument of the source vector type, and returns a vector type of the target scalar type and the /// same length. Cvt { @@ -311,7 +314,7 @@ impl Op { OpSig::ElementRotate { .. } => (vec![vec.clone()], vec), OpSig::ElementShift { .. } => (vec![vec.clone(), splat_arg_ty(vec_ty)], vec), OpSig::Slide { .. } => (vec![vec.clone(), vec.clone()], vec), - OpSig::SwizzleDynWithinBlocks => { + OpSig::SwizzleDynWithinBlocks | OpSig::SwizzleDynPrecise => { let bytes_ty = vec_ty.bytes_ty().rust(); (vec![vec.clone(), quote! { #bytes_ty<#simd_ty> }], vec) } @@ -426,7 +429,7 @@ impl Op { let arg1 = &arg_names[1]; quote! { (#arg0, #arg1: impl SimdInto) -> Self } } - OpSig::SwizzleDynWithinBlocks => { + OpSig::SwizzleDynWithinBlocks | OpSig::SwizzleDynPrecise => { let arg0 = &arg_names[0]; let arg1 = &arg_names[1]; quote! { (#arg0, #arg1: impl SimdInto) -> Self } @@ -627,6 +630,13 @@ const BASE_OPS: &[Op] = &[ The `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\n\ Out-of-range index behavior varies by platform.", ), + Op::new( + "swizzle_dyn_precise", + OpKind::BaseTraitMethod, + OpSig::SwizzleDynPrecise, + "Dynamically swizzle this vector's bytes across the whole vector.\n\n\ + The `indices` operand is a same-width byte vector. For each output byte, index values within the vector's byte length select the corresponding byte from the input vector. Out-of-range indices produce zero bytes.", + ), ]; const MASK_REPRESENTATION_OPS: &[Op] = &[ @@ -1560,6 +1570,7 @@ impl OpSig { | Self::AsArray { .. } | Self::StoreArray | Self::MaskSet + | Self::SwizzleDynPrecise | Self::Slide { granularity: SlideGranularity::AcrossBlocks, .. @@ -1597,7 +1608,7 @@ impl OpSig { | Self::MaskReduce { .. } | Self::MaskToBitmask | Self::AsArray { .. } => &["a"], - Self::SwizzleDynWithinBlocks => &["a", "indices"], + Self::SwizzleDynWithinBlocks | Self::SwizzleDynPrecise => &["a", "indices"], Self::Binary | Self::Compare | Self::Combine { .. } @@ -1629,7 +1640,7 @@ impl OpSig { | Self::WidenNarrow { .. } | Self::MaskReduce { .. } | Self::AsArray { .. } => &["self"], - Self::SwizzleDynWithinBlocks => &["self", "indices"], + Self::SwizzleDynWithinBlocks | Self::SwizzleDynPrecise => &["self", "indices"], Self::Binary | Self::Compare | Self::Zip { .. } @@ -1693,6 +1704,7 @@ impl OpSig { | Self::AsArray { .. } | Self::StoreArray | Self::SwizzleDynWithinBlocks + | Self::SwizzleDynPrecise | Self::Slide { .. } => return None, }; Some(args) diff --git a/fearless_simd_tests/tests/harness/ops/mod.rs b/fearless_simd_tests/tests/harness/ops/mod.rs index a7ecf411..46e7e432 100644 --- a/fearless_simd_tests/tests/harness/ops/mod.rs +++ b/fearless_simd_tests/tests/harness/ops/mod.rs @@ -75,6 +75,7 @@ mod store_array; mod store_interleaved_128; mod store_slice; mod sub; +mod swizzle_dyn_precise; mod swizzle_dyn_within_blocks; mod to_bitmask; mod to_bytes; diff --git a/fearless_simd_tests/tests/harness/ops/swizzle_dyn_precise.rs b/fearless_simd_tests/tests/harness/ops/swizzle_dyn_precise.rs new file mode 100644 index 00000000..df7b05b8 --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/swizzle_dyn_precise.rs @@ -0,0 +1,159 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +fn expected_swizzle_precise(bytes: [u8; N], indices: [u8; N]) -> [u8; N] { + core::array::from_fn(|i| { + let index = indices[i] as usize; + bytes.get(index).copied().unwrap_or(0) + }) +} + +#[simd_test] +fn swizzle_dyn_precise_u8x16(simd: S) { + let bytes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let indices = [15, 14, 0, 1, 16, 17, 31, 127, 128, 255, 8, 7, 6, 5, 4, 3]; + let expected = expected_swizzle_precise(bytes, indices); + + let value = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let result = value.swizzle_dyn_precise(index_vec); + + assert_eq!(*result, expected); +} + +#[simd_test] +fn swizzle_dyn_precise_u8x32_crosses_blocks(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| u8::try_from(i + 1).unwrap()); + let indices = [ + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 0, 1, 2, 3, 4, 5, 6, 7, 32, + 33, 127, 128, 255, 15, 16, 31, + ]; + let expected = expected_swizzle_precise(bytes, indices); + + let value = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let result = value.swizzle_dyn_precise(index_vec); + + assert_eq!(*result, expected); +} + +#[simd_test] +fn swizzle_dyn_precise_u8x32_all_indices(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| u8::try_from(i + 1).unwrap()); + + for start in (0..=224).step_by(32) { + let indices = core::array::from_fn(|i| u8::try_from(start + i).unwrap()); + let expected = expected_swizzle_precise(bytes, indices); + + let value = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let result = value.swizzle_dyn_precise(index_vec); + + assert_eq!(*result, expected); + } +} + +#[simd_test] +fn swizzle_dyn_precise_u8x64_crosses_blocks(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(|i| u8::try_from(i + 1).unwrap()); + let indices = [ + 63, 48, 32, 16, 0, 15, 31, 47, 62, 49, 33, 17, 1, 14, 30, 46, 45, 29, 13, 2, 18, 34, 50, + 61, 60, 44, 28, 12, 3, 19, 35, 51, 52, 36, 20, 4, 11, 27, 43, 59, 58, 42, 26, 10, 5, 21, + 37, 53, 64, 65, 127, 128, 255, 54, 38, 22, 6, 9, 25, 41, 57, 56, 40, 24, + ]; + let expected = expected_swizzle_precise(bytes, indices); + + let value = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let result = value.swizzle_dyn_precise(index_vec); + + assert_eq!(*result, expected); +} + +#[simd_test] +fn swizzle_dyn_precise_bitcast_f32x8(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| u8::try_from(i * 3 + 1).unwrap()); + let indices = [ + 16, 17, 18, 19, 0, 1, 2, 3, 31, 30, 29, 28, 32, 33, 128, 255, 4, 5, 6, 7, 20, 21, 22, 23, + 27, 26, 25, 24, 15, 14, 13, 12, + ]; + let expected = expected_swizzle_precise(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let value: f32x8 = byte_vec.bitcast(); + let index_vec = u8x32::simd_from(simd, indices); + let result_bytes: u8x32 = value.swizzle_dyn_precise(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_precise_generic_indices(simd: S) { + #[inline(always)] + fn do_swizzle>(value: V, indices: V::Bytes) -> V { + value.swizzle_dyn_precise(indices) + } + + let bytes: [u8; 32] = core::array::from_fn(|i| u8::try_from(i + 1).unwrap()); + let indices = [ + 16, 17, 18, 19, 20, 21, 22, 23, 31, 30, 29, 28, 27, 26, 25, 24, 0, 1, 2, 3, 4, 5, 6, 7, 32, + 64, 127, 128, 255, 8, 9, 10, + ]; + let expected = expected_swizzle_precise(bytes, indices); + + let value = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let result = do_swizzle::>(value, index_vec); + + assert_eq!(*result, expected); +} + +#[simd_test] +#[ignore = "this test is slow"] +// run with: cargo test --release swizzle_dyn_precise_random_u8_all_widths -- --ignored +fn swizzle_dyn_precise_random_u8_all_widths(simd: S) { + let mut rng = fastrand::Rng::with_seed(0x5eed_5eed_cafe_f00d); + + for iteration in 0..100_000 { + let mut bytes: [u8; 64] = [0; 64]; + let mut indices: [u8; 64] = [0; 64]; + rng.fill(&mut bytes); + rng.fill(&mut indices); + + let bytes16 = &bytes[..16]; + let indices16 = &indices[..16]; + let expected16 = expected_swizzle_precise_from_slice::<16>(bytes16, indices16); + let value16 = u8x16::from_slice(simd, bytes16); + let index_vec16 = u8x16::from_slice(simd, indices16); + let result16 = value16.swizzle_dyn_precise(index_vec16); + assert_eq!(*result16, expected16, "iteration {iteration}, width 16"); + + let bytes32 = &bytes[..32]; + let indices32 = &indices[..32]; + let expected32 = expected_swizzle_precise_from_slice::<32>(bytes32, indices32); + let value32 = u8x32::from_slice(simd, bytes32); + let index_vec32 = u8x32::from_slice(simd, indices32); + let result32 = value32.swizzle_dyn_precise(index_vec32); + assert_eq!(*result32, expected32, "iteration {iteration}, width 32"); + + let bytes64 = &bytes[..64]; + let indices64 = &indices[..64]; + let expected64 = expected_swizzle_precise_from_slice::<64>(bytes64, indices64); + let value64 = u8x64::from_slice(simd, bytes64); + let index_vec64 = u8x64::from_slice(simd, indices64); + let result64 = value64.swizzle_dyn_precise(index_vec64); + assert_eq!(*result64, expected64, "iteration {iteration}, width 64"); + } +} + +fn expected_swizzle_precise_from_slice(bytes: &[u8], indices: &[u8]) -> [u8; N] { + assert_eq!(bytes.len(), N); + assert_eq!(indices.len(), N); + core::array::from_fn(|i| { + let index = indices[i] as usize; + bytes.get(index).copied().unwrap_or(0) + }) +}