diff --git a/CHANGELOG.md b/CHANGELOG.md index b6c39b67..5f8b295b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ You can find its changes [documented below](#060-2026-07-10). ### Changed +- Breaking change: the `load_interleaved_128_*` and `store_interleaved_128_*` methods, which exchanged one 512-bit vector, have been replaced by `load_four_interleaved_*` and `store_four_interleaved_*`. The new methods exchange an array of four 128-bit vectors directly and are available for all non-mask scalar types. + - The `new_unchecked()` function on SIMD level tokens such as `Avx2` has been renamed to `assume_supported()` and is now safe to call from contexts that already contain the appropriate `#[target_feature]` annotations. Functions without such annotations can still call `assume_supported()` with an `unsafe` block. ([#293][] by [@Shnatsel][]) - On x86_64 targets with static SSE2 support, `Level::baseline()` now returns `Sse2` instead of `Fallback`. ([#270][] by [@Shnatsel][]) - The `fxsr` CPU feature is now required for all x86 SIMD levels. It is present in hardware on all SIMD-capable CPUs, but it is possible to disable it in some emulators combined with a custom Rust target specification. ([#270][] by [@Shnatsel][]) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index b0b269e8..266be45c 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -557,6 +557,89 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[f32; 16usize]) -> [f32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); + let v1: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); + let v2: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); + let v3: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + let _ = token; + let v0: __m128 = vectors[0].into(); + let v1: __m128 = vectors[1].into(); + let v2: __m128 = vectors[2].into(); + let v3: __m128 = vectors[3].into(); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -1110,6 +1193,95 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[i8; 64usize]) -> [i8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, vectors: [i8x16; 4usize], dest: &mut [i8; 64usize]) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1582,6 +1754,95 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[u8; 64usize]) -> [u8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, vectors: [u8x16; 4usize], dest: &mut [u8; 64usize]) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -2163,6 +2424,99 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[i16; 32usize]) -> [i16x8; 4usize] { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u16x8(self, val: u16) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2570,6 +2924,99 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[u16; 32usize]) -> [u16x8; 4usize] { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { crate::kernel!( #[inline(always)] @@ -3110,7 +3557,90 @@ impl Simd for Avx2 { _mm_sub_epi32(_mm_setzero_si128(), a.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a) + } + #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[i32; 16usize]) -> [i32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); } #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { @@ -3504,6 +4034,89 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[u32; 16usize]) -> [u32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -4115,6 +4728,71 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[f64; 8usize]) -> [f64x2; 4usize] { + let (chunks, []) = src.as_chunks::<4>() else { + unreachable!() + }; + let v0: __m256d = + crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[0]); + let v1: __m256d = + crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[1]); + let lo = _mm256_unpacklo_pd(v0, v1); + let hi = _mm256_unpackhi_pd(v0, v1); + let out0 = _mm256_permute2f128_pd::<0x20>(lo, hi); + let out1 = _mm256_permute2f128_pd::<0x31>(lo, hi); + let outputs: [__m128d; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, vectors: [f64x2; 4usize], dest: &mut [f64; 8usize]) -> () { + let _ = token; + let inputs: [__m128d; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let wide_inputs: [__m256d; 2] = crate::transmute::checked_transmute_copy(&inputs); + let v0 = wide_inputs[0]; + let v1 = wide_inputs[1]; + let lo = _mm256_permute2f128_pd::<0x20>(v0, v1); + let hi = _mm256_permute2f128_pd::<0x31>(v0, v1); + let out0 = _mm256_unpacklo_pd(lo, hi); + let out1 = _mm256_unpackhi_pd(lo, hi); + let (chunks, []) = dest.as_chunks_mut::<4>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m256d, [f64; 4]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m256d, [f64; 4]>( + out1, + &mut chunks[1], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] @@ -4471,6 +5149,71 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[i64; 8usize]) -> [i64x2; 4usize] { + let (chunks, []) = src.as_chunks::<4>() else { + unreachable!() + }; + let v0: __m256i = + crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[0]); + let v1: __m256i = + crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[1]); + let lo = _mm256_unpacklo_epi64(v0, v1); + let hi = _mm256_unpackhi_epi64(v0, v1); + let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); + let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, vectors: [i64x2; 4usize], dest: &mut [i64; 8usize]) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let wide_inputs: [__m256i; 2] = crate::transmute::checked_transmute_copy(&inputs); + let v0 = wide_inputs[0]; + let v1 = wide_inputs[1]; + let lo = _mm256_permute2x128_si256::<0x20>(v0, v1); + let hi = _mm256_permute2x128_si256::<0x31>(v0, v1); + let out0 = _mm256_unpacklo_epi64(lo, hi); + let out1 = _mm256_unpackhi_epi64(lo, hi); + let (chunks, []) = dest.as_chunks_mut::<4>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m256i, [i64; 4]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m256i, [i64; 4]>( + out1, + &mut chunks[1], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -4814,6 +5557,71 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[u64; 8usize]) -> [u64x2; 4usize] { + let (chunks, []) = src.as_chunks::<4>() else { + unreachable!() + }; + let v0: __m256i = + crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[0]); + let v1: __m256i = + crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[1]); + let lo = _mm256_unpacklo_epi64(v0, v1); + let hi = _mm256_unpackhi_epi64(v0, v1); + let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); + let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, vectors: [u64x2; 4usize], dest: &mut [u64; 8usize]) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let wide_inputs: [__m256i; 2] = crate::transmute::checked_transmute_copy(&inputs); + let v0 = wide_inputs[0]; + let v1 = wide_inputs[1]; + let lo = _mm256_permute2x128_si256::<0x20>(v0, v1); + let hi = _mm256_permute2x128_si256::<0x31>(v0, v1); + let out0 = _mm256_unpacklo_epi64(lo, hi); + let out1 = _mm256_unpackhi_epi64(lo, hi); + let (chunks, []) = dest.as_chunks_mut::<4>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>( + out1, + &mut chunks[1], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { crate::kernel!( #[inline(always)] @@ -11446,98 +12254,23 @@ impl Simd for Avx2 { } #[inline(always)] fn select_f32x16(self, a: mask32x16, b: f32x16, c: f32x16) -> f32x16 { - let (a0, a1) = self.split_mask32x16(a); - let (b0, b1) = self.split_f32x16(b); - let (c0, c1) = self.split_f32x16(c); - self.combine_f32x8(self.select_f32x8(a0, b0, c0), self.select_f32x8(a1, b1, c1)) - } - #[inline(always)] - fn split_f32x16(self, a: f32x16) -> (f32x8, f32x8) { - ( - f32x8 { - val: crate::support::Aligned256(a.val.0[0]), - simd: self, - }, - f32x8 { - val: crate::support::Aligned256(a.val.0[1]), - simd: self, - }, - ) - } - #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[f32; 16usize]) -> f32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); - let v1: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); - let v2: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); - let v3: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - token.combine_f32x8( - token.combine_f32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_f32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: f32x16, dest: &mut [f32; 16usize]) -> () { - let (v01, v23) = token.split_f32x16(a); - let (v0, v1) = token.split_f32x8(v01); - let (v2, v3) = token.split_f32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); + let (a0, a1) = self.split_mask32x16(a); + let (b0, b1) = self.split_f32x16(b); + let (c0, c1) = self.split_f32x16(c); + self.combine_f32x8(self.select_f32x8(a0, b0, c0), self.select_f32x8(a1, b1, c1)) + } + #[inline(always)] + fn split_f32x16(self, a: f32x16) -> (f32x8, f32x8) { + ( + f32x8 { + val: crate::support::Aligned256(a.val.0[0]), + simd: self, + }, + f32x8 { + val: crate::support::Aligned256(a.val.0[1]), + simd: self, + }, + ) } #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { @@ -12645,91 +13378,6 @@ impl Simd for Avx2 { ) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[u8; 64usize]) -> u8x64 { - let (chunks, []) = src.as_chunks::<16usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u8x32( - token.combine_u8x16(out0.simd_into(token), out1.simd_into(token)), - token.combine_u8x16(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u8x64, dest: &mut [u8; 64usize]) -> () { - let (v01, v23) = token.split_u8x64(a); - let (v0, v1) = token.split_u8x32(v01); - let (v2, v3) = token.split_u8x32(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let out0 = _mm_shuffle_epi8(out0, mask); - let out1 = _mm_shuffle_epi8(out1, mask); - let out2 = _mm_shuffle_epi8(out2, mask); - let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<16usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { let half = self.splat_mask8x32(val); self.combine_mask8x32(half, half) @@ -13714,91 +14362,6 @@ impl Simd for Avx2 { ) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[u16; 32usize]) -> u16x32 { - let (chunks, []) = src.as_chunks::<8usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u16x16( - token.combine_u16x8(out0.simd_into(token), out1.simd_into(token)), - token.combine_u16x8(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u16x32, dest: &mut [u16; 32usize]) -> () { - let (v01, v23) = token.split_u16x32(a); - let (v0, v1) = token.split_u16x16(v01); - let (v2, v3) = token.split_u16x16(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); - let out0 = _mm_shuffle_epi8(out0, mask); - let out1 = _mm_shuffle_epi8(out1, mask); - let out2 = _mm_shuffle_epi8(out2, mask); - let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<8usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { crate::kernel!( #[inline(always)] @@ -14642,81 +15205,6 @@ impl Simd for Avx2 { ) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[u32; 16usize]) -> u32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u32x8( - token.combine_u32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_u32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u32x16, dest: &mut [u32; 16usize]) -> () { - let (v01, v23) = token.split_u32x16(a); - let (v0, v1) = token.split_u32x8(v01); - let (v2, v3) = token.split_u32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { let (a0, a1) = self.split_u32x16(a); self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) @@ -15849,54 +16337,6 @@ impl Simd for Avx2 { ) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[u64; 8usize]) -> u64x8 { - let (chunks, []) = src.as_chunks::<4>() else { - unreachable!() - }; - let v0: __m256i = - crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[0]); - let v1: __m256i = - crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[1]); - let lo = _mm256_unpacklo_epi64(v0, v1); - let hi = _mm256_unpackhi_epi64(v0, v1); - let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); - let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); - token.combine_u64x4(out0.simd_into(token), out1.simd_into(token)) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u64x8, dest: &mut [u64; 8usize]) -> () { - let (v0, v1) = token.split_u64x8(a); - let v0: __m256i = v0.into(); - let v1: __m256i = v1.into(); - let lo = _mm256_permute2x128_si256::<0x20>(v0, v1); - let hi = _mm256_permute2x128_si256::<0x31>(v0, v1); - let out0 = _mm256_unpacklo_epi64(lo, hi); - let out1 = _mm256_unpackhi_epi64(lo, hi); - let (chunks, []) = dest.as_chunks_mut::<4>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>( - out1, - &mut chunks[1], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index ab36aa87..a391c70d 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -853,6 +853,58 @@ impl Simd for Avx512 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[f32; 16usize]) -> [f32x4; 4usize] { + let lanes: __m512 = + crate::transmute::checked_transmute_copy::<[f32; 16usize], __m512>(src); + let lanes = _mm512_permutexvar_ps( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ); + let outputs: [__m128; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + let _ = token; + let inputs: [__m128; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512 = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_ps( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512, [f32; 16usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -1422,6 +1474,68 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[i8; 64usize]) -> [i8x16; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[i8; 64usize], __m512i>(src); + let lanes = _mm512_permutexvar_epi8( + _mm512_set_epi8( + 63, 59, 55, 51, 47, 43, 39, 35, 31, 27, 23, 19, 15, 11, 7, 3, 62, 58, 54, + 50, 46, 42, 38, 34, 30, 26, 22, 18, 14, 10, 6, 2, 61, 57, 53, 49, 45, 41, + 37, 33, 29, 25, 21, 17, 13, 9, 5, 1, 60, 56, 52, 48, 44, 40, 36, 32, 28, + 24, 20, 16, 12, 8, 4, 0, + ), + lanes, + ); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_epi8( + _mm512_set_epi8( + 63, 47, 31, 15, 62, 46, 30, 14, 61, 45, 29, 13, 60, 44, 28, 12, 59, 43, 27, + 11, 58, 42, 26, 10, 57, 41, 25, 9, 56, 40, 24, 8, 55, 39, 23, 7, 54, 38, + 22, 6, 53, 37, 21, 5, 52, 36, 20, 4, 51, 35, 19, 3, 50, 34, 18, 2, 49, 33, + 17, 1, 48, 32, 16, 0, + ), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512i, [i8; 64usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1925,6 +2039,68 @@ impl Simd for Avx512 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[u8; 64usize]) -> [u8x16; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[u8; 64usize], __m512i>(src); + let lanes = _mm512_permutexvar_epi8( + _mm512_set_epi8( + 63, 59, 55, 51, 47, 43, 39, 35, 31, 27, 23, 19, 15, 11, 7, 3, 62, 58, 54, + 50, 46, 42, 38, 34, 30, 26, 22, 18, 14, 10, 6, 2, 61, 57, 53, 49, 45, 41, + 37, 33, 29, 25, 21, 17, 13, 9, 5, 1, 60, 56, 52, 48, 44, 40, 36, 32, 28, + 24, 20, 16, 12, 8, 4, 0, + ), + lanes, + ); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_epi8( + _mm512_set_epi8( + 63, 47, 31, 15, 62, 46, 30, 14, 61, 45, 29, 13, 60, 44, 28, 12, 59, 43, 27, + 11, 58, 42, 26, 10, 57, 41, 25, 9, 56, 40, 24, 8, 55, 39, 23, 7, 54, 38, + 22, 6, 53, 37, 21, 5, 52, 36, 20, 4, 51, 35, 19, 3, 50, 34, 18, 2, 49, 33, + 17, 1, 48, 32, 16, 0, + ), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512i, [u8; 64usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -2479,6 +2655,64 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[i16; 32usize]) -> [i16x8; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[i16; 32usize], __m512i>(src); + let lanes = _mm512_permutexvar_epi16( + _mm512_set_epi16( + 31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2, 29, 25, 21, 17, + 13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0, + ), + lanes, + ); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_epi16( + _mm512_set_epi16( + 31, 23, 15, 7, 30, 22, 14, 6, 29, 21, 13, 5, 28, 20, 12, 4, 27, 19, 11, 3, + 26, 18, 10, 2, 25, 17, 9, 1, 24, 16, 8, 0, + ), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512i, [i16; 32usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u16x8(self, val: u16) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2903,6 +3137,64 @@ impl Simd for Avx512 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[u16; 32usize]) -> [u16x8; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[u16; 32usize], __m512i>(src); + let lanes = _mm512_permutexvar_epi16( + _mm512_set_epi16( + 31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2, 29, 25, 21, 17, + 13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0, + ), + lanes, + ); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_epi16( + _mm512_set_epi16( + 31, 23, 15, 7, 30, 22, 14, 6, 29, 21, 13, 5, 28, 20, 12, 4, 27, 19, 11, 3, + 26, 18, 10, 2, 25, 17, 9, 1, 24, 16, 8, 0, + ), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512i, [u16; 32usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { mask16x8 { val: (if val { 255u64 } else { 0 }) as _, @@ -3421,6 +3713,58 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[i32; 16usize]) -> [i32x4; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[i32; 16usize], __m512i>(src); + let lanes = _mm512_permutexvar_epi32( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_epi32( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512i, [i32; 16usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -3829,6 +4173,58 @@ impl Simd for Avx512 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[u32; 16usize]) -> [u32x4; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[u32; 16usize], __m512i>(src); + let lanes = _mm512_permutexvar_epi32( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_epi32( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512i, [u32; 16usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -4409,6 +4805,52 @@ impl Simd for Avx512 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[f64; 8usize]) -> [f64x2; 4usize] { + let lanes: __m512d = + crate::transmute::checked_transmute_copy::<[f64; 8usize], __m512d>(src); + let lanes = _mm512_permutexvar_pd(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes); + let outputs: [__m128d; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + let _ = token; + let inputs: [__m128d; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512d = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_pd(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), lanes); + crate::transmute::checked_transmute_store::<__m512d, [f64; 8usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] @@ -4807,6 +5249,54 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[i64; 8usize]) -> [i64x2; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[i64; 8usize], __m512i>(src); + let lanes = + _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = + _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), lanes); + crate::transmute::checked_transmute_store::<__m512i, [i64; 8usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5195,6 +5685,54 @@ impl Simd for Avx512 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[u64; 8usize]) -> [u64x2; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[u64; 8usize], __m512i>(src); + let lanes = + _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = + _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), lanes); + crate::transmute::checked_transmute_store::<__m512i, [u64; 8usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { mask64x2 { val: (if val { 3u64 } else { 0 }) as _, @@ -12057,36 +12595,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[f32; 16usize]) -> f32x16 { - let lanes: __m512 = - crate::transmute::checked_transmute_copy::<[f32; 16usize], __m512>(src); - _mm512_permutexvar_ps( - _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), - lanes, - ) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: f32x16, dest: &mut [f32; 16usize]) -> () { - let lanes = _mm512_permutexvar_ps( - _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), - a.into(), - ); - crate::transmute::checked_transmute_store::<__m512, [f32; 16usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { crate::kernel!( #[inline(always)] @@ -13774,46 +14282,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[u8; 64usize]) -> u8x64 { - let lanes: __m512i = - crate::transmute::checked_transmute_copy::<[u8; 64usize], __m512i>(src); - _mm512_permutexvar_epi8( - _mm512_set_epi8( - 63, 59, 55, 51, 47, 43, 39, 35, 31, 27, 23, 19, 15, 11, 7, 3, 62, 58, 54, - 50, 46, 42, 38, 34, 30, 26, 22, 18, 14, 10, 6, 2, 61, 57, 53, 49, 45, 41, - 37, 33, 29, 25, 21, 17, 13, 9, 5, 1, 60, 56, 52, 48, 44, 40, 36, 32, 28, - 24, 20, 16, 12, 8, 4, 0, - ), - lanes, - ) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: u8x64, dest: &mut [u8; 64usize]) -> () { - let lanes = _mm512_permutexvar_epi8( - _mm512_set_epi8( - 63, 47, 31, 15, 62, 46, 30, 14, 61, 45, 29, 13, 60, 44, 28, 12, 59, 43, 27, - 11, 58, 42, 26, 10, 57, 41, 25, 9, 56, 40, 24, 8, 55, 39, 23, 7, 54, 38, - 22, 6, 53, 37, 21, 5, 52, 36, 20, 4, 51, 35, 19, 3, 50, 34, 18, 2, 49, 33, - 17, 1, 48, 32, 16, 0, - ), - a.into(), - ); - crate::transmute::checked_transmute_store::<__m512i, [u8; 64usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { mask8x64 { val: if val { u64::MAX } else { 0 }, @@ -15188,42 +15656,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[u16; 32usize]) -> u16x32 { - let lanes: __m512i = - crate::transmute::checked_transmute_copy::<[u16; 32usize], __m512i>(src); - _mm512_permutexvar_epi16( - _mm512_set_epi16( - 31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2, 29, 25, 21, 17, - 13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0, - ), - lanes, - ) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: u16x32, dest: &mut [u16; 32usize]) -> () { - let lanes = _mm512_permutexvar_epi16( - _mm512_set_epi16( - 31, 23, 15, 7, 30, 22, 14, 6, 29, 21, 13, 5, 28, 20, 12, 4, 27, 19, 11, 3, - 26, 18, 10, 2, 25, 17, 9, 1, 24, 16, 8, 0, - ), - a.into(), - ); - crate::transmute::checked_transmute_store::<__m512i, [u16; 32usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { crate::kernel!( #[inline(always)] @@ -16454,36 +16886,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[u32; 16usize]) -> u32x16 { - let lanes: __m512i = - crate::transmute::checked_transmute_copy::<[u32; 16usize], __m512i>(src); - _mm512_permutexvar_epi32( - _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), - lanes, - ) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: u32x16, dest: &mut [u32; 16usize]) -> () { - let lanes = _mm512_permutexvar_epi32( - _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), - a.into(), - ); - crate::transmute::checked_transmute_store::<__m512i, [u32; 16usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { crate::kernel!( #[inline(always)] @@ -18161,31 +18563,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[u64; 8usize]) -> u64x8 { - let lanes: __m512i = - crate::transmute::checked_transmute_copy::<[u64; 8usize], __m512i>(src); - _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: u64x8, dest: &mut [u64; 8usize]) -> () { - let lanes = - _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), a.into()); - crate::transmute::checked_transmute_store::<__m512i, [u64; 8usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { mask64x8 { val: (if val { 255u64 } else { 0 }) as _, diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index a77d897d..a8fcaba6 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -609,6 +609,40 @@ impl Simd for Fallback { result.simd_into(self) } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + [ + [src[0usize], src[4usize], src[8usize], src[12usize]].simd_into(self), + [src[1usize], src[5usize], src[9usize], src[13usize]].simd_into(self), + [src[2usize], src[6usize], src[10usize], src[14usize]].simd_into(self), + [src[3usize], src[7usize], src[11usize], src[15usize]].simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + ]; + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { [ a[0usize] as u32, @@ -1406,6 +1440,160 @@ impl Simd for Fallback { .simd_into(self) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + src[32usize], + src[36usize], + src[40usize], + src[44usize], + src[48usize], + src[52usize], + src[56usize], + src[60usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + src[33usize], + src[37usize], + src[41usize], + src[45usize], + src[49usize], + src[53usize], + src[57usize], + src[61usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + src[34usize], + src[38usize], + src[42usize], + src[46usize], + src[50usize], + src[54usize], + src[58usize], + src[62usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + src[35usize], + src[39usize], + src[43usize], + src[47usize], + src[51usize], + src[55usize], + src[59usize], + src[63usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + vectors[0usize][8usize], + vectors[1usize][8usize], + vectors[2usize][8usize], + vectors[3usize][8usize], + vectors[0usize][9usize], + vectors[1usize][9usize], + vectors[2usize][9usize], + vectors[3usize][9usize], + vectors[0usize][10usize], + vectors[1usize][10usize], + vectors[2usize][10usize], + vectors[3usize][10usize], + vectors[0usize][11usize], + vectors[1usize][11usize], + vectors[2usize][11usize], + vectors[3usize][11usize], + vectors[0usize][12usize], + vectors[1usize][12usize], + vectors[2usize][12usize], + vectors[3usize][12usize], + vectors[0usize][13usize], + vectors[1usize][13usize], + vectors[2usize][13usize], + vectors[3usize][13usize], + vectors[0usize][14usize], + vectors[1usize][14usize], + vectors[2usize][14usize], + vectors[3usize][14usize], + vectors[0usize][15usize], + vectors[1usize][15usize], + vectors[2usize][15usize], + vectors[3usize][15usize], + ]; + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { [val; 16usize].simd_into(self) } @@ -2141,6 +2329,160 @@ impl Simd for Fallback { result.simd_into(self) } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + src[32usize], + src[36usize], + src[40usize], + src[44usize], + src[48usize], + src[52usize], + src[56usize], + src[60usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + src[33usize], + src[37usize], + src[41usize], + src[45usize], + src[49usize], + src[53usize], + src[57usize], + src[61usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + src[34usize], + src[38usize], + src[42usize], + src[46usize], + src[50usize], + src[54usize], + src[58usize], + src[62usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + src[35usize], + src[39usize], + src[43usize], + src[47usize], + src[51usize], + src[55usize], + src[59usize], + src[63usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + vectors[0usize][8usize], + vectors[1usize][8usize], + vectors[2usize][8usize], + vectors[3usize][8usize], + vectors[0usize][9usize], + vectors[1usize][9usize], + vectors[2usize][9usize], + vectors[3usize][9usize], + vectors[0usize][10usize], + vectors[1usize][10usize], + vectors[2usize][10usize], + vectors[3usize][10usize], + vectors[0usize][11usize], + vectors[1usize][11usize], + vectors[2usize][11usize], + vectors[3usize][11usize], + vectors[0usize][12usize], + vectors[1usize][12usize], + vectors[2usize][12usize], + vectors[3usize][12usize], + vectors[0usize][13usize], + vectors[1usize][13usize], + vectors[2usize][13usize], + vectors[3usize][13usize], + vectors[0usize][14usize], + vectors[1usize][14usize], + vectors[2usize][14usize], + vectors[3usize][14usize], + vectors[0usize][15usize], + vectors[1usize][15usize], + vectors[2usize][15usize], + vectors[3usize][15usize], + ]; + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { [ a[0usize] as u16, @@ -3024,6 +3366,96 @@ impl Simd for Fallback { .simd_into(self) } #[inline(always)] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + ]; + } + #[inline(always)] fn splat_u16x8(self, val: u16) -> u16x8 { [val; 8usize].simd_into(self) } @@ -3536,6 +3968,96 @@ impl Simd for Fallback { result.simd_into(self) } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + ]; + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { let val: i16 = if val { !0 } else { 0 }; [val; 8usize].simd_into(self) @@ -4153,6 +4675,40 @@ impl Simd for Fallback { .simd_into(self) } #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + [ + [src[0usize], src[4usize], src[8usize], src[12usize]].simd_into(self), + [src[1usize], src[5usize], src[9usize], src[13usize]].simd_into(self), + [src[2usize], src[6usize], src[10usize], src[14usize]].simd_into(self), + [src[3usize], src[7usize], src[11usize], src[15usize]].simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + ]; + } + #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { [ a[0usize] as f32, @@ -4555,6 +5111,40 @@ impl Simd for Fallback { result.simd_into(self) } #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + [ + [src[0usize], src[4usize], src[8usize], src[12usize]].simd_into(self), + [src[1usize], src[5usize], src[9usize], src[13usize]].simd_into(self), + [src[2usize], src[6usize], src[10usize], src[14usize]].simd_into(self), + [src[3usize], src[7usize], src[11usize], src[15usize]].simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + ]; + } + #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { [ a[0usize] as f32, @@ -5062,6 +5652,32 @@ impl Simd for Fallback { result.simd_into(self) } #[inline(always)] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { + [ + [src[0usize], src[4usize]].simd_into(self), + [src[1usize], src[5usize]].simd_into(self), + [src[2usize], src[6usize]].simd_into(self), + [src[3usize], src[7usize]].simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + ]; + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { [val; 2usize].simd_into(self) } @@ -5400,6 +6016,32 @@ impl Simd for Fallback { [i64::wrapping_neg(a[0usize]), i64::wrapping_neg(a[1usize])].simd_into(self) } #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + [ + [src[0usize], src[4usize]].simd_into(self), + [src[1usize], src[5usize]].simd_into(self), + [src[2usize], src[6usize]].simd_into(self), + [src[3usize], src[7usize]].simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + ]; + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { [val; 2usize].simd_into(self) } @@ -5734,6 +6376,32 @@ impl Simd for Fallback { result.simd_into(self) } #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + [ + [src[0usize], src[4usize]].simd_into(self), + [src[1usize], src[5usize]].simd_into(self), + [src[2usize], src[6usize]].simd_into(self), + [src[3usize], src[7usize]].simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + ]; + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { let val: i64 = if val { !0 } else { 0 }; [val; 2usize].simd_into(self) @@ -9980,36 +10648,6 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - *dest = [ - a[0usize], a[4usize], a[8usize], a[12usize], a[1usize], a[5usize], a[9usize], - a[13usize], a[2usize], a[6usize], a[10usize], a[14usize], a[3usize], a[7usize], - a[11usize], a[15usize], - ]; - } - #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) @@ -11057,91 +11695,6 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[16usize], - src[20usize], - src[24usize], - src[28usize], - src[32usize], - src[36usize], - src[40usize], - src[44usize], - src[48usize], - src[52usize], - src[56usize], - src[60usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[17usize], - src[21usize], - src[25usize], - src[29usize], - src[33usize], - src[37usize], - src[41usize], - src[45usize], - src[49usize], - src[53usize], - src[57usize], - src[61usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[18usize], - src[22usize], - src[26usize], - src[30usize], - src[34usize], - src[38usize], - src[42usize], - src[46usize], - src[50usize], - src[54usize], - src[58usize], - src[62usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - src[19usize], - src[23usize], - src[27usize], - src[31usize], - src[35usize], - src[39usize], - src[43usize], - src[47usize], - src[51usize], - src[55usize], - src[59usize], - src[63usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - *dest = [ - a[0usize], a[16usize], a[32usize], a[48usize], a[1usize], a[17usize], a[33usize], - a[49usize], a[2usize], a[18usize], a[34usize], a[50usize], a[3usize], a[19usize], - a[35usize], a[51usize], a[4usize], a[20usize], a[36usize], a[52usize], a[5usize], - a[21usize], a[37usize], a[53usize], a[6usize], a[22usize], a[38usize], a[54usize], - a[7usize], a[23usize], a[39usize], a[55usize], a[8usize], a[24usize], a[40usize], - a[56usize], a[9usize], a[25usize], a[41usize], a[57usize], a[10usize], a[26usize], - a[42usize], a[58usize], a[11usize], a[27usize], a[43usize], a[59usize], a[12usize], - a[28usize], a[44usize], a[60usize], a[13usize], a[29usize], a[45usize], a[61usize], - a[14usize], a[30usize], a[46usize], a[62usize], a[15usize], a[31usize], a[47usize], - a[63usize], - ]; - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { let half = self.splat_mask8x32(val); self.combine_mask8x32(half, half) @@ -12028,54 +12581,6 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[16usize], - src[20usize], - src[24usize], - src[28usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[17usize], - src[21usize], - src[25usize], - src[29usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[18usize], - src[22usize], - src[26usize], - src[30usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - src[19usize], - src[23usize], - src[27usize], - src[31usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - *dest = [ - a[0usize], a[8usize], a[16usize], a[24usize], a[1usize], a[9usize], a[17usize], - a[25usize], a[2usize], a[10usize], a[18usize], a[26usize], a[3usize], a[11usize], - a[19usize], a[27usize], a[4usize], a[12usize], a[20usize], a[28usize], a[5usize], - a[13usize], a[21usize], a[29usize], a[6usize], a[14usize], a[22usize], a[30usize], - a[7usize], a[15usize], a[23usize], a[31usize], - ]; - } - #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { let (a0, a1) = self.split_u16x32(a); self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) @@ -12835,36 +13340,6 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - *dest = [ - a[0usize], a[4usize], a[8usize], a[12usize], a[1usize], a[5usize], a[9usize], - a[13usize], a[2usize], a[6usize], a[10usize], a[14usize], a[3usize], a[7usize], - a[11usize], a[15usize], - ]; - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { let (a0, a1) = self.split_u32x16(a); self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) @@ -13884,26 +14359,6 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - [ - src[0usize], - src[4usize], - src[1usize], - src[5usize], - src[2usize], - src[6usize], - src[3usize], - src[7usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - *dest = [ - a[0usize], a[2usize], a[4usize], a[6usize], a[1usize], a[3usize], a[5usize], a[7usize], - ]; - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index 3ccdd88b..54207fdc 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -550,6 +550,30 @@ impl Simd for Neon { } } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + let native = unsafe { vld4q_f32(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + let v0: float32x4_t = vectors[0usize].into(); + let v1: float32x4_t = vectors[1usize].into(); + let v2: float32x4_t = vectors[2usize].into(); + let v3: float32x4_t = vectors[3usize].into(); + unsafe { + vst4q_f32(dest.as_mut_ptr(), float32x4x4_t(v0, v1, v2, v3)); + } + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -1008,6 +1032,30 @@ impl Simd for Neon { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + let native = unsafe { vld4q_s8(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + let v0: int8x16_t = vectors[0usize].into(); + let v1: int8x16_t = vectors[1usize].into(); + let v2: int8x16_t = vectors[2usize].into(); + let v3: int8x16_t = vectors[3usize].into(); + unsafe { + vst4q_s8(dest.as_mut_ptr(), int8x16x4_t(v0, v1, v2, v3)); + } + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1428,6 +1476,30 @@ impl Simd for Neon { } } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + let native = unsafe { vld4q_u8(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + let v0: uint8x16_t = vectors[0usize].into(); + let v1: uint8x16_t = vectors[1usize].into(); + let v2: uint8x16_t = vectors[2usize].into(); + let v3: uint8x16_t = vectors[3usize].into(); + unsafe { + vst4q_u8(dest.as_mut_ptr(), uint8x16x4_t(v0, v1, v2, v3)); + } + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -2014,6 +2086,30 @@ impl Simd for Neon { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { + let native = unsafe { vld4q_s16(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + let v0: int16x8_t = vectors[0usize].into(); + let v1: int16x8_t = vectors[1usize].into(); + let v2: int16x8_t = vectors[2usize].into(); + let v3: int16x8_t = vectors[3usize].into(); + unsafe { + vst4q_s16(dest.as_mut_ptr(), int16x8x4_t(v0, v1, v2, v3)); + } + } + #[inline(always)] fn splat_u16x8(self, val: u16) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2402,6 +2498,30 @@ impl Simd for Neon { } } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + let native = unsafe { vld4q_u16(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + let v0: uint16x8_t = vectors[0usize].into(); + let v1: uint16x8_t = vectors[1usize].into(); + let v2: uint16x8_t = vectors[2usize].into(); + let v3: uint16x8_t = vectors[3usize].into(); + unsafe { + vst4q_u16(dest.as_mut_ptr(), uint16x8x4_t(v0, v1, v2, v3)); + } + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { crate::kernel!( #[inline(always)] @@ -2952,6 +3072,30 @@ impl Simd for Neon { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + let native = unsafe { vld4q_s32(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + let v0: int32x4_t = vectors[0usize].into(); + let v1: int32x4_t = vectors[1usize].into(); + let v2: int32x4_t = vectors[2usize].into(); + let v3: int32x4_t = vectors[3usize].into(); + unsafe { + vst4q_s32(dest.as_mut_ptr(), int32x4x4_t(v0, v1, v2, v3)); + } + } + #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -3334,6 +3478,30 @@ impl Simd for Neon { } } #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + let native = unsafe { vld4q_u32(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + let v0: uint32x4_t = vectors[0usize].into(); + let v1: uint32x4_t = vectors[1usize].into(); + let v2: uint32x4_t = vectors[2usize].into(); + let v3: uint32x4_t = vectors[3usize].into(); + unsafe { + vst4q_u32(dest.as_mut_ptr(), uint32x4x4_t(v0, v1, v2, v3)); + } + } + #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -3948,6 +4116,30 @@ impl Simd for Neon { } } #[inline(always)] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { + let native = unsafe { vld4q_f64(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + let v0: float64x2_t = vectors[0usize].into(); + let v1: float64x2_t = vectors[1usize].into(); + let v2: float64x2_t = vectors[2usize].into(); + let v3: float64x2_t = vectors[3usize].into(); + unsafe { + vst4q_f64(dest.as_mut_ptr(), float64x2x4_t(v0, v1, v2, v3)); + } + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] @@ -4316,6 +4508,30 @@ impl Simd for Neon { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + let native = unsafe { vld4q_s64(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + let v0: int64x2_t = vectors[0usize].into(); + let v1: int64x2_t = vectors[1usize].into(); + let v2: int64x2_t = vectors[2usize].into(); + let v3: int64x2_t = vectors[3usize].into(); + unsafe { + vst4q_s64(dest.as_mut_ptr(), int64x2x4_t(v0, v1, v2, v3)); + } + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -4674,6 +4890,30 @@ impl Simd for Neon { } } #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + let native = unsafe { vld4q_u64(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + let v0: uint64x2_t = vectors[0usize].into(); + let v1: uint64x2_t = vectors[1usize].into(); + let v2: uint64x2_t = vectors[2usize].into(); + let v3: uint64x2_t = vectors[3usize].into(); + unsafe { + vst4q_u64(dest.as_mut_ptr(), uint64x2x4_t(v0, v1, v2, v3)); + } + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { crate::kernel!( #[inline(always)] @@ -9470,14 +9710,6 @@ impl Simd for Neon { ) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - unsafe { vld4q_f32(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - unsafe { vst4q_f32(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) @@ -10649,14 +10881,6 @@ impl Simd for Neon { ) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - unsafe { vld4q_u8(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - unsafe { vst4q_u8(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { let half = self.splat_mask8x32(val); self.combine_mask8x32(half, half) @@ -11672,14 +11896,6 @@ impl Simd for Neon { ) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - unsafe { vld4q_u16(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - unsafe { vst4q_u16(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { let (a0, a1) = self.split_u16x32(a); self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) @@ -12568,14 +12784,6 @@ impl Simd for Neon { ) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - unsafe { vld4q_u32(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - unsafe { vst4q_u32(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { let (a0, a1) = self.split_u32x16(a); self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) @@ -13786,14 +13994,6 @@ impl Simd for Neon { ) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - unsafe { vld4q_u64(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - unsafe { vst4q_u64(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index db05d7ef..3be38a63 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -257,6 +257,16 @@ pub trait Simd: fn select_f32x4(self, a: mask32x4, b: f32x4, c: f32x4) -> f32x4; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_f32x4(self, a: f32x4, b: f32x4) -> f32x8; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> (); #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms below AVX-512, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32`.\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4; #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] @@ -349,6 +359,16 @@ pub trait Simd: fn combine_i8x16(self, a: i8x16, b: i8x16) -> i8x32; #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i8x16(self, a: i8x16) -> i8x16; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u8x16(self, val: u8) -> u8x16; #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] @@ -431,6 +451,16 @@ pub trait Simd: fn max_u8x16(self, a: u8x16, b: u8x16) -> u8x16; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> (); #[doc = "Zero-extend each element to a wider integer type.\n\nThe number of elements in the result is half that of the input."] fn widen_u8x16(self, a: u8x16) -> u16x16; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] @@ -552,6 +582,16 @@ pub trait Simd: fn combine_i16x8(self, a: i16x8, b: i16x8) -> i16x16; #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i16x8(self, a: i16x8) -> i16x8; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u16x8(self, val: u16) -> u16x8; #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] @@ -634,6 +674,16 @@ pub trait Simd: fn max_u16x8(self, a: u16x8, b: u16x8) -> u16x8; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_u16x8(self, a: u16x8, b: u16x8) -> u16x16; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> (); #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask16x8(self, val: bool) -> mask16x8; #[doc = "Create a SIMD mask from a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are ignored."] @@ -753,6 +803,16 @@ pub trait Simd: fn combine_i32x4(self, a: i32x4, b: i32x4) -> i32x8; #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i32x4(self, a: i32x4) -> i32x4; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> (); #[doc = "Convert each signed 32-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4; #[doc = "Create a SIMD vector with all elements set to the given value."] @@ -837,6 +897,16 @@ pub trait Simd: fn max_u32x4(self, a: u32x4, b: u32x4) -> u32x4; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_u32x4(self, a: u32x4, b: u32x4) -> u32x8; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> (); #[doc = "Convert each unsigned 32-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] @@ -970,6 +1040,16 @@ pub trait Simd: fn select_f64x2(self, a: mask64x2, b: f64x2, c: f64x2) -> f64x2; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_f64x2(self, a: f64x2, b: f64x2) -> f64x4; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_i64x2(self, val: i64) -> i64x2; #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] @@ -1054,6 +1134,16 @@ pub trait Simd: fn combine_i64x2(self, a: i64x2, b: i64x2) -> i64x4; #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i64x2(self, a: i64x2) -> i64x2; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u64x2(self, val: u64) -> u64x2; #[doc = "Concatenate `[self, rhs]` and extract `Self::N` elements starting at index `SHIFT`.\n\n`SHIFT` must be within [0, `Self::N`].\n\nThis can be used to implement a \"shift items\" operation by providing all zeroes as one operand. For a left shift, the right-hand side should be all zeroes. For a right shift by `M` items, the left-hand side should be all zeroes, and the shift amount will be `Self::N - M`.\n\nThis can also be used to rotate items within a vector by providing the same vector as both operands.\n\n```text\n\nslide::<1>([a b c d], [e f g h]) == [b c d e]\n\n```"] @@ -1136,6 +1226,16 @@ pub trait Simd: fn max_u64x2(self, a: u64x2, b: u64x2) -> u64x2; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_u64x2(self, a: u64x2, b: u64x2) -> u64x4; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> (); #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask64x2(self, val: bool) -> mask64x2; #[doc = "Create a SIMD mask from a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are ignored."] @@ -2319,10 +2419,6 @@ pub trait Simd: fn select_f32x16(self, a: mask32x16, b: f32x16, c: f32x16) -> f32x16; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_f32x16(self, a: f32x16) -> (f32x8, f32x8); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> (); #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms below AVX-512, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32`.\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16; #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] @@ -2497,10 +2593,6 @@ pub trait Simd: fn max_u8x64(self, a: u8x64, b: u8x64) -> u8x64; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> (); #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask8x64(self, val: bool) -> mask8x64; #[doc = "Create a SIMD mask from a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are ignored."] @@ -2710,10 +2802,6 @@ pub trait Simd: fn max_u16x32(self, a: u16x32, b: u16x32) -> u16x32; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> (); #[doc = "Truncate each element to a narrower integer type.\n\nThe number of elements in the result is twice that of the input."] fn narrow_u16x32(self, a: u16x32) -> u8x32; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] @@ -2927,10 +3015,6 @@ pub trait Simd: fn max_u32x16(self, a: u32x16, b: u32x16) -> u32x16; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> (); #[doc = "Convert each unsigned 32-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] @@ -3230,10 +3314,6 @@ pub trait Simd: fn max_u64x8(self, a: u64x8, b: u64x8) -> u64x8; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> (); #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask64x8(self, val: bool) -> mask64x8; #[doc = "Create a SIMD mask from a compact bitmask.\n\nBit `i` maps to lane `i`, with lane 0 in the least significant bit. Bits above the number of lanes in this mask are ignored."] diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index 6629149b..860b9741 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -671,6 +671,89 @@ impl Simd for Sse2 { } } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[f32; 16usize]) -> [f32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); + let v1: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); + let v2: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); + let v3: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse2, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + let _ = token; + let v0: __m128 = vectors[0].into(); + let v1: __m128 = vectors[1].into(); + let v2: __m128 = vectors[2].into(); + let v3: __m128 = vectors[3].into(); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { [ a[0usize] as u32, @@ -1252,6 +1335,160 @@ impl Simd for Sse2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + src[32usize], + src[36usize], + src[40usize], + src[44usize], + src[48usize], + src[52usize], + src[56usize], + src[60usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + src[33usize], + src[37usize], + src[41usize], + src[45usize], + src[49usize], + src[53usize], + src[57usize], + src[61usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + src[34usize], + src[38usize], + src[42usize], + src[46usize], + src[50usize], + src[54usize], + src[58usize], + src[62usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + src[35usize], + src[39usize], + src[43usize], + src[47usize], + src[51usize], + src[55usize], + src[59usize], + src[63usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + vectors[0usize][8usize], + vectors[1usize][8usize], + vectors[2usize][8usize], + vectors[3usize][8usize], + vectors[0usize][9usize], + vectors[1usize][9usize], + vectors[2usize][9usize], + vectors[3usize][9usize], + vectors[0usize][10usize], + vectors[1usize][10usize], + vectors[2usize][10usize], + vectors[3usize][10usize], + vectors[0usize][11usize], + vectors[1usize][11usize], + vectors[2usize][11usize], + vectors[3usize][11usize], + vectors[0usize][12usize], + vectors[1usize][12usize], + vectors[2usize][12usize], + vectors[3usize][12usize], + vectors[0usize][13usize], + vectors[1usize][13usize], + vectors[2usize][13usize], + vectors[3usize][13usize], + vectors[0usize][14usize], + vectors[1usize][14usize], + vectors[2usize][14usize], + vectors[3usize][14usize], + vectors[0usize][15usize], + vectors[1usize][15usize], + vectors[2usize][15usize], + vectors[3usize][15usize], + ]; + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1793,6 +2030,160 @@ impl Simd for Sse2 { } } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + src[32usize], + src[36usize], + src[40usize], + src[44usize], + src[48usize], + src[52usize], + src[56usize], + src[60usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + src[33usize], + src[37usize], + src[41usize], + src[45usize], + src[49usize], + src[53usize], + src[57usize], + src[61usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + src[34usize], + src[38usize], + src[42usize], + src[46usize], + src[50usize], + src[54usize], + src[58usize], + src[62usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + src[35usize], + src[39usize], + src[43usize], + src[47usize], + src[51usize], + src[55usize], + src[59usize], + src[63usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + vectors[0usize][8usize], + vectors[1usize][8usize], + vectors[2usize][8usize], + vectors[3usize][8usize], + vectors[0usize][9usize], + vectors[1usize][9usize], + vectors[2usize][9usize], + vectors[3usize][9usize], + vectors[0usize][10usize], + vectors[1usize][10usize], + vectors[2usize][10usize], + vectors[3usize][10usize], + vectors[0usize][11usize], + vectors[1usize][11usize], + vectors[2usize][11usize], + vectors[3usize][11usize], + vectors[0usize][12usize], + vectors[1usize][12usize], + vectors[2usize][12usize], + vectors[3usize][12usize], + vectors[0usize][13usize], + vectors[1usize][13usize], + vectors[2usize][13usize], + vectors[3usize][13usize], + vectors[0usize][14usize], + vectors[1usize][14usize], + vectors[2usize][14usize], + vectors[3usize][14usize], + vectors[0usize][15usize], + vectors[1usize][15usize], + vectors[2usize][15usize], + vectors[3usize][15usize], + ]; + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -2430,6 +2821,96 @@ impl Simd for Sse2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + ]; + } + #[inline(always)] fn splat_u16x8(self, val: u16) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2924,6 +3405,96 @@ impl Simd for Sse2 { } } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + ]; + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { crate::kernel!( #[inline(always)] @@ -3541,6 +4112,89 @@ impl Simd for Sse2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[i32; 16usize]) -> [i32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse2, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -4031,6 +4685,89 @@ impl Simd for Sse2 { } } #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[u32; 16usize]) -> [u32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse2, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { [ a[0usize] as f32, @@ -4663,6 +5400,77 @@ impl Simd for Sse2 { } } #[inline(always)] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[f64; 8usize]) -> [f64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[0]); + let v1: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[1]); + let v2: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[2]); + let v3: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[3]); + let out0 = _mm_unpacklo_pd(v0, v2); + let out1 = _mm_unpackhi_pd(v0, v2); + let out2 = _mm_unpacklo_pd(v1, v3); + let out3 = _mm_unpackhi_pd(v1, v3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, vectors: [f64x2; 4usize], dest: &mut [f64; 8usize]) -> () { + let _ = token; + let v0: __m128d = vectors[0].into(); + let v1: __m128d = vectors[1].into(); + let v2: __m128d = vectors[2].into(); + let v3: __m128d = vectors[3].into(); + let out0 = _mm_unpacklo_pd(v0, v1); + let out1 = _mm_unpacklo_pd(v2, v3); + let out2 = _mm_unpackhi_pd(v0, v1); + let out3 = _mm_unpackhi_pd(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] @@ -5065,7 +5873,78 @@ impl Simd for Sse2 { _mm_sub_epi64(_mm_setzero_si128(), a.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, a) + } + #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[i64; 8usize]) -> [i64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[3]); + let out0 = _mm_unpacklo_epi64(v0, v2); + let out1 = _mm_unpackhi_epi64(v0, v2); + let out2 = _mm_unpacklo_epi64(v1, v3); + let out3 = _mm_unpackhi_epi64(v1, v3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, vectors: [i64x2; 4usize], dest: &mut [i64; 8usize]) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let out0 = _mm_unpacklo_epi64(v0, v1); + let out1 = _mm_unpacklo_epi64(v2, v3); + let out2 = _mm_unpackhi_epi64(v0, v1); + let out3 = _mm_unpackhi_epi64(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); } #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { @@ -5465,6 +6344,77 @@ impl Simd for Sse2 { } } #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[u64; 8usize]) -> [u64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[3]); + let out0 = _mm_unpacklo_epi64(v0, v2); + let out1 = _mm_unpackhi_epi64(v0, v2); + let out2 = _mm_unpacklo_epi64(v1, v3); + let out3 = _mm_unpackhi_epi64(v1, v3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, vectors: [u64x2; 4usize], dest: &mut [u64; 8usize]) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let out0 = _mm_unpacklo_epi64(v0, v1); + let out1 = _mm_unpacklo_epi64(v2, v3); + let out2 = _mm_unpackhi_epi64(v0, v1); + let out3 = _mm_unpackhi_epi64(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { crate::kernel!( #[inline(always)] @@ -9927,81 +10877,6 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, src: &[f32; 16usize]) -> f32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); - let v1: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); - let v2: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); - let v3: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - token.combine_f32x8( - token.combine_f32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_f32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, a: f32x16, dest: &mut [f32; 16usize]) -> () { - let (v01, v23) = token.split_f32x16(a); - let (v0, v1) = token.split_f32x8(v01); - let (v2, v3) = token.split_f32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) @@ -11077,91 +11952,6 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[16usize], - src[20usize], - src[24usize], - src[28usize], - src[32usize], - src[36usize], - src[40usize], - src[44usize], - src[48usize], - src[52usize], - src[56usize], - src[60usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[17usize], - src[21usize], - src[25usize], - src[29usize], - src[33usize], - src[37usize], - src[41usize], - src[45usize], - src[49usize], - src[53usize], - src[57usize], - src[61usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[18usize], - src[22usize], - src[26usize], - src[30usize], - src[34usize], - src[38usize], - src[42usize], - src[46usize], - src[50usize], - src[54usize], - src[58usize], - src[62usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - src[19usize], - src[23usize], - src[27usize], - src[31usize], - src[35usize], - src[39usize], - src[43usize], - src[47usize], - src[51usize], - src[55usize], - src[59usize], - src[63usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - *dest = [ - a[0usize], a[16usize], a[32usize], a[48usize], a[1usize], a[17usize], a[33usize], - a[49usize], a[2usize], a[18usize], a[34usize], a[50usize], a[3usize], a[19usize], - a[35usize], a[51usize], a[4usize], a[20usize], a[36usize], a[52usize], a[5usize], - a[21usize], a[37usize], a[53usize], a[6usize], a[22usize], a[38usize], a[54usize], - a[7usize], a[23usize], a[39usize], a[55usize], a[8usize], a[24usize], a[40usize], - a[56usize], a[9usize], a[25usize], a[41usize], a[57usize], a[10usize], a[26usize], - a[42usize], a[58usize], a[11usize], a[27usize], a[43usize], a[59usize], a[12usize], - a[28usize], a[44usize], a[60usize], a[13usize], a[29usize], a[45usize], a[61usize], - a[14usize], a[30usize], a[46usize], a[62usize], a[15usize], a[31usize], a[47usize], - a[63usize], - ]; - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { let half = self.splat_mask8x32(val); self.combine_mask8x32(half, half) @@ -12081,54 +12871,6 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[16usize], - src[20usize], - src[24usize], - src[28usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[17usize], - src[21usize], - src[25usize], - src[29usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[18usize], - src[22usize], - src[26usize], - src[30usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - src[19usize], - src[23usize], - src[27usize], - src[31usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - *dest = [ - a[0usize], a[8usize], a[16usize], a[24usize], a[1usize], a[9usize], a[17usize], - a[25usize], a[2usize], a[10usize], a[18usize], a[26usize], a[3usize], a[11usize], - a[19usize], a[27usize], a[4usize], a[12usize], a[20usize], a[28usize], a[5usize], - a[13usize], a[21usize], a[29usize], a[6usize], a[14usize], a[22usize], a[30usize], - a[7usize], a[15usize], a[23usize], a[31usize], - ]; - } - #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { let (a0, a1) = self.split_u16x32(a); self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) @@ -12930,81 +13672,6 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, src: &[u32; 16usize]) -> u32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u32x8( - token.combine_u32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_u32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, a: u32x16, dest: &mut [u32; 16usize]) -> () { - let (v01, v23) = token.split_u32x16(a); - let (v0, v1) = token.split_u32x8(v01); - let (v2, v3) = token.split_u32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { let (a0, a1) = self.split_u32x16(a); self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) @@ -14071,73 +14738,6 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, src: &[u64; 8usize]) -> u64x8 { - let (chunks, []) = src.as_chunks::<2usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[3]); - let out0 = _mm_unpacklo_epi64(v0, v2); - let out1 = _mm_unpackhi_epi64(v0, v2); - let out2 = _mm_unpacklo_epi64(v1, v3); - let out3 = _mm_unpackhi_epi64(v1, v3); - token.combine_u64x4( - token.combine_u64x2(out0.simd_into(token), out1.simd_into(token)), - token.combine_u64x2(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, a: u64x8, dest: &mut [u64; 8usize]) -> () { - let (v01, v23) = token.split_u64x8(a); - let (v0, v1) = token.split_u64x4(v01); - let (v2, v3) = token.split_u64x4(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let out0 = _mm_unpacklo_epi64(v0, v1); - let out1 = _mm_unpacklo_epi64(v2, v3); - let out2 = _mm_unpackhi_epi64(v0, v1); - let out3 = _mm_unpackhi_epi64(v2, v3); - let (chunks, []) = dest.as_chunks_mut::<2usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index cb98cec6..5b0b6195 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -540,6 +540,89 @@ impl Simd for Sse4_2 { } } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[f32; 16usize]) -> [f32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); + let v1: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); + let v2: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); + let v3: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + let _ = token; + let v0: __m128 = vectors[0].into(); + let v1: __m128 = vectors[1].into(); + let v2: __m128 = vectors[2].into(); + let v3: __m128 = vectors[3].into(); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -1090,6 +1173,99 @@ impl Simd for Sse4_2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[i8; 64usize]) -> [i8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1559,6 +1735,99 @@ impl Simd for Sse4_2 { } } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[u8; 64usize]) -> [u8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -2137,6 +2406,99 @@ impl Simd for Sse4_2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[i16; 32usize]) -> [i16x8; 4usize] { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u16x8(self, val: u16) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2541,6 +2903,99 @@ impl Simd for Sse4_2 { } } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[u16; 32usize]) -> [u16x8; 4usize] { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { crate::kernel!( #[inline(always)] @@ -3078,6 +3533,89 @@ impl Simd for Sse4_2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[i32; 16usize]) -> [i32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -3446,24 +3984,107 @@ impl Simd for Sse4_2 { _mm_min_epu32(a.into(), b.into()).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a, b) + } + #[inline(always)] + fn max_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u32x4, b: u32x4) -> u32x4 { + _mm_max_epu32(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn combine_u32x4(self, a: u32x4, b: u32x4) -> u32x8 { + u32x8 { + val: crate::support::Aligned256([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[u32; 16usize]) -> [u32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) } #[inline(always)] - fn max_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { crate::kernel!( #[inline(always)] - fn kernel(token: Sse4_2, a: u32x4, b: u32x4) -> u32x4 { - _mm_max_epu32(a.into(), b.into()).simd_into(token) + fn kernel( + token: Sse4_2, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out3, + &mut chunks[3], + ); } ); - kernel(self, a, b) - } - #[inline(always)] - fn combine_u32x4(self, a: u32x4, b: u32x4) -> u32x8 { - u32x8 { - val: crate::support::Aligned256([a.val.0, b.val.0]), - simd: self, - } + kernel(self, vectors, dest); } #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { @@ -4059,6 +4680,81 @@ impl Simd for Sse4_2 { } } #[inline(always)] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[f64; 8usize]) -> [f64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[0]); + let v1: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[1]); + let v2: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[2]); + let v3: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[3]); + let out0 = _mm_unpacklo_pd(v0, v2); + let out1 = _mm_unpackhi_pd(v0, v2); + let out2 = _mm_unpacklo_pd(v1, v3); + let out3 = _mm_unpackhi_pd(v1, v3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + let _ = token; + let v0: __m128d = vectors[0].into(); + let v1: __m128d = vectors[1].into(); + let v2: __m128d = vectors[2].into(); + let v3: __m128d = vectors[3].into(); + let out0 = _mm_unpacklo_pd(v0, v1); + let out1 = _mm_unpacklo_pd(v2, v3); + let out2 = _mm_unpackhi_pd(v0, v1); + let out3 = _mm_unpackhi_pd(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] @@ -4403,6 +5099,81 @@ impl Simd for Sse4_2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[i64; 8usize]) -> [i64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[3]); + let out0 = _mm_unpacklo_epi64(v0, v2); + let out1 = _mm_unpackhi_epi64(v0, v2); + let out2 = _mm_unpacklo_epi64(v1, v3); + let out3 = _mm_unpackhi_epi64(v1, v3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let out0 = _mm_unpacklo_epi64(v0, v1); + let out1 = _mm_unpacklo_epi64(v2, v3); + let out2 = _mm_unpackhi_epi64(v0, v1); + let out3 = _mm_unpackhi_epi64(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -4739,6 +5510,81 @@ impl Simd for Sse4_2 { } } #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[u64; 8usize]) -> [u64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[3]); + let out0 = _mm_unpacklo_epi64(v0, v2); + let out1 = _mm_unpackhi_epi64(v0, v2); + let out2 = _mm_unpacklo_epi64(v1, v3); + let out3 = _mm_unpackhi_epi64(v1, v3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let out0 = _mm_unpacklo_epi64(v0, v1); + let out1 = _mm_unpacklo_epi64(v2, v3); + let out2 = _mm_unpackhi_epi64(v0, v1); + let out3 = _mm_unpackhi_epi64(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { crate::kernel!( #[inline(always)] @@ -9343,85 +10189,10 @@ impl Simd for Sse4_2 { simd: self, }, f32x8 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), - simd: self, - }, - ) - } - #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[f32; 16usize]) -> f32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); - let v1: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); - let v2: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); - let v3: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - token.combine_f32x8( - token.combine_f32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_f32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: f32x16, dest: &mut [f32; 16usize]) -> () { - let (v01, v23) = token.split_f32x16(a); - let (v0, v1) = token.split_f32x8(v01); - let (v2, v3) = token.split_f32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) } #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { @@ -10499,91 +11270,6 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[u8; 64usize]) -> u8x64 { - let (chunks, []) = src.as_chunks::<16usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u8x32( - token.combine_u8x16(out0.simd_into(token), out1.simd_into(token)), - token.combine_u8x16(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: u8x64, dest: &mut [u8; 64usize]) -> () { - let (v01, v23) = token.split_u8x64(a); - let (v0, v1) = token.split_u8x32(v01); - let (v2, v3) = token.split_u8x32(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let out0 = _mm_shuffle_epi8(out0, mask); - let out1 = _mm_shuffle_epi8(out1, mask); - let out2 = _mm_shuffle_epi8(out2, mask); - let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<16usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { let half = self.splat_mask8x32(val); self.combine_mask8x32(half, half) @@ -11544,91 +12230,6 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[u16; 32usize]) -> u16x32 { - let (chunks, []) = src.as_chunks::<8usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u16x16( - token.combine_u16x8(out0.simd_into(token), out1.simd_into(token)), - token.combine_u16x8(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: u16x32, dest: &mut [u16; 32usize]) -> () { - let (v01, v23) = token.split_u16x32(a); - let (v0, v1) = token.split_u16x16(v01); - let (v2, v3) = token.split_u16x16(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); - let out0 = _mm_shuffle_epi8(out0, mask); - let out1 = _mm_shuffle_epi8(out1, mask); - let out2 = _mm_shuffle_epi8(out2, mask); - let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<8usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { let (a0, a1) = self.split_u16x32(a); self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) @@ -12430,81 +13031,6 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[u32; 16usize]) -> u32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u32x8( - token.combine_u32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_u32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: u32x16, dest: &mut [u32; 16usize]) -> () { - let (v01, v23) = token.split_u32x16(a); - let (v0, v1) = token.split_u32x8(v01); - let (v2, v3) = token.split_u32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { let (a0, a1) = self.split_u32x16(a); self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) @@ -13571,73 +14097,6 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[u64; 8usize]) -> u64x8 { - let (chunks, []) = src.as_chunks::<2usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[3]); - let out0 = _mm_unpacklo_epi64(v0, v2); - let out1 = _mm_unpackhi_epi64(v0, v2); - let out2 = _mm_unpacklo_epi64(v1, v3); - let out3 = _mm_unpackhi_epi64(v1, v3); - token.combine_u64x4( - token.combine_u64x2(out0.simd_into(token), out1.simd_into(token)), - token.combine_u64x2(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: u64x8, dest: &mut [u64; 8usize]) -> () { - let (v01, v23) = token.split_u64x8(a); - let (v0, v1) = token.split_u64x4(v01); - let (v2, v3) = token.split_u64x4(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let out0 = _mm_unpacklo_epi64(v0, v1); - let out1 = _mm_unpacklo_epi64(v2, v3); - let out2 = _mm_unpackhi_epi64(v0, v1); - let out3 = _mm_unpackhi_epi64(v2, v3); - let (chunks, []) = dest.as_chunks_mut::<2usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index fe9e4176..037a4f16 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -372,6 +372,56 @@ impl Simd for WasmSimd128 { } } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[3]); + let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); + let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); + let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); + let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); + let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); + let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); + let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); + let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); + let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); + let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); + let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); + let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); + let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); + let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); + let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { #[cfg(target_feature = "relaxed-simd")] { @@ -711,6 +761,80 @@ impl Simd for WasmSimd128 { i8x16_neg(a.into()).simd_into(self) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[3]); + let v01_lower = + u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v0, v1); + let v23_lower = + u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v2, v3); + let v01_upper = + u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v0, v1); + let v23_upper = + u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v2, v3); + let out0 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( + v01_lower, v23_lower, + ); + let out1 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( + v01_lower, v23_lower, + ); + let out2 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( + v01_upper, v23_upper, + ); + let out3 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( + v01_upper, v23_upper, + ); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = + u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v0, v2); + let v13_lower = + u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v1, v3); + let v02_upper = + u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v0, v2); + let v13_upper = + u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v1, v3); + let out0 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( + v02_lower, v13_lower, + ); + let out1 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( + v02_lower, v13_lower, + ); + let out2 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( + v02_upper, v13_upper, + ); + let out3 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( + v02_upper, v13_upper, + ); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { u8x16_splat(val).simd_into(self) } @@ -1016,6 +1140,80 @@ impl Simd for WasmSimd128 { } } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[3]); + let v01_lower = + u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v0, v1); + let v23_lower = + u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v2, v3); + let v01_upper = + u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v0, v1); + let v23_upper = + u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v2, v3); + let out0 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( + v01_lower, v23_lower, + ); + let out1 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( + v01_lower, v23_lower, + ); + let out2 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( + v01_upper, v23_upper, + ); + let out3 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( + v01_upper, v23_upper, + ); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = + u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v0, v2); + let v13_lower = + u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v1, v3); + let v02_upper = + u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v0, v2); + let v13_upper = + u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v1, v3); + let out0 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( + v02_lower, v13_lower, + ); + let out1 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( + v02_lower, v13_lower, + ); + let out2 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( + v02_upper, v13_upper, + ); + let out3 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( + v02_upper, v13_upper, + ); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { let low = u16x8_extend_low_u8x16(a.into()); let high = u16x8_extend_high_u8x16(a.into()); @@ -1359,6 +1557,56 @@ impl Simd for WasmSimd128 { i16x8_neg(a.into()).simd_into(self) } #[inline(always)] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[3]); + let v01_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v0, v1); + let v23_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v2, v3); + let v01_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v0, v1); + let v23_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v2, v3); + let out0 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_lower, v23_lower); + let out1 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_lower, v23_lower); + let out2 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_upper, v23_upper); + let out3 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_upper, v23_upper); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v0, v2); + let v13_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v1, v3); + let v02_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v0, v2); + let v13_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v1, v3); + let out0 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_lower, v13_lower); + let out1 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_lower, v13_lower); + let out2 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_upper, v13_upper); + let out3 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_u16x8(self, val: u16) -> u16x8 { u16x8_splat(val).simd_into(self) } @@ -1604,6 +1852,56 @@ impl Simd for WasmSimd128 { } } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[3]); + let v01_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v0, v1); + let v23_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v2, v3); + let v01_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v0, v1); + let v23_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v2, v3); + let out0 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_lower, v23_lower); + let out1 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_lower, v23_lower); + let out2 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_upper, v23_upper); + let out3 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_upper, v23_upper); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v0, v2); + let v13_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v1, v3); + let v02_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v0, v2); + let v13_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v1, v3); + let out0 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_lower, v13_lower); + let out1 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_lower, v13_lower); + let out2 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_upper, v13_upper); + let out3 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { let val: i16 = if val { !0 } else { 0 }; i16x8_splat(val).simd_into(self) @@ -1915,6 +2213,56 @@ impl Simd for WasmSimd128 { i32x4_neg(a.into()).simd_into(self) } #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[3]); + let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); + let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); + let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); + let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); + let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); + let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); + let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); + let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); + let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); + let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); + let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); + let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); + let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); + let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); + let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { f32x4_convert_i32x4(a.into()).simd_into(self) } @@ -2140,6 +2488,56 @@ impl Simd for WasmSimd128 { } } #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[3]); + let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); + let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); + let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); + let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); + let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); + let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); + let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); + let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); + let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); + let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); + let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); + let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); + let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); + let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); + let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { f32x4_convert_u32x4(a.into()).simd_into(self) } @@ -2494,6 +2892,48 @@ impl Simd for WasmSimd128 { } } #[inline(always)] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[3]); + let out0 = u64x2_shuffle::<0, 2>(v0, v2); + let out1 = u64x2_shuffle::<1, 3>(v0, v2); + let out2 = u64x2_shuffle::<0, 2>(v1, v3); + let out3 = u64x2_shuffle::<1, 3>(v1, v3); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let out0 = u64x2_shuffle::<0, 2>(v0, v1); + let out1 = u64x2_shuffle::<0, 2>(v2, v3); + let out2 = u64x2_shuffle::<1, 3>(v0, v1); + let out3 = u64x2_shuffle::<1, 3>(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { i64x2_splat(val).simd_into(self) } @@ -2715,6 +3155,48 @@ impl Simd for WasmSimd128 { i64x2_neg(a.into()).simd_into(self) } #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[3]); + let out0 = u64x2_shuffle::<0, 2>(v0, v2); + let out1 = u64x2_shuffle::<1, 3>(v0, v2); + let out2 = u64x2_shuffle::<0, 2>(v1, v3); + let out3 = u64x2_shuffle::<1, 3>(v1, v3); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let out0 = u64x2_shuffle::<0, 2>(v0, v1); + let out1 = u64x2_shuffle::<0, 2>(v2, v3); + let out2 = u64x2_shuffle::<1, 3>(v0, v1); + let out3 = u64x2_shuffle::<1, 3>(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { u64x2_splat(val).simd_into(self) } @@ -2952,6 +3434,48 @@ impl Simd for WasmSimd128 { } } #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[3]); + let out0 = u64x2_shuffle::<0, 2>(v0, v2); + let out1 = u64x2_shuffle::<1, 3>(v0, v2); + let out2 = u64x2_shuffle::<0, 2>(v1, v3); + let out3 = u64x2_shuffle::<1, 3>(v1, v3); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let out0 = u64x2_shuffle::<0, 2>(v0, v1); + let out1 = u64x2_shuffle::<0, 2>(v2, v3); + let out2 = u64x2_shuffle::<1, 3>(v0, v1); + let out3 = u64x2_shuffle::<1, 3>(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { let val: i64 = if val { !0 } else { 0 }; i64x2_splat(val).simd_into(self) @@ -7361,52 +7885,6 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[3]); - let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); - let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); - let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); - let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); - let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); - let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); - let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); - let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); - let combined_lower = self.combine_f32x4(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_f32x4(out2.simd_into(self), out3.simd_into(self)); - self.combine_f32x8(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - let (lower, upper) = self.split_f32x16(a); - let (v0_vec, v1_vec) = self.split_f32x8(lower); - let (v2_vec, v3_vec) = self.split_f32x8(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); - let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); - let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); - let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); - let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); - let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); - let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); - let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) @@ -8474,76 +8952,6 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - let (chunks, []) = src.as_chunks::<16usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[3]); - let v01_lower = - u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v0, v1); - let v23_lower = - u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v2, v3); - let v01_upper = - u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v0, v1); - let v23_upper = - u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v2, v3); - let out0 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( - v01_lower, v23_lower, - ); - let out1 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( - v01_lower, v23_lower, - ); - let out2 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( - v01_upper, v23_upper, - ); - let out3 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( - v01_upper, v23_upper, - ); - let combined_lower = self.combine_u8x16(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_u8x16(out2.simd_into(self), out3.simd_into(self)); - self.combine_u8x32(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - let (lower, upper) = self.split_u8x64(a); - let (v0_vec, v1_vec) = self.split_u8x32(lower); - let (v2_vec, v3_vec) = self.split_u8x32(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let v02_lower = - u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v0, v2); - let v13_lower = - u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v1, v3); - let v02_upper = - u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v0, v2); - let v13_upper = - u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v1, v3); - let out0 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( - v02_lower, v13_lower, - ); - let out1 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( - v02_lower, v13_lower, - ); - let out2 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( - v02_upper, v13_upper, - ); - let out3 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( - v02_upper, v13_upper, - ); - let (chunks, []) = dest.as_chunks_mut::<16usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { let half = self.splat_mask8x32(val); self.combine_mask8x32(half, half) @@ -9461,52 +9869,6 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - let (chunks, []) = src.as_chunks::<8usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[3]); - let v01_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v0, v1); - let v23_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v2, v3); - let v01_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v0, v1); - let v23_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v2, v3); - let out0 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_lower, v23_lower); - let out1 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_lower, v23_lower); - let out2 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_upper, v23_upper); - let out3 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_upper, v23_upper); - let combined_lower = self.combine_u16x8(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_u16x8(out2.simd_into(self), out3.simd_into(self)); - self.combine_u16x16(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - let (lower, upper) = self.split_u16x32(a); - let (v0_vec, v1_vec) = self.split_u16x16(lower); - let (v2_vec, v3_vec) = self.split_u16x16(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let v02_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v0, v2); - let v13_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v1, v3); - let v02_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v0, v2); - let v13_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v1, v3); - let out0 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_lower, v13_lower); - let out1 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_lower, v13_lower); - let out2 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_upper, v13_upper); - let out3 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_upper, v13_upper); - let (chunks, []) = dest.as_chunks_mut::<8usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { let (a0, a1) = self.split_u16x32(a); self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) @@ -10297,52 +10659,6 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[3]); - let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); - let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); - let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); - let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); - let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); - let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); - let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); - let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); - let combined_lower = self.combine_u32x4(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_u32x4(out2.simd_into(self), out3.simd_into(self)); - self.combine_u32x8(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - let (lower, upper) = self.split_u32x16(a); - let (v0_vec, v1_vec) = self.split_u32x8(lower); - let (v2_vec, v3_vec) = self.split_u32x8(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); - let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); - let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); - let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); - let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); - let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); - let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); - let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { let (a0, a1) = self.split_u32x16(a); self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) @@ -11406,44 +11722,6 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - let (chunks, []) = src.as_chunks::<2usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[3]); - let out0 = u64x2_shuffle::<0, 2>(v0, v2); - let out1 = u64x2_shuffle::<1, 3>(v0, v2); - let out2 = u64x2_shuffle::<0, 2>(v1, v3); - let out3 = u64x2_shuffle::<1, 3>(v1, v3); - let combined_lower = self.combine_u64x2(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_u64x2(out2.simd_into(self), out3.simd_into(self)); - self.combine_u64x4(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - let (lower, upper) = self.split_u64x8(a); - let (v0_vec, v1_vec) = self.split_u64x4(lower); - let (v2_vec, v3_vec) = self.split_u64x4(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let out0 = u64x2_shuffle::<0, 2>(v0, v1); - let out1 = u64x2_shuffle::<0, 2>(v2, v3); - let out2 = u64x2_shuffle::<1, 3>(v0, v1); - let out3 = u64x2_shuffle::<1, 3>(v2, v3); - let (chunks, []) = dest.as_chunks_mut::<2usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index 1774c0a7..25faf879 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -277,26 +277,8 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { OpSig::MaskSet => { panic!("Mask set must operate on the full mask vector") } - OpSig::LoadInterleaved { - block_size, - block_count, - } => { - let split_len = (block_size * block_count) as usize / (ty.scalar_bits * 2); - let ty_rust = ty.rust(); - quote! { - #method_sig { - let (chunks, _) = src.as_chunks::<#split_len>(); - let lo = self.#do_half(&chunks[0]); - let hi = self.#do_half(&chunks[1]); - #ty_rust { - val: crate::transmute::checked_transmute_copy(&[lo.val, hi.val]), - simd: self, - } - } - } - } - OpSig::StoreInterleaved { .. } => { - panic!("The generic fallback is not implemented for this operation") + OpSig::LoadInterleaved { .. } | OpSig::StoreInterleaved { .. } => { + panic!("Interleaved memory operations must operate on full 128-bit vectors") } OpSig::Split { .. } | OpSig::Combine { .. } => { panic!("These operations require more information about the target platform"); diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index de5c8043..975e1671 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -517,13 +517,23 @@ impl Level for Fallback { block_size, block_count, } => { - let len = (block_size * block_count) as usize / vec_ty.scalar_bits; - let items = - interleave_indices(len, block_count as usize, |idx| quote! { src[#idx] }); + let block_count = block_count as usize; + let elems_per_vec = block_size as usize / vec_ty.scalar_bits; + let vectors = (0..block_count).map(|channel| { + let items = make_list( + (0..elems_per_vec) + .map(|lane| { + let idx = lane * block_count + channel; + quote! { src[#idx] } + }) + .collect(), + ); + quote! { #items.simd_into(self) } + }); quote! { #method_sig { - #items.simd_into(self) + [#(#vectors),*] } } } @@ -531,9 +541,17 @@ impl Level for Fallback { block_size, block_count, } => { - let len = (block_size * block_count) as usize / vec_ty.scalar_bits; - let items = - interleave_indices(len, len / block_count as usize, |idx| quote! { a[#idx] }); + let block_count = block_count as usize; + let elems_per_vec = block_size as usize / vec_ty.scalar_bits; + let items = make_list( + (0..elems_per_vec) + .flat_map(|lane| { + (0..block_count).map(move |channel| { + quote! { vectors[#channel][#lane] } + }) + }) + .collect(), + ); quote! { #method_sig { @@ -567,19 +585,6 @@ impl Level for Fallback { } } -fn interleave_indices( - len: usize, - stride: usize, - func: impl FnMut(usize) -> TokenStream, -) -> TokenStream { - let indices = { - let indices = (0..len).collect::>(); - interleave(&indices, stride) - }; - - make_list(indices.into_iter().map(func).collect::>()) -} - fn lane(value: TokenStream, vec_ty: &VecType, idx: usize) -> TokenStream { if vec_ty.scalar == ScalarType::Mask { quote! { #value.val.0[#idx] } @@ -606,15 +611,3 @@ fn rhs_reference(method: &str) -> bool { fn make_list(items: Vec) -> TokenStream { quote!([#( #items, )*]) } - -fn interleave(input: &[usize], width: usize) -> Vec { - let height = input.len() / width; - - let mut output = Vec::with_capacity(input.len()); - for col in 0..width { - for row in 0..height { - output.push(input[row * width + col]); - } - } - output -} diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index 3c17becb..0412f74d 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -18,6 +18,19 @@ use crate::{ #[derive(Clone, Copy)] pub(crate) struct Neon; +fn neon_multi_vector_ty(vec_ty: &VecType, count: u16) -> Ident { + let scalar = match vec_ty.scalar { + ScalarType::Float => "float", + ScalarType::Unsigned => "uint", + ScalarType::Int => "int", + ScalarType::Mask => unreachable!("interleaved operations are not defined for masks"), + }; + Ident::new( + &format!("{scalar}{}x{}x{count}_t", vec_ty.scalar_bits, vec_ty.len), + Span::call_site(), + ) +} + impl Level for Neon { fn name(&self) -> &'static str { "Neon" @@ -137,22 +150,18 @@ impl Level for Neon { block_size, block_count, } => { - assert_eq!(block_count, 4, "only count of 4 is currently supported"); - let intrinsic = { - // The function expects 64-bit or 128-bit - let ty = VecType::new( - vec_ty.scalar, - vec_ty.scalar_bits, - block_size as usize / vec_ty.scalar_bits, - ); - simple_intrinsic("vld4", &ty) - }; + assert_eq!( + vec_ty.n_bits(), + block_size as usize, + "NEON interleaved loads require full block-sized vectors" + ); + let intrinsic = simple_intrinsic(&format!("vld{block_count}"), vec_ty); + let fields = (0..block_count).map(Literal::u16_unsuffixed); quote! { #method_sig { - unsafe { - #intrinsic(src.as_ptr()).simd_into(self) - } + let native = unsafe { #intrinsic(src.as_ptr()) }; + [#(native.#fields.simd_into(self)),*] } } } @@ -160,22 +169,26 @@ impl Level for Neon { block_size, block_count, } => { - assert_eq!(block_count, 4, "only count of 4 is currently supported"); - let intrinsic = { - // The function expects 64-bit or 128-bit - let ty = VecType::new( - vec_ty.scalar, - vec_ty.scalar_bits, - block_size as usize / vec_ty.scalar_bits, - ); - simple_intrinsic("vst4", &ty) - }; + assert_eq!( + vec_ty.n_bits(), + block_size as usize, + "NEON interleaved stores require full block-sized vectors" + ); + let intrinsic = simple_intrinsic(&format!("vst{block_count}"), vec_ty); + let aggregate_ty = neon_multi_vector_ty(vec_ty, block_count); + let indices = 0..block_count as usize; + let native_ty = self.arch_ty(vec_ty); + let native_values = (0..block_count as usize) + .map(|idx| format_ident!("v{idx}")) + .collect::>(); + let native_decls = native_values.iter().zip(indices).map(|(value, idx)| { + quote! { let #value: #native_ty = vectors[#idx].into(); } + }); quote! { #method_sig { - unsafe { - #intrinsic(dest.as_mut_ptr(), a.into()) - } + #(#native_decls)* + unsafe { #intrinsic(dest.as_mut_ptr(), #aggregate_ty(#(#native_values),*)); } } } } diff --git a/fearless_simd_gen/src/mk_simd_trait.rs b/fearless_simd_gen/src/mk_simd_trait.rs index 9f50dfee..c39e7a51 100644 --- a/fearless_simd_gen/src/mk_simd_trait.rs +++ b/fearless_simd_gen/src/mk_simd_trait.rs @@ -20,8 +20,12 @@ pub(crate) fn mk_simd_trait() -> TokenStream { for op in ops_for_type(vec_ty) { let method_sig = op.simd_trait_method_sig(vec_ty); let doc = op.format_docstring(TyFlavor::SimdTrait); + let doc_alias = op + .doc_alias() + .map(|alias| quote! { #[doc(alias = #alias)] }); methods.extend(quote! { #[doc = #doc] + #doc_alias #method_sig; }); } diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index 15bafb0f..a6251648 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -699,19 +699,6 @@ impl Level for WasmSimd128 { _ => panic!("unsupported scalar_bits"), }; - let block_ty = vec_ty.block_ty(); - let block_ty_2x = - VecType::new(block_ty.scalar, block_ty.scalar_bits, block_ty.len * 2); - - let combine_method = generic_op_name("combine", &block_ty); - let combine_method_2x = generic_op_name("combine", &block_ty_2x); - - let combine_code = quote! { - let combined_lower = self.#combine_method(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.#combine_method(out2.simd_into(self), out3.simd_into(self)); - self.#combine_method_2x(combined_lower, combined_upper) - }; - let shuffle_code = if vec_ty.scalar_bits == 64 { quote! { let out0 = #shuffle_fn::<#i1>(v0, v2); @@ -756,7 +743,12 @@ impl Level for WasmSimd128 { ); #shuffle_code - #combine_code + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] } } } @@ -788,24 +780,11 @@ impl Level for WasmSimd128 { _ => panic!("unsupported scalar_bits"), }; - let block_ty = vec_ty.block_ty(); - let block_ty_2x = - VecType::new(block_ty.scalar, block_ty.scalar_bits, block_ty.len * 2); - let block_ty_4x = - VecType::new(block_ty.scalar, block_ty.scalar_bits, block_ty.len * 4); - - let split_method = generic_op_name("split", &block_ty_2x); - let split_method_2x = generic_op_name("split", &block_ty_4x); - let split_code = quote! { - let (lower, upper) = self.#split_method_2x(a); - let (v0_vec, v1_vec) = self.#split_method(lower); - let (v2_vec, v3_vec) = self.#split_method(upper); - - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); }; let shuffle_code = if vec_ty.scalar_bits == 64 { diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index b9f6284c..19e05a18 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -3246,7 +3246,7 @@ impl X86 { "only 128-bit blocks are currently supported" ); assert_eq!(block_count, 4, "only count of 4 is currently supported"); - if *self == Self::Avx512 && vec_ty.n_bits() == 512 { + if *self == Self::Avx512 { return self.handle_avx512_load_interleaved(op, vec_ty, block_size, block_count); } if *self == Self::Sse2 && matches!(vec_ty.scalar_bits, 8 | 16) { @@ -3254,8 +3254,8 @@ impl X86 { } match vec_ty.scalar_bits { 64 | 32 | 16 | 8 => { - let avx2_u64 = *self == Self::Avx2 && vec_ty.scalar_bits == 64; - let block_len = if avx2_u64 { + let avx2_64 = *self == Self::Avx2 && vec_ty.scalar_bits == 64; + let block_len = if avx2_64 { 4 } else { block_size as usize / vec_ty.scalar_bits @@ -3263,24 +3263,23 @@ impl X86 { let block_ty = VecType::new(vec_ty.scalar, vec_ty.scalar_bits, block_len); let scalar_ty = block_ty.scalar.rust(block_ty.scalar_bits); let native_ty = self.arch_ty(&block_ty); + let native_block_ty = self.arch_ty(vec_ty); let vec_32 = block_ty.reinterpret(block_ty.scalar, 32); let unpacklo_32 = simple_sign_unaware_intrinsic("unpacklo", &vec_32); let unpackhi_32 = simple_sign_unaware_intrinsic("unpackhi", &vec_32); let vec_64 = block_ty.reinterpret(block_ty.scalar, 64); let unpacklo_64 = simple_sign_unaware_intrinsic("unpacklo", &vec_64); let unpackhi_64 = simple_sign_unaware_intrinsic("unpackhi", &vec_64); - - let vec_combined = - VecType::new(block_ty.scalar, block_ty.scalar_bits, block_ty.len * 2); - let combine_half = Ident::new( - &format!("combine_{}", block_ty.rust_name()), - Span::call_site(), - ); - let combine_full = Ident::new( - &format!("combine_{}", vec_combined.rust_name()), - Span::call_site(), + let permute_128 = intrinsic_ident( + match vec_ty.scalar { + ScalarType::Float => "permute2f128", + _ => "permute2x128", + }, + coarse_type(&block_ty), + 256, ); - if avx2_u64 { + + if avx2_64 { return self.kernel_method(op, vec_ty, |token| { quote! { let (chunks, []) = src.as_chunks::<4>() else { @@ -3291,10 +3290,17 @@ impl X86 { let lo = #unpacklo_64(v0, v1); // [0,4,2,6] let hi = #unpackhi_64(v0, v1); // [1,5,3,7] - let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); // [0,4,1,5] - let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); // [2,6,3,7] - - #token.#combine_half(out0.simd_into(#token), out1.simd_into(#token)) + let out0 = #permute_128::<0x20>(lo, hi); // [0,4,1,5] + let out1 = #permute_128::<0x31>(lo, hi); // [2,6,3,7] + let outputs: [#native_block_ty; 4] = + crate::transmute::checked_transmute_copy(&[out0, out1]); + + [ + outputs[0].simd_into(#token), + outputs[1].simd_into(#token), + outputs[2].simd_into(#token), + outputs[3].simd_into(#token), + ] } }); } @@ -3402,10 +3408,12 @@ impl X86 { #initial_unpack #final_unpack - #token.#combine_full( - #token.#combine_half(out0.simd_into(#token), out1.simd_into(#token)), - #token.#combine_half(out2.simd_into(#token), out3.simd_into(#token)), - ) + [ + out0.simd_into(#token), + out1.simd_into(#token), + out2.simd_into(#token), + out3.simd_into(#token), + ] } }) } @@ -3427,16 +3435,22 @@ impl X86 { assert_eq!(block_count, 4, "only count of 4 is currently supported"); assert_eq!( vec_ty.n_bits(), - 512, - "AVX-512 interleaved loads only specialize 512-bit vectors" + 128, + "AVX-512 interleaved loads return 128-bit vectors" + ); + let total_ty = VecType::new( + vec_ty.scalar, + vec_ty.scalar_bits, + vec_ty.len * block_count as usize, ); let scalar_ty = vec_ty.scalar.rust(vec_ty.scalar_bits); - let native_ty = self.arch_ty(vec_ty); - let len = vec_ty.len; - let permute = avx512_permutexvar_intrinsic(vec_ty); + let native_ty = self.arch_ty(&total_ty); + let native_block_ty = self.arch_ty(vec_ty); + let len = total_ty.len; + let permute = avx512_permutexvar_intrinsic(&total_ty); let indices = avx512_index_vector( - vec_ty, - interleaved_load_indices(vec_ty.len, block_count as usize), + &total_ty, + interleaved_load_indices(total_ty.len, block_count as usize), ); self.kernel_method(op, vec_ty, |token| { @@ -3445,7 +3459,15 @@ impl X86 { crate::transmute::checked_transmute_copy::<[#scalar_ty; #len], #native_ty>( src, ); - #permute(#indices, lanes).simd_into(#token) + let lanes = #permute(#indices, lanes); + let outputs: [#native_block_ty; 4] = + crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(#token), + outputs[1].simd_into(#token), + outputs[2].simd_into(#token), + outputs[3].simd_into(#token), + ] } }) } @@ -3462,7 +3484,7 @@ impl X86 { "only 128-bit blocks are currently supported" ); assert_eq!(block_count, 4, "only count of 4 is currently supported"); - if *self == Self::Avx512 && vec_ty.n_bits() == 512 { + if *self == Self::Avx512 { return self.handle_avx512_store_interleaved(op, vec_ty, block_size, block_count); } if *self == Self::Sse2 && matches!(vec_ty.scalar_bits, 8 | 16) { @@ -3470,8 +3492,8 @@ impl X86 { } match vec_ty.scalar_bits { 64 | 32 | 16 | 8 => { - let avx2_u64 = *self == Self::Avx2 && vec_ty.scalar_bits == 64; - let block_len = if avx2_u64 { + let avx2_64 = *self == Self::Avx2 && vec_ty.scalar_bits == 64; + let block_len = if avx2_64 { 4 } else { block_size as usize / vec_ty.scalar_bits @@ -3479,34 +3501,42 @@ impl X86 { let block_ty = VecType::new(vec_ty.scalar, vec_ty.scalar_bits, block_len); let scalar_ty = block_ty.scalar.rust(block_ty.scalar_bits); let native_ty = self.arch_ty(&block_ty); + let native_block_ty = self.arch_ty(vec_ty); let vec_32 = block_ty.reinterpret(block_ty.scalar, 32); let unpacklo_32 = simple_sign_unaware_intrinsic("unpacklo", &vec_32); let unpackhi_32 = simple_sign_unaware_intrinsic("unpackhi", &vec_32); let vec_64 = block_ty.reinterpret(block_ty.scalar, 64); let unpacklo_64 = simple_sign_unaware_intrinsic("unpacklo", &vec_64); let unpackhi_64 = simple_sign_unaware_intrinsic("unpackhi", &vec_64); - - let vec_combined = - VecType::new(block_ty.scalar, block_ty.scalar_bits, block_ty.len * 2); - let split_half = Ident::new( - &format!("split_{}", vec_combined.rust_name()), - Span::call_site(), + let permute_128 = intrinsic_ident( + match vec_ty.scalar { + ScalarType::Float => "permute2f128", + _ => "permute2x128", + }, + coarse_type(&block_ty), + 256, ); - let split_full = - Ident::new(&format!("split_{}", vec_ty.rust_name()), Span::call_site()); // For 64-bit values, AVX2 permits a more efficient implementation. // It is special-cased here as a full early return because plumbing it // through the rest of the logic in this function complicates it significantly. - if avx2_u64 { + if avx2_64 { return self.kernel_method(op, vec_ty, |token| { quote! { - let (v0, v1) = #token.#split_full(a); - let v0: #native_ty = v0.into(); - let v1: #native_ty = v1.into(); - - let lo = _mm256_permute2x128_si256::<0x20>(v0, v1); // [0,1,4,5] - let hi = _mm256_permute2x128_si256::<0x31>(v0, v1); // [2,3,6,7] + let _ = #token; + let inputs: [#native_block_ty; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let wide_inputs: [#native_ty; 2] = + crate::transmute::checked_transmute_copy(&inputs); + let v0 = wide_inputs[0]; + let v1 = wide_inputs[1]; + + let lo = #permute_128::<0x20>(v0, v1); // [0,1,4,5] + let hi = #permute_128::<0x31>(v0, v1); // [2,3,6,7] let out0 = #unpacklo_64(lo, hi); // [0,2,4,6] let out1 = #unpackhi_64(lo, hi); // [1,3,5,7] @@ -3602,13 +3632,11 @@ impl X86 { self.kernel_method(op, vec_ty, |token| { quote! { - let (v01, v23) = #token.#split_full(a); - let (v0, v1) = #token.#split_half(v01); - let (v2, v3) = #token.#split_half(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); + let _ = #token; + let v0: #native_ty = vectors[0].into(); + let v1: #native_ty = vectors[1].into(); + let v2: #native_ty = vectors[2].into(); + let v3: #native_ty = vectors[3].into(); #initial_unpack #final_unpack @@ -3644,21 +3672,35 @@ impl X86 { assert_eq!(block_count, 4, "only count of 4 is currently supported"); assert_eq!( vec_ty.n_bits(), - 512, - "AVX-512 interleaved stores only specialize 512-bit vectors" + 128, + "AVX-512 interleaved stores accept 128-bit vectors" + ); + let total_ty = VecType::new( + vec_ty.scalar, + vec_ty.scalar_bits, + vec_ty.len * block_count as usize, ); let scalar_ty = vec_ty.scalar.rust(vec_ty.scalar_bits); - let native_ty = self.arch_ty(vec_ty); - let len = vec_ty.len; - let permute = avx512_permutexvar_intrinsic(vec_ty); + let native_ty = self.arch_ty(&total_ty); + let native_block_ty = self.arch_ty(vec_ty); + let len = total_ty.len; + let permute = avx512_permutexvar_intrinsic(&total_ty); let indices = avx512_index_vector( - vec_ty, - interleaved_store_indices(vec_ty.len, block_count as usize), + &total_ty, + interleaved_store_indices(total_ty.len, block_count as usize), ); - self.kernel_method(op, vec_ty, |_| { + self.kernel_method(op, vec_ty, |token| { quote! { - let lanes = #permute(#indices, a.into()); + let _ = #token; + let inputs: [#native_block_ty; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: #native_ty = crate::transmute::checked_transmute_copy(&inputs); + let lanes = #permute(#indices, lanes); crate::transmute::checked_transmute_store::<#native_ty, [#scalar_ty; #len]>( lanes, dest, diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index 3dd2bf52..e401c55b 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -107,14 +107,15 @@ pub(crate) enum OpSig { /// Takes a mutable mask vector, a lane index, and a boolean, and updates the lane in place. MaskSet, /// Takes an argument of an array of a certain scalar type, with the length (`block_size` * `block_count`) / [scalar - /// type's byte size]. Returns a vector type of that scalar type and length. + /// type's byte size]. Returns `block_count` vectors whose width is `block_size`. /// /// First argument is the base block size (i.e. 128), second argument is how many blocks. For example, /// `LoadInterleaved(128, 4)` would correspond to the NEON instructions `vld4q_f32`, while `LoadInterleaved(64, 4)` /// would correspond to `vld4_f32`. LoadInterleaved { block_size: u16, block_count: u16 }, - /// The inverse of [`OpSig::LoadInterleaved`]. Takes a vector argument with the length (`block_size` * `block_count`) / - /// [scalar type's byte size], and a mutable reference to a scalar array of the same length, and returns nothing. + /// The inverse of [`OpSig::LoadInterleaved`]. Takes `block_count` vectors whose width is `block_size` and a mutable + /// reference to a scalar array with the length (`block_size` * `block_count`) / [scalar type's byte size], and + /// returns nothing. StoreInterleaved { block_size: u16, block_count: u16 }, } @@ -175,6 +176,14 @@ impl Op { } } + pub(crate) fn doc_alias(&self) -> Option<&'static str> { + match self.method { + "load_four_interleaved" => Some("load_interleaved"), + "store_four_interleaved" => Some("store_interleaved"), + _ => None, + } + } + pub(crate) fn simd_trait_method_sig(&self, vec_ty: &VecType) -> TokenStream { let method_ident = generic_op_name(self.method, vec_ty); let sig = self.simd_trait_sig_parts(vec_ty, quote! { Self }); @@ -257,14 +266,16 @@ impl Op { block_count, } => { let arg_ty = load_interleaved_arg_ty(*block_size, *block_count, vec_ty); - (vec![arg_ty], vec) + let block_count = *block_count as usize; + (vec![arg_ty], quote! { [#vec; #block_count] }) } OpSig::StoreInterleaved { block_size, block_count, } => { let arg_ty = store_interleaved_arg_ty(*block_size, *block_count, vec_ty); - (vec![vec.clone(), arg_ty], quote! { () }) + let block_count = *block_count as usize; + (vec![quote! { [#vec; #block_count] }, arg_ty], quote! { () }) } OpSig::Compare => { let result = vec_ty.mask_ty().rust(); @@ -1264,46 +1275,51 @@ pub(crate) fn ops_for_type(ty: &VecType) -> Vec { ops.push(NEGATE_INT); } - if ty.scalar == ScalarType::Float && ty.scalar_bits == 64 { - return ops; - } - - if matches!(ty.scalar, ScalarType::Unsigned | ScalarType::Float) && ty.n_bits() == 512 { + if matches!( + ty.scalar, + ScalarType::Int | ScalarType::Unsigned | ScalarType::Float + ) && ty.n_bits() == 128 + { ops.push(Op::new( - "load_interleaved_128", + "load_four_interleaved", OpKind::AssociatedOnly, OpSig::LoadInterleaved { block_size: 128, block_count: 4, }, - "Load elements from an array with 4-way interleaving.\n\n\ - This is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded \ - vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them \ - into one vector.\n\n\ - For example, with 32-bit lanes, memory laid out as \ - `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as \ - `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`.", + "Load four 128-bit vectors from an array with 4-way interleaving.\n\n\ + This is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\n\ + For example, with 32-bit lanes, memory laid out as\ + `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as\ + `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`.", )); } - if matches!(ty.scalar, ScalarType::Unsigned | ScalarType::Float) && ty.n_bits() == 512 { + if matches!( + ty.scalar, + ScalarType::Int | ScalarType::Unsigned | ScalarType::Float + ) && ty.n_bits() == 128 + { ops.push(Op::new( - "store_interleaved_128", + "store_four_interleaved", OpKind::AssociatedOnly, OpSig::StoreInterleaved { block_size: 128, block_count: 4, }, - "Store elements to an array with 4-way interleaving.\n\n\ - This is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: \ - `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit \ - vectors into lane-interleaved memory.\n\n\ - For example, with 32-bit lanes, a vector containing \ - `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as \ - `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`.", + "Store four 128-bit vectors to an array with 4-way interleaving.\n\n\ + This is the inverse of `load_four_interleaved`.\n\n\ + This is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\n\ + For example, with 32-bit lanes, vectors containing \ + `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as \ + `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`.", )); } + if ty.scalar == ScalarType::Float && ty.scalar_bits == 64 { + return ops; + } + if matches!(ty.scalar, ScalarType::Unsigned) { if let Some(target_ty) = ty.widened() { ops.push(Op::new( @@ -1510,7 +1526,7 @@ impl OpSig { Self::Ternary | Self::Select => &["a", "b", "c"], Self::Shift => &["a", "shift"], Self::LoadInterleaved { .. } => &["src"], - Self::StoreInterleaved { .. } => &["a", "dest"], + Self::StoreInterleaved { .. } => &["vectors", "dest"], } } fn vec_trait_arg_names(&self) -> &'static [&'static str] { diff --git a/fearless_simd_tests/tests/harness/ops/load_four_interleaved.rs b/fearless_simd_tests/tests/harness/ops/load_four_interleaved.rs new file mode 100644 index 00000000..0fe8bd3f --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/load_four_interleaved.rs @@ -0,0 +1,325 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +// One concrete test row per supported vector type. + +#[simd_test] +fn load_four_interleaved_f32x4(simd: S) { + let data = [ + 0.0, + 4.0, + 8.0, + f32::MIN, + f32::NAN, + -0.0, + 9.0, + 13.0, + f32::INFINITY, + 6.0, + -10.0, + f32::MAX, + -3.0, + f32::NEG_INFINITY, + 11.0, + 15.0, + ]; + let expected = [ + [0.0, f32::NAN, f32::INFINITY, -3.0], + [4.0, -0.0, 6.0, f32::NEG_INFINITY], + [8.0, 9.0, -10.0, 11.0], + [f32::MIN, 13.0, f32::MAX, 15.0], + ]; + + // Note: f32::NAN != f32::NAN hence we compare the bit pattern. + let result = simd.load_four_interleaved_f32x4(&data); + assert_eq!( + result.map(|vector| (*vector).map(f32::to_bits)), + expected.map(|vector| vector.map(f32::to_bits)), + ); +} + +#[simd_test] +fn load_four_interleaved_f64x2(simd: S) { + let data = [ + 0.0, + f64::NAN, + f64::INFINITY, + -3.0, + -0.0, + f64::MIN, + f64::NEG_INFINITY, + f64::MAX, + ]; + let expected = [ + [0.0, -0.0], + [f64::NAN, f64::MIN], + [f64::INFINITY, f64::NEG_INFINITY], + [-3.0, f64::MAX], + ]; + + // Note: f64::NAN != f64::NAN hence we compare the bit pattern. + let result = simd.load_four_interleaved_f64x2(&data); + assert_eq!( + result.map(|vector| (*vector).map(f64::to_bits)), + expected.map(|vector| vector.map(f64::to_bits)), + ); +} + +#[simd_test] +fn load_four_interleaved_i8x16(simd: S) { + #[rustfmt::skip] + let data: [i8; 64] = [ + i8::MIN, -31, -30, -29, -28, -27, -26, -25, + -24, -23, -22, -21, -20, -19, -18, -17, + -16, -15, -14, -13, -12, -11, -10, -9, + -8, -7, -6, -5, -4, -3, -2, -1, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, i8::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_i8x16(&data) + .map(|vector| *vector), + [ + [ + i8::MIN, + -28, + -24, + -20, + -16, + -12, + -8, + -4, + 0, + 4, + 8, + 12, + 16, + 20, + 24, + 28 + ], + [ + -31, -27, -23, -19, -15, -11, -7, -3, 1, 5, 9, 13, 17, 21, 25, 29 + ], + [ + -30, -26, -22, -18, -14, -10, -6, -2, 2, 6, 10, 14, 18, 22, 26, 30 + ], + [ + -29, + -25, + -21, + -17, + -13, + -9, + -5, + -1, + 3, + 7, + 11, + 15, + 19, + 23, + 27, + i8::MAX + ], + ] + ); +} + +#[simd_test] +fn load_four_interleaved_u8x16(simd: S) { + #[rustfmt::skip] + let data: [u8; 64] = [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, u8::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_u8x16(&data) + .map(|vector| *vector), + [ + [0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60], + [1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61], + [2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62], + [ + 3, + 7, + 11, + 15, + 19, + 23, + 27, + 31, + 35, + 39, + 43, + 47, + 51, + 55, + 59, + u8::MAX + ], + ] + ); +} + +#[simd_test] +fn load_four_interleaved_i16x8(simd: S) { + let data: [i16; 32] = [ + i16::MIN, + -15, + -14, + -13, + -12, + -11, + -10, + -9, + -8, + -7, + -6, + -5, + -4, + -3, + -2, + -1, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + i16::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_i16x8(&data) + .map(|vector| *vector), + [ + [i16::MIN, -12, -8, -4, 0, 4, 8, 12], + [-15, -11, -7, -3, 1, 5, 9, 13], + [-14, -10, -6, -2, 2, 6, 10, 14], + [-13, -9, -5, -1, 3, 7, 11, i16::MAX], + ] + ); +} + +#[simd_test] +fn load_four_interleaved_u16x8(simd: S) { + #[rustfmt::skip] + let data: [u16; 32] = [ + u16::MIN, 2, 3, 4, 5, 6, 7, 8, + 10, 20, 30, 40, 50, 60, 70, 80, + 100, 200, 300, 400, 500, 600, 700, 800, + 1000, 2000, 3000, 4000, 5000, 6000, 7000, u16::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_u16x8(&data) + .map(|vector| *vector), + [ + [u16::MIN, 5, 10, 50, 100, 500, 1000, 5000], + [2, 6, 20, 60, 200, 600, 2000, 6000], + [3, 7, 30, 70, 300, 700, 3000, 7000], + [4, 8, 40, 80, 400, 800, 4000, u16::MAX], + ] + ); +} + +#[simd_test] +fn load_four_interleaved_i32x4(simd: S) { + let data: [i32; 16] = [ + i32::MIN, + -7, + -6, + -5, + -4, + -3, + -2, + -1, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + i32::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_i32x4(&data) + .map(|vector| *vector), + [ + [i32::MIN, -4, 0, 4], + [-7, -3, 1, 5], + [-6, -2, 2, 6], + [-5, -1, 3, i32::MAX] + ] + ); +} + +#[simd_test] +fn load_four_interleaved_u32x4(simd: S) { + #[rustfmt::skip] + let data: [u32; 16] = [ + 1, 2, 3, 4, + 10, 20, 30, 40, + 100, 200, 300, 400, + 1000, 2000, 3000, u32::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_u32x4(&data) + .map(|vector| *vector), + [ + [1, 10, 100, 1000], + [2, 20, 200, 2000], + [3, 30, 300, 3000], + [4, 40, 400, u32::MAX], + ] + ); +} + +#[simd_test] +fn load_four_interleaved_i64x2(simd: S) { + let data = [ + i64::MIN, + -3_i64, + -2_i64, + -1_i64, + 0_i64, + 1_i64, + 2_i64, + i64::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_i64x2(&data) + .map(|vector| *vector), + [[i64::MIN, 0], [-3, 1], [-2, 2], [-1, i64::MAX]] + ); +} + +#[simd_test] +fn load_four_interleaved_u64x2(simd: S) { + let data = [u64::MIN, 3_u64, 5_u64, 7_u64, 2_u64, 4_u64, 6_u64, u64::MAX]; + assert_eq!( + simd.load_four_interleaved_u64x2(&data) + .map(|vector| *vector), + [[u64::MIN, 2], [3, 4], [5, 6], [7, u64::MAX]] + ); +} diff --git a/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs b/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs deleted file mode 100644 index 5dea1ba2..00000000 --- a/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2026 the Fearless_SIMD Authors -// SPDX-License-Identifier: Apache-2.0 OR MIT - -use fearless_simd::*; -use fearless_simd_dev_macros::simd_test; - -// One concrete test row per supported vector type. - -#[simd_test] -fn load_interleaved_128_f32x16(simd: S) { - let data = [ - 0.0, - 4.0, - 8.0, - f32::MIN, - f32::NAN, - -0.0, - 9.0, - 13.0, - f32::INFINITY, - 6.0, - -10.0, - f32::MAX, - -3.0, - f32::NEG_INFINITY, - 11.0, - 15.0, - ]; - let expected = [ - 0.0, - f32::NAN, - f32::INFINITY, - -3.0, - 4.0, - -0.0, - 6.0, - f32::NEG_INFINITY, - 8.0, - 9.0, - -10.0, - 11.0, - f32::MIN, - 13.0, - f32::MAX, - 15.0, - ]; - - // Note: f32::NAN != f32::NAN hence we compare the bit pattern. - let result = simd.load_interleaved_128_f32x16(&data); - assert_eq!((*result).map(f32::to_bits), expected.map(f32::to_bits),); -} - -#[simd_test] -fn load_interleaved_128_u32x16(simd: S) { - #[rustfmt::skip] - let data: [u32; 16] = [ - 1, 2, 3, 4, - 10, 20, 30, 40, - 100, 200, 300, 400, - 1000, 2000, 3000, 4000, - ]; - assert_eq!( - *simd.load_interleaved_128_u32x16(&data), - [ - 1, 10, 100, 1000, 2, 20, 200, 2000, 3, 30, 300, 3000, 4, 40, 400, 4000 - ] - ); -} - -#[simd_test] -fn load_interleaved_128_u16x32(simd: S) { - #[rustfmt::skip] - let data: [u16; 32] = [ - 1, 2, 3, 4, - 5, 6, 7, 8, - - 10, 20, 30, 40, - 50, 60, 70, 80, - - 100, 200, 300, 400, - 500, 600, 700, 800, - - 1000, 2000, 3000, 4000, - 5000, 6000, 7000, 8000, - ]; - assert_eq!( - *simd.load_interleaved_128_u16x32(&data), - [ - 1, 5, 10, 50, 100, 500, 1000, 5000, 2, 6, 20, 60, 200, 600, 2000, 6000, 3, 7, 30, 70, - 300, 700, 3000, 7000, 4, 8, 40, 80, 400, 800, 4000, 8000 - ] - ); -} - -#[simd_test] -fn load_interleaved_128_u8x64(simd: S) { - #[rustfmt::skip] - let data: [u8; 64] = [ - 0, 1, 2, 3, - 4, 5, 6, 7, - 8, 9, 10, 11, - 12, 13, 14, 15, - - 16, 17, 18, 19, - 20, 21, 22, 23, - 24, 25, 26, 27, - 28, 29, 30, 31, - - 32, 33, 34, 35, - 36, 37, 38, 39, - 40, 41, 42, 43, - 44, 45, 46, 47, - - 48, 49, 50, 51, - 52, 53, 54, 55, - 56, 57, 58, 59, - 60, 61, 62, 63, - ]; - assert_eq!( - *simd.load_interleaved_128_u8x64(&data), - [ - 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 1, 5, 9, 13, 17, 21, 25, - 29, 33, 37, 41, 45, 49, 53, 57, 61, 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, - 54, 58, 62, 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 - ] - ); -} - -// Additional concrete rows for this operation. - -#[simd_test] -fn load_interleaved_128_u64x8(simd: S) { - let data = [1_u64, 3_u64, 5_u64, 7_u64, 2_u64, 4_u64, 6_u64, 8_u64]; - let a = simd.load_interleaved_128_u64x8(&data); - assert_eq!(*a, [1_u64, 2_u64, 3_u64, 4_u64, 5_u64, 6_u64, 7_u64, 8_u64]); -} diff --git a/fearless_simd_tests/tests/harness/ops/mod.rs b/fearless_simd_tests/tests/harness/ops/mod.rs index 46e7e432..0e6f6adf 100644 --- a/fearless_simd_tests/tests/harness/ops/mod.rs +++ b/fearless_simd_tests/tests/harness/ops/mod.rs @@ -36,7 +36,7 @@ mod index; mod interleave; mod load_array; mod load_array_ref; -mod load_interleaved_128; +mod load_four_interleaved; mod max; mod max_precise; mod min; @@ -72,7 +72,7 @@ mod splat; mod split; mod sqrt; mod store_array; -mod store_interleaved_128; +mod store_four_interleaved; mod store_slice; mod sub; mod swizzle_dyn_precise; diff --git a/fearless_simd_tests/tests/harness/ops/store_four_interleaved.rs b/fearless_simd_tests/tests/harness/ops/store_four_interleaved.rs new file mode 100644 index 00000000..d32fe1ca --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/store_four_interleaved.rs @@ -0,0 +1,500 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +// One concrete test row per supported vector type. + +#[simd_test] +fn store_four_interleaved_f32x4(simd: S) { + let vectors = [ + f32x4::from_slice(simd, &[0.0, f32::NAN, f32::INFINITY, -3.0]), + f32x4::from_slice(simd, &[4.0, -0.0, 6.0, f32::NEG_INFINITY]), + f32x4::from_slice(simd, &[8.0, 9.0, -10.0, 11.0]), + f32x4::from_slice(simd, &[f32::MIN, 13.0, f32::MAX, 15.0]), + ]; + let mut dest = [0.0_f32; 16]; + simd.store_four_interleaved_f32x4(vectors, &mut dest); + + let expected = [ + 0.0, + 4.0, + 8.0, + f32::MIN, + f32::NAN, + -0.0, + 9.0, + 13.0, + f32::INFINITY, + 6.0, + -10.0, + f32::MAX, + -3.0, + f32::NEG_INFINITY, + 11.0, + 15.0, + ]; + + // Note: f32::NAN != f32::NAN hence we compare the bit pattern. + assert_eq!(dest.map(f32::to_bits), expected.map(f32::to_bits)); +} + +#[simd_test] +fn store_four_interleaved_f64x2(simd: S) { + let vectors = [ + f64x2::from_slice(simd, &[0.0, -0.0]), + f64x2::from_slice(simd, &[f64::NAN, f64::MIN]), + f64x2::from_slice(simd, &[f64::INFINITY, f64::NEG_INFINITY]), + f64x2::from_slice(simd, &[-3.0, f64::MAX]), + ]; + let mut dest = [0.0_f64; 8]; + simd.store_four_interleaved_f64x2(vectors, &mut dest); + + let expected = [ + 0.0, + f64::NAN, + f64::INFINITY, + -3.0, + -0.0, + f64::MIN, + f64::NEG_INFINITY, + f64::MAX, + ]; + + // Note: f64::NAN != f64::NAN hence we compare the bit pattern. + assert_eq!(dest.map(f64::to_bits), expected.map(f64::to_bits)); +} + +#[simd_test] +fn store_four_interleaved_i8x16(simd: S) { + let vectors = [ + i8x16::from_slice( + simd, + &[ + i8::MIN, + -31, + -30, + -29, + -28, + -27, + -26, + -25, + -24, + -23, + -22, + -21, + -20, + -19, + -18, + -17, + ], + ), + i8x16::from_slice( + simd, + &[ + -16, -15, -14, -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, + ], + ), + i8x16::from_slice( + simd, + &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + ), + i8x16::from_slice( + simd, + &[ + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + i8::MAX, + ], + ), + ]; + let mut dest = [0_i8; 64]; + simd.store_four_interleaved_i8x16(vectors, &mut dest); + + let expected = [ + i8::MIN, + -16, + 0, + 16, + -31, + -15, + 1, + 17, + -30, + -14, + 2, + 18, + -29, + -13, + 3, + 19, + -28, + -12, + 4, + 20, + -27, + -11, + 5, + 21, + -26, + -10, + 6, + 22, + -25, + -9, + 7, + 23, + -24, + -8, + 8, + 24, + -23, + -7, + 9, + 25, + -22, + -6, + 10, + 26, + -21, + -5, + 11, + 27, + -20, + -4, + 12, + 28, + -19, + -3, + 13, + 29, + -18, + -2, + 14, + 30, + -17, + -1, + 15, + i8::MAX, + ]; + + assert_eq!(dest, expected); +} + +#[simd_test] +fn store_four_interleaved_u8x16(simd: S) { + let vectors = [ + u8x16::from_slice( + simd, + &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + ), + u8x16::from_slice( + simd, + &[ + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ], + ), + u8x16::from_slice( + simd, + &[ + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + ], + ), + u8x16::from_slice( + simd, + &[ + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + u8::MAX, + ], + ), + ]; + let mut dest = [0_u8; 64]; + simd.store_four_interleaved_u8x16(vectors, &mut dest); + + let expected = [ + 0, + 16, + 32, + 48, + 1, + 17, + 33, + 49, + 2, + 18, + 34, + 50, + 3, + 19, + 35, + 51, + 4, + 20, + 36, + 52, + 5, + 21, + 37, + 53, + 6, + 22, + 38, + 54, + 7, + 23, + 39, + 55, + 8, + 24, + 40, + 56, + 9, + 25, + 41, + 57, + 10, + 26, + 42, + 58, + 11, + 27, + 43, + 59, + 12, + 28, + 44, + 60, + 13, + 29, + 45, + 61, + 14, + 30, + 46, + 62, + 15, + 31, + 47, + u8::MAX, + ]; + + assert_eq!(dest, expected); +} + +#[simd_test] +fn store_four_interleaved_i16x8(simd: S) { + let vectors = [ + i16x8::from_slice(simd, &[i16::MIN, -15, -14, -13, -12, -11, -10, -9]), + i16x8::from_slice(simd, &[-8, -7, -6, -5, -4, -3, -2, -1]), + i16x8::from_slice(simd, &[0, 1, 2, 3, 4, 5, 6, 7]), + i16x8::from_slice(simd, &[8, 9, 10, 11, 12, 13, 14, i16::MAX]), + ]; + let mut dest = [0_i16; 32]; + simd.store_four_interleaved_i16x8(vectors, &mut dest); + + let expected = [ + i16::MIN, + -8, + 0, + 8, + -15, + -7, + 1, + 9, + -14, + -6, + 2, + 10, + -13, + -5, + 3, + 11, + -12, + -4, + 4, + 12, + -11, + -3, + 5, + 13, + -10, + -2, + 6, + 14, + -9, + -1, + 7, + i16::MAX, + ]; + + assert_eq!(dest, expected); +} + +#[simd_test] +fn store_four_interleaved_u16x8(simd: S) { + let vectors = [ + u16x8::from_slice(simd, &[0, 1, 2, 3, 4, 5, 6, 7]), + u16x8::from_slice(simd, &[100, 101, 102, 103, 104, 105, 106, 107]), + u16x8::from_slice(simd, &[200, 201, 202, 203, 204, 205, 206, 207]), + u16x8::from_slice(simd, &[300, 301, 302, 303, 304, 305, 306, u16::MAX]), + ]; + let mut dest = [0_u16; 32]; + simd.store_four_interleaved_u16x8(vectors, &mut dest); + + let expected = [ + 0, + 100, + 200, + 300, + 1, + 101, + 201, + 301, + 2, + 102, + 202, + 302, + 3, + 103, + 203, + 303, + 4, + 104, + 204, + 304, + 5, + 105, + 205, + 305, + 6, + 106, + 206, + 306, + 7, + 107, + 207, + u16::MAX, + ]; + + assert_eq!(dest, expected); +} + +#[simd_test] +fn store_four_interleaved_i32x4(simd: S) { + let vectors = [ + i32x4::from_slice(simd, &[i32::MIN, -7, -6, -5]), + i32x4::from_slice(simd, &[-4, -3, -2, -1]), + i32x4::from_slice(simd, &[0, 1, 2, 3]), + i32x4::from_slice(simd, &[4, 5, 6, i32::MAX]), + ]; + let mut dest = [0_i32; 16]; + simd.store_four_interleaved_i32x4(vectors, &mut dest); + assert_eq!( + dest, + [ + i32::MIN, + -4, + 0, + 4, + -7, + -3, + 1, + 5, + -6, + -2, + 2, + 6, + -5, + -1, + 3, + i32::MAX + ] + ); +} + +#[simd_test] +fn store_four_interleaved_u32x4(simd: S) { + let vectors = [ + u32x4::from_slice(simd, &[0, 1, u32::MAX, 3]), + u32x4::from_slice(simd, &[1000, 1001, 1002, 1003]), + u32x4::from_slice(simd, &[2000, 2001, 2002, 2003]), + u32x4::from_slice(simd, &[u32::MIN, 3001, 3002, u32::MAX - 1]), + ]; + let mut dest = [0_u32; 16]; + simd.store_four_interleaved_u32x4(vectors, &mut dest); + assert_eq!( + dest, + [ + 0, + 1000, + 2000, + u32::MIN, + 1, + 1001, + 2001, + 3001, + u32::MAX, + 1002, + 2002, + 3002, + 3, + 1003, + 2003, + u32::MAX - 1, + ] + ); +} + +#[simd_test] +fn store_four_interleaved_i64x2(simd: S) { + let vectors = [ + i64x2::from_slice(simd, &[i64::MIN, -3]), + i64x2::from_slice(simd, &[-2, -1]), + i64x2::from_slice(simd, &[0, 1]), + i64x2::from_slice(simd, &[2, i64::MAX]), + ]; + let mut dest = [0_i64; 8]; + simd.store_four_interleaved_i64x2(vectors, &mut dest); + assert_eq!(dest, [i64::MIN, -2, 0, 2, -3, -1, 1, i64::MAX]); +} + +#[simd_test] +fn store_four_interleaved_u64x2(simd: S) { + let vectors = [ + u64x2::from_slice(simd, &[u64::MIN, 2]), + u64x2::from_slice(simd, &[3, 4]), + u64x2::from_slice(simd, &[5, 6]), + u64x2::from_slice(simd, &[7, u64::MAX]), + ]; + let mut dest = [0_u64; 8]; + simd.store_four_interleaved_u64x2(vectors, &mut dest); + assert_eq!(dest, [u64::MIN, 3, 5, 7, 2, 4, 6, u64::MAX]); +} diff --git a/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs b/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs deleted file mode 100644 index 5e106794..00000000 --- a/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2026 the Fearless_SIMD Authors -// SPDX-License-Identifier: Apache-2.0 OR MIT - -use fearless_simd::*; -use fearless_simd_dev_macros::simd_test; - -// One concrete test row per supported vector type. - -#[simd_test] -fn store_interleaved_128_f32x16(simd: S) { - let input = [ - 0.0, - f32::NAN, - f32::INFINITY, - -3.0, - 4.0, - -0.0, - 6.0, - f32::NEG_INFINITY, - 8.0, - 9.0, - -10.0, - 11.0, - f32::MIN, - 13.0, - f32::MAX, - 15.0, - ]; - let a = f32x16::from_slice(simd, &input); - let mut dest = [0.0_f32; 16]; - simd.store_interleaved_128_f32x16(a, &mut dest); - - let expected = [ - 0.0, - 4.0, - 8.0, - f32::MIN, - f32::NAN, - -0.0, - 9.0, - 13.0, - f32::INFINITY, - 6.0, - -10.0, - f32::MAX, - -3.0, - f32::NEG_INFINITY, - 11.0, - 15.0, - ]; - - // Note: f32::NAN != f32::NAN hence we compare the bit pattern. - assert_eq!(dest.map(f32::to_bits), expected.map(f32::to_bits)); -} - -#[simd_test] -fn store_interleaved_128_u8x64(simd: S) { - let input: [u8; 64] = [ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, - ]; - let a = u8x64::from_slice(simd, &input); - let mut dest = [0_u8; 64]; - simd.store_interleaved_128_u8x64(a, &mut dest); - - let expected = [ - 0, 16, 32, 48, 1, 17, 33, 49, 2, 18, 34, 50, 3, 19, 35, 51, 4, 20, 36, 52, 5, 21, 37, 53, - 6, 22, 38, 54, 7, 23, 39, 55, 8, 24, 40, 56, 9, 25, 41, 57, 10, 26, 42, 58, 11, 27, 43, 59, - 12, 28, 44, 60, 13, 29, 45, 61, 14, 30, 46, 62, 15, 31, 47, 63, - ]; - - assert_eq!(dest, expected); -} - -#[simd_test] -fn store_interleaved_128_u16x32(simd: S) { - let input: [u16; 32] = [ - 0, 1, 2, 3, 4, 5, 6, 7, 100, 101, 102, 103, 104, 105, 106, 107, 200, 201, 202, 203, 204, - 205, 206, 207, 300, 301, 302, 303, 304, 305, 306, 307, - ]; - let a = u16x32::from_slice(simd, &input); - let mut dest = [0_u16; 32]; - simd.store_interleaved_128_u16x32(a, &mut dest); - - let expected = [ - 0, 100, 200, 300, 1, 101, 201, 301, 2, 102, 202, 302, 3, 103, 203, 303, 4, 104, 204, 304, - 5, 105, 205, 305, 6, 106, 206, 306, 7, 107, 207, 307, - ]; - - assert_eq!(dest, expected); -} - -#[simd_test] -fn store_interleaved_128_u32x16(simd: S) { - let input: [u32; 16] = [ - 0, - 1, - u32::MAX, - 3, - 1000, - 1001, - 1002, - 1003, - 2000, - 2001, - 2002, - 2003, - u32::MIN, - 3001, - 3002, - u32::MAX - 1, - ]; - let a = u32x16::from_slice(simd, &input); - let mut dest = [0_u32; 16]; - simd.store_interleaved_128_u32x16(a, &mut dest); - - let expected = [ - 0, - 1000, - 2000, - u32::MIN, - 1, - 1001, - 2001, - 3001, - u32::MAX, - 1002, - 2002, - 3002, - 3, - 1003, - 2003, - u32::MAX - 1, - ]; - - assert_eq!(dest, expected); -} - -// Additional concrete rows for this operation. - -#[simd_test] -fn store_interleaved_128_u64x8(simd: S) { - let a = u64x8::from_slice( - simd, - &[1_u64, 2_u64, 3_u64, 4_u64, 5_u64, 6_u64, 7_u64, 8_u64], - ); - let mut out = [0_u64; 8]; - simd.store_interleaved_128_u64x8(a, &mut out); - assert_eq!( - out, - [1_u64, 3_u64, 5_u64, 7_u64, 2_u64, 4_u64, 6_u64, 8_u64] - ); -}