From c3a6ce6a9d189a8e0e4762f2efcbf0844ca9c450 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Mon, 3 Aug 2026 10:07:09 +0100 Subject: [PATCH 1/5] Support load/store_interleaved_128 on signed integer types --- fearless_simd/src/generated/avx2.rs | 293 ++++++++++++++++ fearless_simd/src/generated/avx512.rs | 131 ++++++++ fearless_simd/src/generated/fallback.rs | 183 ++++++++++ fearless_simd/src/generated/neon.rs | 32 ++ fearless_simd/src/generated/simd_trait.rs | 16 + fearless_simd/src/generated/sse2.rs | 275 +++++++++++++++ fearless_simd/src/generated/sse4_2.rs | 312 ++++++++++++++++++ fearless_simd/src/generated/wasm.rs | 200 +++++++++++ fearless_simd_gen/src/ops.rs | 12 +- .../tests/harness/ops/load_interleaved_128.rs | 79 +++++ .../harness/ops/store_interleaved_128.rs | 64 ++++ 11 files changed, 1595 insertions(+), 2 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index c0a8ec0a..40743992 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -12392,6 +12392,91 @@ impl Simd for Avx2 { self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) } #[inline(always)] + fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[i8; 64usize]) -> i8x64 { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + token.combine_i8x32( + token.combine_i8x16(out0.simd_into(token), out1.simd_into(token)), + token.combine_i8x16(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i8x64, dest: &mut [i8; 64usize]) -> () { + let (v01, v23) = token.split_i8x64(a); + let (v0, v1) = token.split_i8x32(v01); + let (v2, v3) = token.split_i8x32(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { let half = self.splat_u8x32(val); self.combine_u8x32(half, half) @@ -13596,6 +13681,91 @@ impl Simd for Avx2 { self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) } #[inline(always)] + fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[i16; 32usize]) -> i16x32 { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + token.combine_i16x16( + token.combine_i16x8(out0.simd_into(token), out1.simd_into(token)), + token.combine_i16x8(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i16x32, dest: &mut [i16; 32usize]) -> () { + let (v01, v23) = token.split_i16x32(a); + let (v0, v1) = token.split_i16x16(v01); + let (v2, v3) = token.split_i16x16(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { let half = self.splat_u16x16(val); self.combine_u16x16(half, half) @@ -14606,6 +14776,81 @@ impl Simd for Avx2 { self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] + fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[i32; 16usize]) -> i32x16 { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + token.combine_i32x8( + token.combine_i32x4(out0.simd_into(token), out1.simd_into(token)), + token.combine_i32x4(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i32x16, dest: &mut [i32; 16usize]) -> () { + let (v01, v23) = token.split_i32x16(a); + let (v0, v1) = token.split_i32x8(v01); + let (v2, v3) = token.split_i32x8(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { let (a0, a1) = self.split_i32x16(a); self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) @@ -15874,6 +16119,54 @@ impl Simd for Avx2 { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] + fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[i64; 8usize]) -> i64x8 { + let (chunks, []) = src.as_chunks::<4>() else { + unreachable!() + }; + let v0: __m256i = + crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[0]); + let v1: __m256i = + crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[1]); + let lo = _mm256_unpacklo_epi64(v0, v1); + let hi = _mm256_unpackhi_epi64(v0, v1); + let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); + let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); + token.combine_i64x4(out0.simd_into(token), out1.simd_into(token)) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x8, dest: &mut [i64; 8usize]) -> () { + let (v0, v1) = token.split_i64x8(a); + let v0: __m256i = v0.into(); + let v1: __m256i = v1.into(); + let lo = _mm256_permute2x128_si256::<0x20>(v0, v1); + let hi = _mm256_permute2x128_si256::<0x31>(v0, v1); + let out0 = _mm256_unpacklo_epi64(lo, hi); + let out1 = _mm256_unpackhi_epi64(lo, hi); + let (chunks, []) = dest.as_chunks_mut::<4>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m256i, [i64; 4]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m256i, [i64; 4]>( + out1, + &mut chunks[1], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index 85da4d56..11b4d587 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -13165,6 +13165,46 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[i8; 64usize]) -> i8x64 { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[i8; 64usize], __m512i>(src); + _mm512_permutexvar_epi8( + _mm512_set_epi8( + 63, 59, 55, 51, 47, 43, 39, 35, 31, 27, 23, 19, 15, 11, 7, 3, 62, 58, 54, + 50, 46, 42, 38, 34, 30, 26, 22, 18, 14, 10, 6, 2, 61, 57, 53, 49, 45, 41, + 37, 33, 29, 25, 21, 17, 13, 9, 5, 1, 60, 56, 52, 48, 44, 40, 36, 32, 28, + 24, 20, 16, 12, 8, 4, 0, + ), + lanes, + ) + .simd_into(token) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, dest: &mut [i8; 64usize]) -> () { + let lanes = _mm512_permutexvar_epi8( + _mm512_set_epi8( + 63, 47, 31, 15, 62, 46, 30, 14, 61, 45, 29, 13, 60, 44, 28, 12, 59, 43, 27, + 11, 58, 42, 26, 10, 57, 41, 25, 9, 56, 40, 24, 8, 55, 39, 23, 7, 54, 38, + 22, 6, 53, 37, 21, 5, 52, 36, 20, 4, 51, 35, 19, 3, 50, 34, 18, 2, 49, 33, + 17, 1, 48, 32, 16, 0, + ), + a.into(), + ); + crate::transmute::checked_transmute_store::<__m512i, [i8; 64usize]>(lanes, dest); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { crate::kernel!( #[inline(always)] @@ -14813,6 +14853,42 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[i16; 32usize]) -> i16x32 { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[i16; 32usize], __m512i>(src); + _mm512_permutexvar_epi16( + _mm512_set_epi16( + 31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2, 29, 25, 21, 17, + 13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0, + ), + lanes, + ) + .simd_into(token) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, dest: &mut [i16; 32usize]) -> () { + let lanes = _mm512_permutexvar_epi16( + _mm512_set_epi16( + 31, 23, 15, 7, 30, 22, 14, 6, 29, 21, 13, 5, 28, 20, 12, 4, 27, 19, 11, 3, + 26, 18, 10, 2, 25, 17, 9, 1, 24, 16, 8, 0, + ), + a.into(), + ); + crate::transmute::checked_transmute_store::<__m512i, [i16; 32usize]>(lanes, dest); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { crate::kernel!( #[inline(always)] @@ -16200,6 +16276,36 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[i32; 16usize]) -> i32x16 { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[i32; 16usize], __m512i>(src); + _mm512_permutexvar_epi32( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ) + .simd_into(token) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, dest: &mut [i32; 16usize]) -> () { + let lanes = _mm512_permutexvar_epi32( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + a.into(), + ); + crate::transmute::checked_transmute_store::<__m512i, [i32; 16usize]>(lanes, dest); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { crate::kernel!( #[inline(always)] @@ -18036,6 +18142,31 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[i64; 8usize]) -> i64x8 { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[i64; 8usize], __m512i>(src); + _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes) + .simd_into(token) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x8, dest: &mut [i64; 8usize]) -> () { + let lanes = + _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), a.into()); + crate::transmute::checked_transmute_store::<__m512i, [i64; 8usize]>(lanes, dest); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { crate::kernel!( #[inline(always)] diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index 8e799a5b..66ad3ed2 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -11033,6 +11033,91 @@ impl Simd for Fallback { self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) } #[inline(always)] + fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + src[32usize], + src[36usize], + src[40usize], + src[44usize], + src[48usize], + src[52usize], + src[56usize], + src[60usize], + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + src[33usize], + src[37usize], + src[41usize], + src[45usize], + src[49usize], + src[53usize], + src[57usize], + src[61usize], + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + src[34usize], + src[38usize], + src[42usize], + src[46usize], + src[50usize], + src[54usize], + src[58usize], + src[62usize], + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + src[35usize], + src[39usize], + src[43usize], + src[47usize], + src[51usize], + src[55usize], + src[59usize], + src[63usize], + ] + .simd_into(self) + } + #[inline(always)] + fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { + *dest = [ + a[0usize], a[16usize], a[32usize], a[48usize], a[1usize], a[17usize], a[33usize], + a[49usize], a[2usize], a[18usize], a[34usize], a[50usize], a[3usize], a[19usize], + a[35usize], a[51usize], a[4usize], a[20usize], a[36usize], a[52usize], a[5usize], + a[21usize], a[37usize], a[53usize], a[6usize], a[22usize], a[38usize], a[54usize], + a[7usize], a[23usize], a[39usize], a[55usize], a[8usize], a[24usize], a[40usize], + a[56usize], a[9usize], a[25usize], a[41usize], a[57usize], a[10usize], a[26usize], + a[42usize], a[58usize], a[11usize], a[27usize], a[43usize], a[59usize], a[12usize], + a[28usize], a[44usize], a[60usize], a[13usize], a[29usize], a[45usize], a[61usize], + a[14usize], a[30usize], a[46usize], a[62usize], a[15usize], a[31usize], a[47usize], + a[63usize], + ]; + } + #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { let half = self.splat_u8x32(val); self.combine_u8x32(half, half) @@ -12169,6 +12254,54 @@ impl Simd for Fallback { self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) } #[inline(always)] + fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + ] + .simd_into(self) + } + #[inline(always)] + fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { + *dest = [ + a[0usize], a[8usize], a[16usize], a[24usize], a[1usize], a[9usize], a[17usize], + a[25usize], a[2usize], a[10usize], a[18usize], a[26usize], a[3usize], a[11usize], + a[19usize], a[27usize], a[4usize], a[12usize], a[20usize], a[28usize], a[5usize], + a[13usize], a[21usize], a[29usize], a[6usize], a[14usize], a[22usize], a[30usize], + a[7usize], a[15usize], a[23usize], a[31usize], + ]; + } + #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { let half = self.splat_u16x16(val); self.combine_u16x16(half, half) @@ -13088,6 +13221,36 @@ impl Simd for Fallback { self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] + fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[3usize], + src[7usize], + src[11usize], + src[15usize], + ] + .simd_into(self) + } + #[inline(always)] + fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { + *dest = [ + a[0usize], a[4usize], a[8usize], a[12usize], a[1usize], a[5usize], a[9usize], + a[13usize], a[2usize], a[6usize], a[10usize], a[14usize], a[3usize], a[7usize], + a[11usize], a[15usize], + ]; + } + #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { let (a0, a1) = self.split_i32x16(a); self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) @@ -14243,6 +14406,26 @@ impl Simd for Fallback { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] + fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { + [ + src[0usize], + src[4usize], + src[1usize], + src[5usize], + src[2usize], + src[6usize], + src[3usize], + src[7usize], + ] + .simd_into(self) + } + #[inline(always)] + fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { + *dest = [ + a[0usize], a[2usize], a[4usize], a[6usize], a[1usize], a[3usize], a[5usize], a[7usize], + ]; + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index 26f3d86b..477ff8a9 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -10475,6 +10475,14 @@ impl Simd for Neon { self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) } #[inline(always)] + fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { + unsafe { vld4q_s8(src.as_ptr()).simd_into(self) } + } + #[inline(always)] + fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { + unsafe { vst4q_s8(dest.as_mut_ptr(), a.into()) } + } + #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { let half = self.splat_u8x32(val); self.combine_u8x32(half, half) @@ -11645,6 +11653,14 @@ impl Simd for Neon { self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) } #[inline(always)] + fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { + unsafe { vld4q_s16(src.as_ptr()).simd_into(self) } + } + #[inline(always)] + fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { + unsafe { vst4q_s16(dest.as_mut_ptr(), a.into()) } + } + #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { let half = self.splat_u16x16(val); self.combine_u16x16(half, half) @@ -12635,6 +12651,14 @@ impl Simd for Neon { self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] + fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { + unsafe { vld4q_s32(src.as_ptr()).simd_into(self) } + } + #[inline(always)] + fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { + unsafe { vst4q_s32(dest.as_mut_ptr(), a.into()) } + } + #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { let (a0, a1) = self.split_i32x16(a); self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) @@ -13932,6 +13956,14 @@ impl Simd for Neon { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] + fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { + unsafe { vld4q_s64(src.as_ptr()).simd_into(self) } + } + #[inline(always)] + fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { + unsafe { vst4q_s64(dest.as_mut_ptr(), a.into()) } + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index a75874eb..b06d566a 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -2667,6 +2667,10 @@ pub trait Simd: fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32); #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i8x64(self, a: i8x64) -> i8x64; + #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] + fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64; + #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] + fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u8x64(self, val: u8) -> u8x64; #[doc = "Create a SIMD vector from an array of the same length."] @@ -2900,6 +2904,10 @@ pub trait Simd: fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16); #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i16x32(self, a: i16x32) -> i16x32; + #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] + fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32; + #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] + fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u16x32(self, val: u16) -> u16x32; #[doc = "Create a SIMD vector from an array of the same length."] @@ -3139,6 +3147,10 @@ pub trait Simd: fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8); #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i32x16(self, a: i32x16) -> i32x16; + #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] + fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16; + #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] + fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> (); #[doc = "Convert each signed 32-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16; #[doc = "Create a SIMD vector with all elements set to the given value."] @@ -3482,6 +3494,10 @@ pub trait Simd: fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4); #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i64x8(self, a: i64x8) -> i64x8; + #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] + fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8; + #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] + fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u64x8(self, val: u64) -> u64x8; #[doc = "Create a SIMD vector from an array of the same length."] diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index 477dbe18..f5eb41ea 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -11039,6 +11039,91 @@ impl Simd for Sse2 { self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) } #[inline(always)] + fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + src[32usize], + src[36usize], + src[40usize], + src[44usize], + src[48usize], + src[52usize], + src[56usize], + src[60usize], + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + src[33usize], + src[37usize], + src[41usize], + src[45usize], + src[49usize], + src[53usize], + src[57usize], + src[61usize], + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + src[34usize], + src[38usize], + src[42usize], + src[46usize], + src[50usize], + src[54usize], + src[58usize], + src[62usize], + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + src[35usize], + src[39usize], + src[43usize], + src[47usize], + src[51usize], + src[55usize], + src[59usize], + src[63usize], + ] + .simd_into(self) + } + #[inline(always)] + fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { + *dest = [ + a[0usize], a[16usize], a[32usize], a[48usize], a[1usize], a[17usize], a[33usize], + a[49usize], a[2usize], a[18usize], a[34usize], a[50usize], a[3usize], a[19usize], + a[35usize], a[51usize], a[4usize], a[20usize], a[36usize], a[52usize], a[5usize], + a[21usize], a[37usize], a[53usize], a[6usize], a[22usize], a[38usize], a[54usize], + a[7usize], a[23usize], a[39usize], a[55usize], a[8usize], a[24usize], a[40usize], + a[56usize], a[9usize], a[25usize], a[41usize], a[57usize], a[10usize], a[26usize], + a[42usize], a[58usize], a[11usize], a[27usize], a[43usize], a[59usize], a[12usize], + a[28usize], a[44usize], a[60usize], a[13usize], a[29usize], a[45usize], a[61usize], + a[14usize], a[30usize], a[46usize], a[62usize], a[15usize], a[31usize], a[47usize], + a[63usize], + ]; + } + #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { let half = self.splat_u8x32(val); self.combine_u8x32(half, half) @@ -12208,6 +12293,54 @@ impl Simd for Sse2 { self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) } #[inline(always)] + fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + ] + .simd_into(self) + } + #[inline(always)] + fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { + *dest = [ + a[0usize], a[8usize], a[16usize], a[24usize], a[1usize], a[9usize], a[17usize], + a[25usize], a[2usize], a[10usize], a[18usize], a[26usize], a[3usize], a[11usize], + a[19usize], a[27usize], a[4usize], a[12usize], a[20usize], a[28usize], a[5usize], + a[13usize], a[21usize], a[29usize], a[6usize], a[14usize], a[22usize], a[30usize], + a[7usize], a[15usize], a[23usize], a[31usize], + ]; + } + #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { let half = self.splat_u16x16(val); self.combine_u16x16(half, half) @@ -13169,6 +13302,81 @@ impl Simd for Sse2 { self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] + fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[i32; 16usize]) -> i32x16 { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + token.combine_i32x8( + token.combine_i32x4(out0.simd_into(token), out1.simd_into(token)), + token.combine_i32x4(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, a: i32x16, dest: &mut [i32; 16usize]) -> () { + let (v01, v23) = token.split_i32x16(a); + let (v0, v1) = token.split_i32x8(v01); + let (v2, v3) = token.split_i32x8(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { let (a0, a1) = self.split_i32x16(a); self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) @@ -14416,6 +14624,73 @@ impl Simd for Sse2 { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] + fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[i64; 8usize]) -> i64x8 { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[3]); + let out0 = _mm_unpacklo_epi64(v0, v2); + let out1 = _mm_unpackhi_epi64(v0, v2); + let out2 = _mm_unpacklo_epi64(v1, v3); + let out3 = _mm_unpackhi_epi64(v1, v3); + token.combine_i64x4( + token.combine_i64x2(out0.simd_into(token), out1.simd_into(token)), + token.combine_i64x2(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, a: i64x8, dest: &mut [i64; 8usize]) -> () { + let (v01, v23) = token.split_i64x8(a); + let (v0, v1) = token.split_i64x4(v01); + let (v2, v3) = token.split_i64x4(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let out0 = _mm_unpacklo_epi64(v0, v1); + let out1 = _mm_unpacklo_epi64(v2, v3); + let out2 = _mm_unpackhi_epi64(v0, v1); + let out3 = _mm_unpackhi_epi64(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index db3b01f4..74cbd891 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -10251,6 +10251,91 @@ impl Simd for Sse4_2 { self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) } #[inline(always)] + fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[i8; 64usize]) -> i8x64 { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + token.combine_i8x32( + token.combine_i8x16(out0.simd_into(token), out1.simd_into(token)), + token.combine_i8x16(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i8x64, dest: &mut [i8; 64usize]) -> () { + let (v01, v23) = token.split_i8x64(a); + let (v0, v1) = token.split_i8x32(v01); + let (v2, v3) = token.split_i8x32(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { let half = self.splat_u8x32(val); self.combine_u8x32(half, half) @@ -11461,6 +11546,91 @@ impl Simd for Sse4_2 { self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) } #[inline(always)] + fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[i16; 32usize]) -> i16x32 { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + token.combine_i16x16( + token.combine_i16x8(out0.simd_into(token), out1.simd_into(token)), + token.combine_i16x8(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i16x32, dest: &mut [i16; 32usize]) -> () { + let (v01, v23) = token.split_i16x32(a); + let (v0, v1) = token.split_i16x16(v01); + let (v2, v3) = token.split_i16x16(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { let half = self.splat_u16x16(val); self.combine_u16x16(half, half) @@ -12459,6 +12629,81 @@ impl Simd for Sse4_2 { self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] + fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[i32; 16usize]) -> i32x16 { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + token.combine_i32x8( + token.combine_i32x4(out0.simd_into(token), out1.simd_into(token)), + token.combine_i32x4(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i32x16, dest: &mut [i32; 16usize]) -> () { + let (v01, v23) = token.split_i32x16(a); + let (v0, v1) = token.split_i32x8(v01); + let (v2, v3) = token.split_i32x8(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { let (a0, a1) = self.split_i32x16(a); self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) @@ -13706,6 +13951,73 @@ impl Simd for Sse4_2 { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] + fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[i64; 8usize]) -> i64x8 { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[3]); + let out0 = _mm_unpacklo_epi64(v0, v2); + let out1 = _mm_unpackhi_epi64(v0, v2); + let out2 = _mm_unpacklo_epi64(v1, v3); + let out3 = _mm_unpackhi_epi64(v1, v3); + token.combine_i64x4( + token.combine_i64x2(out0.simd_into(token), out1.simd_into(token)), + token.combine_i64x2(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i64x8, dest: &mut [i64; 8usize]) -> () { + let (v01, v23) = token.split_i64x8(a); + let (v0, v1) = token.split_i64x4(v01); + let (v2, v3) = token.split_i64x4(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let out0 = _mm_unpacklo_epi64(v0, v1); + let out1 = _mm_unpacklo_epi64(v2, v3); + let out2 = _mm_unpackhi_epi64(v0, v1); + let out3 = _mm_unpackhi_epi64(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index 19474924..de23ad08 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -8430,6 +8430,76 @@ impl Simd for WasmSimd128 { self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) } #[inline(always)] + fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[3]); + let v01_lower = + u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v0, v1); + let v23_lower = + u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v2, v3); + let v01_upper = + u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v0, v1); + let v23_upper = + u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v2, v3); + let out0 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( + v01_lower, v23_lower, + ); + let out1 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( + v01_lower, v23_lower, + ); + let out2 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( + v01_upper, v23_upper, + ); + let out3 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( + v01_upper, v23_upper, + ); + let combined_lower = self.combine_i8x16(out0.simd_into(self), out1.simd_into(self)); + let combined_upper = self.combine_i8x16(out2.simd_into(self), out3.simd_into(self)); + self.combine_i8x32(combined_lower, combined_upper) + } + #[inline(always)] + fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { + let (lower, upper) = self.split_i8x64(a); + let (v0_vec, v1_vec) = self.split_i8x32(lower); + let (v2_vec, v3_vec) = self.split_i8x32(upper); + let v0: v128 = v0_vec.into(); + let v1: v128 = v1_vec.into(); + let v2: v128 = v2_vec.into(); + let v3: v128 = v3_vec.into(); + let v02_lower = + u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v0, v2); + let v13_lower = + u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v1, v3); + let v02_upper = + u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v0, v2); + let v13_upper = + u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v1, v3); + let out0 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( + v02_lower, v13_lower, + ); + let out1 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( + v02_lower, v13_lower, + ); + let out2 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( + v02_upper, v13_upper, + ); + let out3 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( + v02_upper, v13_upper, + ); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { let half = self.splat_u8x32(val); self.combine_u8x32(half, half) @@ -9579,6 +9649,52 @@ impl Simd for WasmSimd128 { self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) } #[inline(always)] + fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[3]); + let v01_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v0, v1); + let v23_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v2, v3); + let v01_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v0, v1); + let v23_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v2, v3); + let out0 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_lower, v23_lower); + let out1 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_lower, v23_lower); + let out2 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_upper, v23_upper); + let out3 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_upper, v23_upper); + let combined_lower = self.combine_i16x8(out0.simd_into(self), out1.simd_into(self)); + let combined_upper = self.combine_i16x8(out2.simd_into(self), out3.simd_into(self)); + self.combine_i16x16(combined_lower, combined_upper) + } + #[inline(always)] + fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { + let (lower, upper) = self.split_i16x32(a); + let (v0_vec, v1_vec) = self.split_i16x16(lower); + let (v2_vec, v3_vec) = self.split_i16x16(upper); + let v0: v128 = v0_vec.into(); + let v1: v128 = v1_vec.into(); + let v2: v128 = v2_vec.into(); + let v3: v128 = v3_vec.into(); + let v02_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v0, v2); + let v13_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v1, v3); + let v02_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v0, v2); + let v13_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v1, v3); + let out0 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_lower, v13_lower); + let out1 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_lower, v13_lower); + let out2 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_upper, v13_upper); + let out3 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { let half = self.splat_u16x16(val); self.combine_u16x16(half, half) @@ -10527,6 +10643,52 @@ impl Simd for WasmSimd128 { self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] + fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[3]); + let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); + let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); + let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); + let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); + let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); + let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); + let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); + let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); + let combined_lower = self.combine_i32x4(out0.simd_into(self), out1.simd_into(self)); + let combined_upper = self.combine_i32x4(out2.simd_into(self), out3.simd_into(self)); + self.combine_i32x8(combined_lower, combined_upper) + } + #[inline(always)] + fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { + let (lower, upper) = self.split_i32x16(a); + let (v0_vec, v1_vec) = self.split_i32x8(lower); + let (v2_vec, v3_vec) = self.split_i32x8(upper); + let v0: v128 = v0_vec.into(); + let v1: v128 = v1_vec.into(); + let v2: v128 = v2_vec.into(); + let v3: v128 = v3_vec.into(); + let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); + let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); + let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); + let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); + let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); + let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); + let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); + let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { let (a0, a1) = self.split_i32x16(a); self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) @@ -11742,6 +11904,44 @@ impl Simd for WasmSimd128 { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] + fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[3]); + let out0 = u64x2_shuffle::<0, 2>(v0, v2); + let out1 = u64x2_shuffle::<1, 3>(v0, v2); + let out2 = u64x2_shuffle::<0, 2>(v1, v3); + let out3 = u64x2_shuffle::<1, 3>(v1, v3); + let combined_lower = self.combine_i64x2(out0.simd_into(self), out1.simd_into(self)); + let combined_upper = self.combine_i64x2(out2.simd_into(self), out3.simd_into(self)); + self.combine_i64x4(combined_lower, combined_upper) + } + #[inline(always)] + fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { + let (lower, upper) = self.split_i64x8(a); + let (v0_vec, v1_vec) = self.split_i64x4(lower); + let (v2_vec, v3_vec) = self.split_i64x4(upper); + let v0: v128 = v0_vec.into(); + let v1: v128 = v1_vec.into(); + let v2: v128 = v2_vec.into(); + let v3: v128 = v3_vec.into(); + let out0 = u64x2_shuffle::<0, 2>(v0, v1); + let out1 = u64x2_shuffle::<0, 2>(v2, v3); + let out2 = u64x2_shuffle::<1, 3>(v0, v1); + let out3 = u64x2_shuffle::<1, 3>(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index c04c87f1..ef8c7d73 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -1366,7 +1366,11 @@ pub(crate) fn ops_for_type(ty: &VecType) -> Vec { return ops; } - if matches!(ty.scalar, ScalarType::Unsigned | ScalarType::Float) && ty.n_bits() == 512 { + if matches!( + ty.scalar, + ScalarType::Int | ScalarType::Unsigned | ScalarType::Float + ) && ty.n_bits() == 512 + { ops.push(Op::new( "load_interleaved_128", OpKind::AssociatedOnly, @@ -1384,7 +1388,11 @@ pub(crate) fn ops_for_type(ty: &VecType) -> Vec { )); } - if matches!(ty.scalar, ScalarType::Unsigned | ScalarType::Float) && ty.n_bits() == 512 { + if matches!( + ty.scalar, + ScalarType::Int | ScalarType::Unsigned | ScalarType::Float + ) && ty.n_bits() == 512 + { ops.push(Op::new( "store_interleaved_128", OpKind::AssociatedOnly, diff --git a/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs b/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs index 5dea1ba2..509766df 100644 --- a/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs +++ b/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs @@ -50,6 +50,21 @@ fn load_interleaved_128_f32x16(simd: S) { assert_eq!((*result).map(f32::to_bits), expected.map(f32::to_bits),); } +#[simd_test] +fn load_interleaved_128_i32x16(simd: S) { + #[rustfmt::skip] + let data: [i32; 16] = [ + -8, -7, -6, -5, + -4, -3, -2, -1, + 0, 1, 2, 3, + 4, 5, 6, 7, + ]; + assert_eq!( + *simd.load_interleaved_128_i32x16(&data), + [-8, -4, 0, 4, -7, -3, 1, 5, -6, -2, 2, 6, -5, -1, 3, 7] + ); +} + #[simd_test] fn load_interleaved_128_u32x16(simd: S) { #[rustfmt::skip] @@ -92,6 +107,28 @@ fn load_interleaved_128_u16x32(simd: S) { ); } +#[simd_test] +fn load_interleaved_128_i16x32(simd: S) { + #[rustfmt::skip] + let data: [i16; 32] = [ + -16, -15, -14, -13, + -12, -11, -10, -9, + -8, -7, -6, -5, + -4, -3, -2, -1, + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15, + ]; + assert_eq!( + *simd.load_interleaved_128_i16x32(&data), + [ + -16, -12, -8, -4, 0, 4, 8, 12, -15, -11, -7, -3, 1, 5, 9, 13, -14, -10, -6, -2, 2, 6, + 10, 14, -13, -9, -5, -1, 3, 7, 11, 15 + ] + ); +} + #[simd_test] fn load_interleaved_128_u8x64(simd: S) { #[rustfmt::skip] @@ -126,6 +163,38 @@ fn load_interleaved_128_u8x64(simd: S) { ); } +#[simd_test] +fn load_interleaved_128_i8x64(simd: S) { + #[rustfmt::skip] + let data: [i8; 64] = [ + -32, -31, -30, -29, + -28, -27, -26, -25, + -24, -23, -22, -21, + -20, -19, -18, -17, + -16, -15, -14, -13, + -12, -11, -10, -9, + -8, -7, -6, -5, + -4, -3, -2, -1, + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15, + 16, 17, 18, 19, + 20, 21, 22, 23, + 24, 25, 26, 27, + 28, 29, 30, 31, + ]; + assert_eq!( + *simd.load_interleaved_128_i8x64(&data), + [ + -32, -28, -24, -20, -16, -12, -8, -4, 0, 4, 8, 12, 16, 20, 24, 28, -31, -27, -23, -19, + -15, -11, -7, -3, 1, 5, 9, 13, 17, 21, 25, 29, -30, -26, -22, -18, -14, -10, -6, -2, 2, + 6, 10, 14, 18, 22, 26, 30, -29, -25, -21, -17, -13, -9, -5, -1, 3, 7, 11, 15, 19, 23, + 27, 31 + ] + ); +} + // Additional concrete rows for this operation. #[simd_test] @@ -134,3 +203,13 @@ fn load_interleaved_128_u64x8(simd: S) { let a = simd.load_interleaved_128_u64x8(&data); assert_eq!(*a, [1_u64, 2_u64, 3_u64, 4_u64, 5_u64, 6_u64, 7_u64, 8_u64]); } + +#[simd_test] +fn load_interleaved_128_i64x8(simd: S) { + let data = [-4_i64, -3_i64, -2_i64, -1_i64, 0_i64, 1_i64, 2_i64, 3_i64]; + let a = simd.load_interleaved_128_i64x8(&data); + assert_eq!( + *a, + [-4_i64, 0_i64, -3_i64, 1_i64, -2_i64, 2_i64, -1_i64, 3_i64] + ); +} diff --git a/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs b/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs index 5e106794..dd08466c 100644 --- a/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs +++ b/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs @@ -53,6 +53,26 @@ fn store_interleaved_128_f32x16(simd: S) { assert_eq!(dest.map(f32::to_bits), expected.map(f32::to_bits)); } +#[simd_test] +fn store_interleaved_128_i8x64(simd: S) { + let input: [i8; 64] = [ + -32, -31, -30, -29, -28, -27, -26, -25, -24, -23, -22, -21, -20, -19, -18, -17, -16, -15, + -14, -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ]; + let a = i8x64::from_slice(simd, &input); + let mut dest = [0_i8; 64]; + simd.store_interleaved_128_i8x64(a, &mut dest); + + let expected = [ + -32, -16, 0, 16, -31, -15, 1, 17, -30, -14, 2, 18, -29, -13, 3, 19, -28, -12, 4, 20, -27, + -11, 5, 21, -26, -10, 6, 22, -25, -9, 7, 23, -24, -8, 8, 24, -23, -7, 9, 25, -22, -6, 10, + 26, -21, -5, 11, 27, -20, -4, 12, 28, -19, -3, 13, 29, -18, -2, 14, 30, -17, -1, 15, 31, + ]; + + assert_eq!(dest, expected); +} + #[simd_test] fn store_interleaved_128_u8x64(simd: S) { let input: [u8; 64] = [ @@ -73,6 +93,24 @@ fn store_interleaved_128_u8x64(simd: S) { assert_eq!(dest, expected); } +#[simd_test] +fn store_interleaved_128_i16x32(simd: S) { + let input: [i16; 32] = [ + -16, -15, -14, -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, + ]; + let a = i16x32::from_slice(simd, &input); + let mut dest = [0_i16; 32]; + simd.store_interleaved_128_i16x32(a, &mut dest); + + let expected = [ + -16, -8, 0, 8, -15, -7, 1, 9, -14, -6, 2, 10, -13, -5, 3, 11, -12, -4, 4, 12, -11, -3, 5, + 13, -10, -2, 6, 14, -9, -1, 7, 15, + ]; + + assert_eq!(dest, expected); +} + #[simd_test] fn store_interleaved_128_u16x32(simd: S) { let input: [u16; 32] = [ @@ -91,6 +129,18 @@ fn store_interleaved_128_u16x32(simd: S) { assert_eq!(dest, expected); } +#[simd_test] +fn store_interleaved_128_i32x16(simd: S) { + let input: [i32; 16] = [-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7]; + let a = i32x16::from_slice(simd, &input); + let mut dest = [0_i32; 16]; + simd.store_interleaved_128_i32x16(a, &mut dest); + + let expected = [-8, -4, 0, 4, -7, -3, 1, 5, -6, -2, 2, 6, -5, -1, 3, 7]; + + assert_eq!(dest, expected); +} + #[simd_test] fn store_interleaved_128_u32x16(simd: S) { let input: [u32; 16] = [ @@ -152,3 +202,17 @@ fn store_interleaved_128_u64x8(simd: S) { [1_u64, 3_u64, 5_u64, 7_u64, 2_u64, 4_u64, 6_u64, 8_u64] ); } + +#[simd_test] +fn store_interleaved_128_i64x8(simd: S) { + let a = i64x8::from_slice( + simd, + &[-4_i64, -3_i64, -2_i64, -1_i64, 0_i64, 1_i64, 2_i64, 3_i64], + ); + let mut out = [0_i64; 8]; + simd.store_interleaved_128_i64x8(a, &mut out); + assert_eq!( + out, + [-4_i64, -2_i64, 0_i64, 2_i64, -3_i64, -1_i64, 1_i64, 3_i64] + ); +} From 174963d0ff15655d71d6598786aa3c714418201d Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Mon, 3 Aug 2026 10:12:39 +0100 Subject: [PATCH 2/5] Support load/store_interleaved_128 on f64 --- fearless_simd/src/generated/avx2.rs | 48 +++++++++++++ fearless_simd/src/generated/avx512.rs | 25 +++++++ fearless_simd/src/generated/fallback.rs | 20 ++++++ fearless_simd/src/generated/neon.rs | 8 +++ fearless_simd/src/generated/simd_trait.rs | 4 ++ fearless_simd/src/generated/sse2.rs | 67 +++++++++++++++++++ fearless_simd/src/generated/sse4_2.rs | 67 +++++++++++++++++++ fearless_simd/src/generated/wasm.rs | 38 +++++++++++ fearless_simd_gen/src/mk_x86.rs | 36 +++++++--- fearless_simd_gen/src/ops.rs | 8 +-- .../tests/harness/ops/load_interleaved_128.rs | 28 ++++++++ .../harness/ops/store_interleaved_128.rs | 31 +++++++++ 12 files changed, 366 insertions(+), 14 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 40743992..99c5b3cb 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -15798,6 +15798,54 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[f64; 8usize]) -> f64x8 { + let (chunks, []) = src.as_chunks::<4>() else { + unreachable!() + }; + let v0: __m256d = + crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[0]); + let v1: __m256d = + crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[1]); + let lo = _mm256_unpacklo_pd(v0, v1); + let hi = _mm256_unpackhi_pd(v0, v1); + let out0 = _mm256_permute2f128_pd::<0x20>(lo, hi); + let out1 = _mm256_permute2f128_pd::<0x31>(lo, hi); + token.combine_f64x4(out0.simd_into(token), out1.simd_into(token)) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x8, dest: &mut [f64; 8usize]) -> () { + let (v0, v1) = token.split_f64x8(a); + let v0: __m256d = v0.into(); + let v1: __m256d = v1.into(); + let lo = _mm256_permute2f128_pd::<0x20>(v0, v1); + let hi = _mm256_permute2f128_pd::<0x31>(v0, v1); + let out0 = _mm256_unpacklo_pd(lo, hi); + let out1 = _mm256_unpackhi_pd(lo, hi); + let (chunks, []) = dest.as_chunks_mut::<4>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m256d, [f64; 4]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m256d, [f64; 4]>( + out1, + &mut chunks[1], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index 11b4d587..db53767f 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -17636,6 +17636,31 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[f64; 8usize]) -> f64x8 { + let lanes: __m512d = + crate::transmute::checked_transmute_copy::<[f64; 8usize], __m512d>(src); + _mm512_permutexvar_pd(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes) + .simd_into(token) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, dest: &mut [f64; 8usize]) -> () { + let lanes = + _mm512_permutexvar_pd(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), a.into()); + crate::transmute::checked_transmute_store::<__m512d, [f64; 8usize]>(lanes, dest); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { crate::kernel!( #[inline(always)] diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index 66ad3ed2..69b5967b 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -14099,6 +14099,26 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] + fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { + [ + src[0usize], + src[4usize], + src[1usize], + src[5usize], + src[2usize], + src[6usize], + src[3usize], + src[7usize], + ] + .simd_into(self) + } + #[inline(always)] + fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { + *dest = [ + a[0usize], a[2usize], a[4usize], a[6usize], a[1usize], a[3usize], a[5usize], a[7usize], + ]; + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index 477ff8a9..3e646ae2 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -13596,6 +13596,14 @@ impl Simd for Neon { ) } #[inline(always)] + fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { + unsafe { vld4q_f64(src.as_ptr()).simd_into(self) } + } + #[inline(always)] + fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { + unsafe { vst4q_f64(dest.as_mut_ptr(), a.into()) } + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index b06d566a..b450a482 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -3400,6 +3400,10 @@ pub trait Simd: fn select_f64x8(self, a: mask64x8, b: f64x8, c: f64x8) -> f64x8; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_f64x8(self, a: f64x8) -> (f64x4, f64x4); + #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] + fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8; + #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] + fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_i64x8(self, val: i64) -> i64x8; #[doc = "Create a SIMD vector from an array of the same length."] diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index f5eb41ea..4f1c47e8 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -14303,6 +14303,73 @@ impl Simd for Sse2 { ) } #[inline(always)] + fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[f64; 8usize]) -> f64x8 { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[0]); + let v1: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[1]); + let v2: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[2]); + let v3: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[3]); + let out0 = _mm_unpacklo_pd(v0, v2); + let out1 = _mm_unpackhi_pd(v0, v2); + let out2 = _mm_unpacklo_pd(v1, v3); + let out3 = _mm_unpackhi_pd(v1, v3); + token.combine_f64x4( + token.combine_f64x2(out0.simd_into(token), out1.simd_into(token)), + token.combine_f64x2(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, a: f64x8, dest: &mut [f64; 8usize]) -> () { + let (v01, v23) = token.split_f64x8(a); + let (v0, v1) = token.split_f64x4(v01); + let (v2, v3) = token.split_f64x4(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let out0 = _mm_unpacklo_pd(v0, v1); + let out1 = _mm_unpacklo_pd(v2, v3); + let out2 = _mm_unpackhi_pd(v0, v1); + let out3 = _mm_unpackhi_pd(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 74cbd891..69581ab2 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -13630,6 +13630,73 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[f64; 8usize]) -> f64x8 { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[0]); + let v1: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[1]); + let v2: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[2]); + let v3: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[3]); + let out0 = _mm_unpacklo_pd(v0, v2); + let out1 = _mm_unpackhi_pd(v0, v2); + let out2 = _mm_unpacklo_pd(v1, v3); + let out3 = _mm_unpackhi_pd(v1, v3); + token.combine_f64x4( + token.combine_f64x2(out0.simd_into(token), out1.simd_into(token)), + token.combine_f64x2(out2.simd_into(token), out3.simd_into(token)), + ) + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f64x8, dest: &mut [f64; 8usize]) -> () { + let (v01, v23) = token.split_f64x8(a); + let (v0, v1) = token.split_f64x4(v01); + let (v2, v3) = token.split_f64x4(v23); + let v0 = v0.into(); + let v1 = v1.into(); + let v2 = v2.into(); + let v3 = v3.into(); + let out0 = _mm_unpacklo_pd(v0, v1); + let out1 = _mm_unpacklo_pd(v2, v3); + let out2 = _mm_unpackhi_pd(v0, v1); + let out3 = _mm_unpackhi_pd(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, a, dest); + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index de23ad08..55ddfa6f 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -11584,6 +11584,44 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[3]); + let out0 = u64x2_shuffle::<0, 2>(v0, v2); + let out1 = u64x2_shuffle::<1, 3>(v0, v2); + let out2 = u64x2_shuffle::<0, 2>(v1, v3); + let out3 = u64x2_shuffle::<1, 3>(v1, v3); + let combined_lower = self.combine_f64x2(out0.simd_into(self), out1.simd_into(self)); + let combined_upper = self.combine_f64x2(out2.simd_into(self), out3.simd_into(self)); + self.combine_f64x4(combined_lower, combined_upper) + } + #[inline(always)] + fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { + let (lower, upper) = self.split_f64x8(a); + let (v0_vec, v1_vec) = self.split_f64x4(lower); + let (v2_vec, v3_vec) = self.split_f64x4(upper); + let v0: v128 = v0_vec.into(); + let v1: v128 = v1_vec.into(); + let v2: v128 = v2_vec.into(); + let v3: v128 = v3_vec.into(); + let out0 = u64x2_shuffle::<0, 2>(v0, v1); + let out1 = u64x2_shuffle::<0, 2>(v2, v3); + let out2 = u64x2_shuffle::<1, 3>(v0, v1); + let out3 = u64x2_shuffle::<1, 3>(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index ebc064f0..857d7fc1 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -3201,8 +3201,8 @@ impl X86 { } match vec_ty.scalar_bits { 64 | 32 | 16 | 8 => { - let avx2_u64 = *self == Self::Avx2 && vec_ty.scalar_bits == 64; - let block_len = if avx2_u64 { + let avx2_64 = *self == Self::Avx2 && vec_ty.scalar_bits == 64; + let block_len = if avx2_64 { 4 } else { block_size as usize / vec_ty.scalar_bits @@ -3216,6 +3216,14 @@ impl X86 { let vec_64 = block_ty.reinterpret(block_ty.scalar, 64); let unpacklo_64 = simple_sign_unaware_intrinsic("unpacklo", &vec_64); let unpackhi_64 = simple_sign_unaware_intrinsic("unpackhi", &vec_64); + let permute_128 = intrinsic_ident( + match vec_ty.scalar { + ScalarType::Float => "permute2f128", + _ => "permute2x128", + }, + coarse_type(&block_ty), + 256, + ); let vec_combined = VecType::new(block_ty.scalar, block_ty.scalar_bits, block_ty.len * 2); @@ -3227,7 +3235,7 @@ impl X86 { &format!("combine_{}", vec_combined.rust_name()), Span::call_site(), ); - if avx2_u64 { + if avx2_64 { return self.kernel_method(op, vec_ty, |token| { quote! { let (chunks, []) = src.as_chunks::<4>() else { @@ -3238,8 +3246,8 @@ impl X86 { let lo = #unpacklo_64(v0, v1); // [0,4,2,6] let hi = #unpackhi_64(v0, v1); // [1,5,3,7] - let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); // [0,4,1,5] - let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); // [2,6,3,7] + let out0 = #permute_128::<0x20>(lo, hi); // [0,4,1,5] + let out1 = #permute_128::<0x31>(lo, hi); // [2,6,3,7] #token.#combine_half(out0.simd_into(#token), out1.simd_into(#token)) } @@ -3417,8 +3425,8 @@ impl X86 { } match vec_ty.scalar_bits { 64 | 32 | 16 | 8 => { - let avx2_u64 = *self == Self::Avx2 && vec_ty.scalar_bits == 64; - let block_len = if avx2_u64 { + let avx2_64 = *self == Self::Avx2 && vec_ty.scalar_bits == 64; + let block_len = if avx2_64 { 4 } else { block_size as usize / vec_ty.scalar_bits @@ -3432,6 +3440,14 @@ impl X86 { let vec_64 = block_ty.reinterpret(block_ty.scalar, 64); let unpacklo_64 = simple_sign_unaware_intrinsic("unpacklo", &vec_64); let unpackhi_64 = simple_sign_unaware_intrinsic("unpackhi", &vec_64); + let permute_128 = intrinsic_ident( + match vec_ty.scalar { + ScalarType::Float => "permute2f128", + _ => "permute2x128", + }, + coarse_type(&block_ty), + 256, + ); let vec_combined = VecType::new(block_ty.scalar, block_ty.scalar_bits, block_ty.len * 2); @@ -3445,15 +3461,15 @@ impl X86 { // For 64-bit values, AVX2 permits a more efficient implementation. // It is special-cased here as a full early return because plumbing it // through the rest of the logic in this function complicates it significantly. - if avx2_u64 { + if avx2_64 { return self.kernel_method(op, vec_ty, |token| { quote! { let (v0, v1) = #token.#split_full(a); let v0: #native_ty = v0.into(); let v1: #native_ty = v1.into(); - let lo = _mm256_permute2x128_si256::<0x20>(v0, v1); // [0,1,4,5] - let hi = _mm256_permute2x128_si256::<0x31>(v0, v1); // [2,3,6,7] + let lo = #permute_128::<0x20>(v0, v1); // [0,1,4,5] + let hi = #permute_128::<0x31>(v0, v1); // [2,3,6,7] let out0 = #unpacklo_64(lo, hi); // [0,2,4,6] let out1 = #unpackhi_64(lo, hi); // [1,3,5,7] diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index ef8c7d73..e968129b 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -1362,10 +1362,6 @@ pub(crate) fn ops_for_type(ty: &VecType) -> Vec { ops.push(NEGATE_INT); } - if ty.scalar == ScalarType::Float && ty.scalar_bits == 64 { - return ops; - } - if matches!( ty.scalar, ScalarType::Int | ScalarType::Unsigned | ScalarType::Float @@ -1410,6 +1406,10 @@ pub(crate) fn ops_for_type(ty: &VecType) -> Vec { )); } + if ty.scalar == ScalarType::Float && ty.scalar_bits == 64 { + return ops; + } + if matches!(ty.scalar, ScalarType::Unsigned) { if let Some(target_ty) = ty.widened() { ops.push(Op::new( diff --git a/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs b/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs index 509766df..523c4dae 100644 --- a/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs +++ b/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs @@ -50,6 +50,34 @@ fn load_interleaved_128_f32x16(simd: S) { assert_eq!((*result).map(f32::to_bits), expected.map(f32::to_bits),); } +#[simd_test] +fn load_interleaved_128_f64x8(simd: S) { + let data = [ + 0.0, + f64::NAN, + f64::INFINITY, + -3.0, + -0.0, + f64::MIN, + f64::NEG_INFINITY, + f64::MAX, + ]; + let expected = [ + 0.0, + -0.0, + f64::NAN, + f64::MIN, + f64::INFINITY, + f64::NEG_INFINITY, + -3.0, + f64::MAX, + ]; + + // Note: f64::NAN != f64::NAN hence we compare the bit pattern. + let result = simd.load_interleaved_128_f64x8(&data); + assert_eq!((*result).map(f64::to_bits), expected.map(f64::to_bits)); +} + #[simd_test] fn load_interleaved_128_i32x16(simd: S) { #[rustfmt::skip] diff --git a/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs b/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs index dd08466c..22e67a04 100644 --- a/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs +++ b/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs @@ -53,6 +53,37 @@ fn store_interleaved_128_f32x16(simd: S) { assert_eq!(dest.map(f32::to_bits), expected.map(f32::to_bits)); } +#[simd_test] +fn store_interleaved_128_f64x8(simd: S) { + let input = [ + 0.0, + -0.0, + f64::NAN, + f64::MIN, + f64::INFINITY, + f64::NEG_INFINITY, + -3.0, + f64::MAX, + ]; + let a = f64x8::from_slice(simd, &input); + let mut dest = [0.0_f64; 8]; + simd.store_interleaved_128_f64x8(a, &mut dest); + + let expected = [ + 0.0, + f64::NAN, + f64::INFINITY, + -3.0, + -0.0, + f64::MIN, + f64::NEG_INFINITY, + f64::MAX, + ]; + + // Note: f64::NAN != f64::NAN hence we compare the bit pattern. + assert_eq!(dest.map(f64::to_bits), expected.map(f64::to_bits)); +} + #[simd_test] fn store_interleaved_128_i8x64(simd: S) { let input: [i8; 64] = [ From 62ff2ce2e16ac22bf3840ad567b21e0de94a71c0 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Mon, 3 Aug 2026 11:42:41 +0100 Subject: [PATCH 3/5] Change load_interleaved_128 that returns a 512-bit vector to load_four_interleaved that returns 4 128-bit vectors. This maps to the real-world usage better (in Vello, all uses of this function save one go through manual split/combine first), eliminates pointless split/combine in x86 implementations, and most importantly allows us to add wrappers for vld3q later which would perform the same operation for RGB rather than RGBA. --- CHANGELOG.md | 2 + fearless_simd/src/generated/avx2.rs | 1679 ++++++++-------- fearless_simd/src/generated/avx512.rs | 857 ++++++--- fearless_simd/src/generated/fallback.rs | 1084 +++++++---- fearless_simd/src/generated/neon.rs | 340 +++- fearless_simd/src/generated/simd_trait.rs | 140 +- fearless_simd/src/generated/sse2.rs | 1678 +++++++++------- fearless_simd/src/generated/sse4_2.rs | 1712 +++++++++-------- fearless_simd/src/generated/wasm.rs | 1008 +++++----- fearless_simd_gen/src/generic.rs | 22 +- fearless_simd_gen/src/mk_fallback.rs | 57 +- fearless_simd_gen/src/mk_neon.rs | 61 +- fearless_simd_gen/src/mk_simd_trait.rs | 4 + fearless_simd_gen/src/mk_wasm.rs | 41 +- fearless_simd_gen/src/mk_x86.rs | 134 +- fearless_simd_gen/src/ops.rs | 57 +- .../harness/ops/load_four_interleaved.rs | 325 ++++ .../tests/harness/ops/load_interleaved_128.rs | 243 --- fearless_simd_tests/tests/harness/ops/mod.rs | 4 +- .../harness/ops/store_four_interleaved.rs | 500 +++++ .../harness/ops/store_interleaved_128.rs | 249 --- 21 files changed, 5866 insertions(+), 4331 deletions(-) create mode 100644 fearless_simd_tests/tests/harness/ops/load_four_interleaved.rs delete mode 100644 fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs create mode 100644 fearless_simd_tests/tests/harness/ops/store_four_interleaved.rs delete mode 100644 fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3308f9af..94ac384b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ You can find its changes [documented below](#060-2026-07-10). ### Changed +- Breaking change: the `load_interleaved_128_*` and `store_interleaved_128_*` methods, which exchanged one 512-bit vector, have been replaced by `load_four_interleaved_*` and `store_four_interleaved_*`. The new methods exchange an array of four 128-bit vectors directly and are available for all non-mask scalar types. + - The `new_unchecked()` function on SIMD level tokens such as `Avx2` has been renamed to `assume_supported()` and is now safe to call from contexts that already contain the appropriate `#[target_feature]` annotations. Functions without such annotations can still call `assume_supported()` with an `unsafe` block. ([#293][] by [@Shnatsel][]) - On x86_64 targets with static SSE2 support, `Level::baseline()` now returns `Sse2` instead of `Fallback`. ([#270][] by [@Shnatsel][]) - The `fxsr` CPU feature is now required for all x86 SIMD levels. It is present in hardware on all SIMD-capable CPUs, but it is possible to disable it in some emulators combined with a custom Rust target specification. ([#270][] by [@Shnatsel][]) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 99c5b3cb..3bc56971 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -569,6 +569,89 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[f32; 16usize]) -> [f32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); + let v1: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); + let v2: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); + let v3: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + let _ = token; + let v0: __m128 = vectors[0].into(); + let v1: __m128 = vectors[1].into(); + let v2: __m128 = vectors[2].into(); + let v3: __m128 = vectors[3].into(); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -1134,6 +1217,95 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[i8; 64usize]) -> [i8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, vectors: [i8x16; 4usize], dest: &mut [i8; 64usize]) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1618,6 +1790,95 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[u8; 64usize]) -> [u8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, vectors: [u8x16; 4usize], dest: &mut [u8; 64usize]) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -2222,41 +2483,134 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] - fn splat_u16x8(self, val: u16) -> u16x8 { + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, val: u16) -> u16x8 { - _mm_set1_epi16(val.cast_signed()).simd_into(token) + fn kernel(token: Avx2, src: &[i16; 32usize]) -> [i16x8; 4usize] { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] } ); - kernel(self, val) - } - #[inline(always)] - fn load_array_u16x8(self, val: [u16; 8usize]) -> u16x8 { - u16x8 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } - } - #[inline(always)] - fn load_array_ref_u16x8(self, val: &[u16; 8usize]) -> u16x8 { - u16x8 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } - } - #[inline(always)] - fn as_array_u16x8(self, a: u16x8) -> [u16; 8usize] { - crate::transmute::checked_transmute_copy::<__m128i, [u16; 8usize]>(&a.val.0) - } - #[inline(always)] - fn as_array_ref_u16x8(self, a: &u16x8) -> &[u16; 8usize] { - crate::transmute::checked_cast_ref::<__m128i, [u16; 8usize]>(&a.val.0) + kernel(self, src) } #[inline(always)] - fn as_array_mut_u16x8(self, a: &mut u16x8) -> &mut [u16; 8usize] { - crate::transmute::checked_cast_mut::<__m128i, [u16; 8usize]>(&mut a.val.0) - } + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] + fn splat_u16x8(self, val: u16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, val: u16) -> u16x8 { + _mm_set1_epi16(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u16x8(self, val: [u16; 8usize]) -> u16x8 { + u16x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u16x8(self, val: &[u16; 8usize]) -> u16x8 { + u16x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u16x8(self, a: u16x8) -> [u16; 8usize] { + crate::transmute::checked_transmute_copy::<__m128i, [u16; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u16x8(self, a: &u16x8) -> &[u16; 8usize] { + crate::transmute::checked_cast_ref::<__m128i, [u16; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_mut_u16x8(self, a: &mut u16x8) -> &mut [u16; 8usize] { + crate::transmute::checked_cast_mut::<__m128i, [u16; 8usize]>(&mut a.val.0) + } #[inline(always)] fn store_array_u16x8(self, a: u16x8, dest: &mut [u16; 8usize]) -> () { crate::transmute::checked_transmute_store(a.val.0, dest); @@ -2641,6 +2995,99 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[u16; 32usize]) -> [u16x8; 4usize] { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { crate::kernel!( #[inline(always)] @@ -3207,6 +3654,89 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[i32; 16usize]) -> [i32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -3610,6 +4140,89 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[u32; 16usize]) -> [u32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -4234,14 +4847,79 @@ impl Simd for Avx2 { kernel(self, a, b, c) } #[inline(always)] - fn combine_f64x2(self, a: f64x2, b: f64x2) -> f64x4 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: f64x2, b: f64x2) -> f64x4 { - _mm256_setr_m128d(a.into(), b.into()).simd_into(token) + fn combine_f64x2(self, a: f64x2, b: f64x2) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x2, b: f64x2) -> f64x4 { + _mm256_setr_m128d(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[f64; 8usize]) -> [f64x2; 4usize] { + let (chunks, []) = src.as_chunks::<4>() else { + unreachable!() + }; + let v0: __m256d = + crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[0]); + let v1: __m256d = + crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[1]); + let lo = _mm256_unpacklo_pd(v0, v1); + let hi = _mm256_unpackhi_pd(v0, v1); + let out0 = _mm256_permute2f128_pd::<0x20>(lo, hi); + let out1 = _mm256_permute2f128_pd::<0x31>(lo, hi); + let outputs: [__m128d; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, vectors: [f64x2; 4usize], dest: &mut [f64; 8usize]) -> () { + let _ = token; + let inputs: [__m128d; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let wide_inputs: [__m256d; 2] = crate::transmute::checked_transmute_copy(&inputs); + let v0 = wide_inputs[0]; + let v1 = wide_inputs[1]; + let lo = _mm256_permute2f128_pd::<0x20>(v0, v1); + let hi = _mm256_permute2f128_pd::<0x31>(v0, v1); + let out0 = _mm256_unpacklo_pd(lo, hi); + let out1 = _mm256_unpackhi_pd(lo, hi); + let (chunks, []) = dest.as_chunks_mut::<4>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m256d, [f64; 4]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m256d, [f64; 4]>( + out1, + &mut chunks[1], + ); } ); - kernel(self, a, b) + kernel(self, vectors, dest); } #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { @@ -4612,6 +5290,71 @@ impl Simd for Avx2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[i64; 8usize]) -> [i64x2; 4usize] { + let (chunks, []) = src.as_chunks::<4>() else { + unreachable!() + }; + let v0: __m256i = + crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[0]); + let v1: __m256i = + crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[1]); + let lo = _mm256_unpacklo_epi64(v0, v1); + let hi = _mm256_unpackhi_epi64(v0, v1); + let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); + let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, vectors: [i64x2; 4usize], dest: &mut [i64; 8usize]) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let wide_inputs: [__m256i; 2] = crate::transmute::checked_transmute_copy(&inputs); + let v0 = wide_inputs[0]; + let v1 = wide_inputs[1]; + let lo = _mm256_permute2x128_si256::<0x20>(v0, v1); + let hi = _mm256_permute2x128_si256::<0x31>(v0, v1); + let out0 = _mm256_unpacklo_epi64(lo, hi); + let out1 = _mm256_unpackhi_epi64(lo, hi); + let (chunks, []) = dest.as_chunks_mut::<4>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m256i, [i64; 4]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m256i, [i64; 4]>( + out1, + &mut chunks[1], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -4967,6 +5710,71 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, src: &[u64; 8usize]) -> [u64x2; 4usize] { + let (chunks, []) = src.as_chunks::<4>() else { + unreachable!() + }; + let v0: __m256i = + crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[0]); + let v1: __m256i = + crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[1]); + let lo = _mm256_unpacklo_epi64(v0, v1); + let hi = _mm256_unpackhi_epi64(v0, v1); + let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); + let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, vectors: [u64x2; 4usize], dest: &mut [u64; 8usize]) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let wide_inputs: [__m256i; 2] = crate::transmute::checked_transmute_copy(&inputs); + let v0 = wide_inputs[0]; + let v1 = wide_inputs[1]; + let lo = _mm256_permute2x128_si256::<0x20>(v0, v1); + let hi = _mm256_permute2x128_si256::<0x31>(v0, v1); + let out0 = _mm256_unpacklo_epi64(lo, hi); + let out1 = _mm256_unpackhi_epi64(lo, hi); + let (chunks, []) = dest.as_chunks_mut::<4>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>( + out1, + &mut chunks[1], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { crate::kernel!( #[inline(always)] @@ -11746,81 +12554,6 @@ impl Simd for Avx2 { ) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[f32; 16usize]) -> f32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); - let v1: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); - let v2: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); - let v3: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - token.combine_f32x8( - token.combine_f32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_f32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: f32x16, dest: &mut [f32; 16usize]) -> () { - let (v01, v23) = token.split_f32x16(a); - let (v0, v1) = token.split_f32x8(v01); - let (v2, v3) = token.split_f32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) @@ -12363,118 +13096,33 @@ impl Simd for Avx2 { } #[inline(always)] fn min_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.min_i8x32(a0, b0), self.min_i8x32(a1, b1)) - } - #[inline(always)] - fn max_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - let (b0, b1) = self.split_i8x64(b); - self.combine_i8x32(self.max_i8x32(a0, b0), self.max_i8x32(a1, b1)) - } - #[inline(always)] - fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { - ( - i8x32 { - val: crate::support::Aligned256(a.val.0[0]), - simd: self, - }, - i8x32 { - val: crate::support::Aligned256(a.val.0[1]), - simd: self, - }, - ) - } - #[inline(always)] - fn neg_i8x64(self, a: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) - } - #[inline(always)] - fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[i8; 64usize]) -> i8x64 { - let (chunks, []) = src.as_chunks::<16usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_i8x32( - token.combine_i8x16(out0.simd_into(token), out1.simd_into(token)), - token.combine_i8x16(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i8x64, dest: &mut [i8; 64usize]) -> () { - let (v01, v23) = token.split_i8x64(a); - let (v0, v1) = token.split_i8x32(v01); - let (v2, v3) = token.split_i8x32(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let out0 = _mm_shuffle_epi8(out0, mask); - let out1 = _mm_shuffle_epi8(out1, mask); - let out2 = _mm_shuffle_epi8(out2, mask); - let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<16usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.min_i8x32(a0, b0), self.min_i8x32(a1, b1)) + } + #[inline(always)] + fn max_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (b0, b1) = self.split_i8x64(b); + self.combine_i8x32(self.max_i8x32(a0, b0), self.max_i8x32(a1, b1)) + } + #[inline(always)] + fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { + ( + i8x32 { + val: crate::support::Aligned256(a.val.0[0]), + simd: self, + }, + i8x32 { + val: crate::support::Aligned256(a.val.0[1]), + simd: self, + }, + ) + } + #[inline(always)] + fn neg_i8x64(self, a: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) } #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { @@ -13017,91 +13665,6 @@ impl Simd for Avx2 { ) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[u8; 64usize]) -> u8x64 { - let (chunks, []) = src.as_chunks::<16usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u8x32( - token.combine_u8x16(out0.simd_into(token), out1.simd_into(token)), - token.combine_u8x16(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u8x64, dest: &mut [u8; 64usize]) -> () { - let (v01, v23) = token.split_u8x64(a); - let (v0, v1) = token.split_u8x32(v01); - let (v2, v3) = token.split_u8x32(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let out0 = _mm_shuffle_epi8(out0, mask); - let out1 = _mm_shuffle_epi8(out1, mask); - let out2 = _mm_shuffle_epi8(out2, mask); - let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<16usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { let half = self.splat_mask8x32(val); self.combine_mask8x32(half, half) @@ -13681,91 +14244,6 @@ impl Simd for Avx2 { self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) } #[inline(always)] - fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[i16; 32usize]) -> i16x32 { - let (chunks, []) = src.as_chunks::<8usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_i16x16( - token.combine_i16x8(out0.simd_into(token), out1.simd_into(token)), - token.combine_i16x8(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i16x32, dest: &mut [i16; 32usize]) -> () { - let (v01, v23) = token.split_i16x32(a); - let (v0, v1) = token.split_i16x16(v01); - let (v2, v3) = token.split_i16x16(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); - let out0 = _mm_shuffle_epi8(out0, mask); - let out1 = _mm_shuffle_epi8(out1, mask); - let out2 = _mm_shuffle_epi8(out2, mask); - let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<8usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { let half = self.splat_u16x16(val); self.combine_u16x16(half, half) @@ -14169,110 +14647,25 @@ impl Simd for Avx2 { self.combine_u16x16(self.min_u16x16(a0, b0), self.min_u16x16(a1, b1)) } #[inline(always)] - fn max_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { - let (a0, a1) = self.split_u16x32(a); - let (b0, b1) = self.split_u16x32(b); - self.combine_u16x16(self.max_u16x16(a0, b0), self.max_u16x16(a1, b1)) - } - #[inline(always)] - fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { - ( - u16x16 { - val: crate::support::Aligned256(a.val.0[0]), - simd: self, - }, - u16x16 { - val: crate::support::Aligned256(a.val.0[1]), - simd: self, - }, - ) - } - #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[u16; 32usize]) -> u16x32 { - let (chunks, []) = src.as_chunks::<8usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u16x16( - token.combine_u16x8(out0.simd_into(token), out1.simd_into(token)), - token.combine_u16x8(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u16x32, dest: &mut [u16; 32usize]) -> () { - let (v01, v23) = token.split_u16x32(a); - let (v0, v1) = token.split_u16x16(v01); - let (v2, v3) = token.split_u16x16(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); - let out0 = _mm_shuffle_epi8(out0, mask); - let out1 = _mm_shuffle_epi8(out1, mask); - let out2 = _mm_shuffle_epi8(out2, mask); - let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<8usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] + fn max_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (b0, b1) = self.split_u16x32(b); + self.combine_u16x16(self.max_u16x16(a0, b0), self.max_u16x16(a1, b1)) + } + #[inline(always)] + fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { + ( + u16x16 { + val: crate::support::Aligned256(a.val.0[0]), + simd: self, + }, + u16x16 { + val: crate::support::Aligned256(a.val.0[1]), + simd: self, + }, + ) + } + #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { crate::kernel!( #[inline(always)] @@ -14776,81 +15169,6 @@ impl Simd for Avx2 { self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] - fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[i32; 16usize]) -> i32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_i32x8( - token.combine_i32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_i32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i32x16, dest: &mut [i32; 16usize]) -> () { - let (v01, v23) = token.split_i32x16(a); - let (v0, v1) = token.split_i32x8(v01); - let (v2, v3) = token.split_i32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { let (a0, a1) = self.split_i32x16(a); self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) @@ -15208,81 +15526,6 @@ impl Simd for Avx2 { ) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[u32; 16usize]) -> u32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u32x8( - token.combine_u32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_u32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u32x16, dest: &mut [u32; 16usize]) -> () { - let (v01, v23) = token.split_u32x16(a); - let (v0, v1) = token.split_u32x8(v01); - let (v2, v3) = token.split_u32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { let (a0, a1) = self.split_u32x16(a); self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) @@ -15798,54 +16041,6 @@ impl Simd for Avx2 { ) } #[inline(always)] - fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[f64; 8usize]) -> f64x8 { - let (chunks, []) = src.as_chunks::<4>() else { - unreachable!() - }; - let v0: __m256d = - crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[0]); - let v1: __m256d = - crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[1]); - let lo = _mm256_unpacklo_pd(v0, v1); - let hi = _mm256_unpackhi_pd(v0, v1); - let out0 = _mm256_permute2f128_pd::<0x20>(lo, hi); - let out1 = _mm256_permute2f128_pd::<0x31>(lo, hi); - token.combine_f64x4(out0.simd_into(token), out1.simd_into(token)) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: f64x8, dest: &mut [f64; 8usize]) -> () { - let (v0, v1) = token.split_f64x8(a); - let v0: __m256d = v0.into(); - let v1: __m256d = v1.into(); - let lo = _mm256_permute2f128_pd::<0x20>(v0, v1); - let hi = _mm256_permute2f128_pd::<0x31>(v0, v1); - let out0 = _mm256_unpacklo_pd(lo, hi); - let out1 = _mm256_unpackhi_pd(lo, hi); - let (chunks, []) = dest.as_chunks_mut::<4>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m256d, [f64; 4]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m256d, [f64; 4]>( - out1, - &mut chunks[1], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) @@ -16167,54 +16362,6 @@ impl Simd for Avx2 { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] - fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[i64; 8usize]) -> i64x8 { - let (chunks, []) = src.as_chunks::<4>() else { - unreachable!() - }; - let v0: __m256i = - crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[0]); - let v1: __m256i = - crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[1]); - let lo = _mm256_unpacklo_epi64(v0, v1); - let hi = _mm256_unpackhi_epi64(v0, v1); - let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); - let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); - token.combine_i64x4(out0.simd_into(token), out1.simd_into(token)) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i64x8, dest: &mut [i64; 8usize]) -> () { - let (v0, v1) = token.split_i64x8(a); - let v0: __m256i = v0.into(); - let v1: __m256i = v1.into(); - let lo = _mm256_permute2x128_si256::<0x20>(v0, v1); - let hi = _mm256_permute2x128_si256::<0x31>(v0, v1); - let out0 = _mm256_unpacklo_epi64(lo, hi); - let out1 = _mm256_unpackhi_epi64(lo, hi); - let (chunks, []) = dest.as_chunks_mut::<4>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m256i, [i64; 4]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m256i, [i64; 4]>( - out1, - &mut chunks[1], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) @@ -16531,54 +16678,6 @@ impl Simd for Avx2 { ) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[u64; 8usize]) -> u64x8 { - let (chunks, []) = src.as_chunks::<4>() else { - unreachable!() - }; - let v0: __m256i = - crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[0]); - let v1: __m256i = - crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[1]); - let lo = _mm256_unpacklo_epi64(v0, v1); - let hi = _mm256_unpackhi_epi64(v0, v1); - let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi); - let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi); - token.combine_u64x4(out0.simd_into(token), out1.simd_into(token)) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u64x8, dest: &mut [u64; 8usize]) -> () { - let (v0, v1) = token.split_u64x8(a); - let v0: __m256i = v0.into(); - let v1: __m256i = v1.into(); - let lo = _mm256_permute2x128_si256::<0x20>(v0, v1); - let hi = _mm256_permute2x128_si256::<0x31>(v0, v1); - let out0 = _mm256_unpacklo_epi64(lo, hi); - let out1 = _mm256_unpackhi_epi64(lo, hi); - let (chunks, []) = dest.as_chunks_mut::<4>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>( - out1, - &mut chunks[1], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index db53767f..6bd6e401 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -601,6 +601,58 @@ impl Simd for Avx512 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[f32; 16usize]) -> [f32x4; 4usize] { + let lanes: __m512 = + crate::transmute::checked_transmute_copy::<[f32; 16usize], __m512>(src); + let lanes = _mm512_permutexvar_ps( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ); + let outputs: [__m128; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + let _ = token; + let inputs: [__m128; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512 = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_ps( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512, [f32; 16usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -1182,6 +1234,68 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[i8; 64usize]) -> [i8x16; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[i8; 64usize], __m512i>(src); + let lanes = _mm512_permutexvar_epi8( + _mm512_set_epi8( + 63, 59, 55, 51, 47, 43, 39, 35, 31, 27, 23, 19, 15, 11, 7, 3, 62, 58, 54, + 50, 46, 42, 38, 34, 30, 26, 22, 18, 14, 10, 6, 2, 61, 57, 53, 49, 45, 41, + 37, 33, 29, 25, 21, 17, 13, 9, 5, 1, 60, 56, 52, 48, 44, 40, 36, 32, 28, + 24, 20, 16, 12, 8, 4, 0, + ), + lanes, + ); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_epi8( + _mm512_set_epi8( + 63, 47, 31, 15, 62, 46, 30, 14, 61, 45, 29, 13, 60, 44, 28, 12, 59, 43, 27, + 11, 58, 42, 26, 10, 57, 41, 25, 9, 56, 40, 24, 8, 55, 39, 23, 7, 54, 38, + 22, 6, 53, 37, 21, 5, 52, 36, 20, 4, 51, 35, 19, 3, 50, 34, 18, 2, 49, 33, + 17, 1, 48, 32, 16, 0, + ), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512i, [i8; 64usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1697,6 +1811,68 @@ impl Simd for Avx512 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[u8; 64usize]) -> [u8x16; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[u8; 64usize], __m512i>(src); + let lanes = _mm512_permutexvar_epi8( + _mm512_set_epi8( + 63, 59, 55, 51, 47, 43, 39, 35, 31, 27, 23, 19, 15, 11, 7, 3, 62, 58, 54, + 50, 46, 42, 38, 34, 30, 26, 22, 18, 14, 10, 6, 2, 61, 57, 53, 49, 45, 41, + 37, 33, 29, 25, 21, 17, 13, 9, 5, 1, 60, 56, 52, 48, 44, 40, 36, 32, 28, + 24, 20, 16, 12, 8, 4, 0, + ), + lanes, + ); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_epi8( + _mm512_set_epi8( + 63, 47, 31, 15, 62, 46, 30, 14, 61, 45, 29, 13, 60, 44, 28, 12, 59, 43, 27, + 11, 58, 42, 26, 10, 57, 41, 25, 9, 56, 40, 24, 8, 55, 39, 23, 7, 54, 38, + 22, 6, 53, 37, 21, 5, 52, 36, 20, 4, 51, 35, 19, 3, 50, 34, 18, 2, 49, 33, + 17, 1, 48, 32, 16, 0, + ), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512i, [u8; 64usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -2288,6 +2464,64 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[i16; 32usize]) -> [i16x8; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[i16; 32usize], __m512i>(src); + let lanes = _mm512_permutexvar_epi16( + _mm512_set_epi16( + 31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2, 29, 25, 21, 17, + 13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0, + ), + lanes, + ); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_epi16( + _mm512_set_epi16( + 31, 23, 15, 7, 30, 22, 14, 6, 29, 21, 13, 5, 28, 20, 12, 4, 27, 19, 11, 3, + 26, 18, 10, 2, 25, 17, 9, 1, 24, 16, 8, 0, + ), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512i, [i16; 32usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u16x8(self, val: u16) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2724,6 +2958,64 @@ impl Simd for Avx512 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[u16; 32usize]) -> [u16x8; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[u16; 32usize], __m512i>(src); + let lanes = _mm512_permutexvar_epi16( + _mm512_set_epi16( + 31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2, 29, 25, 21, 17, + 13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0, + ), + lanes, + ); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_epi16( + _mm512_set_epi16( + 31, 23, 15, 7, 30, 22, 14, 6, 29, 21, 13, 5, 28, 20, 12, 4, 27, 19, 11, 3, + 26, 18, 10, 2, 25, 17, 9, 1, 24, 16, 8, 0, + ), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512i, [u16; 32usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { mask16x8 { val: (if val { 255u64 } else { 0 }) as _, @@ -3279,6 +3571,58 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[i32; 16usize]) -> [i32x4; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[i32; 16usize], __m512i>(src); + let lanes = _mm512_permutexvar_epi32( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_epi32( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512i, [i32; 16usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -3699,6 +4043,58 @@ impl Simd for Avx512 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[u32; 16usize]) -> [u32x4; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[u32; 16usize], __m512i>(src); + let lanes = _mm512_permutexvar_epi32( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_epi32( + _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), + lanes, + ); + crate::transmute::checked_transmute_store::<__m512i, [u32; 16usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -4316,6 +4712,52 @@ impl Simd for Avx512 { kernel(self, a, b) } #[inline(always)] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[f64; 8usize]) -> [f64x2; 4usize] { + let lanes: __m512d = + crate::transmute::checked_transmute_copy::<[f64; 8usize], __m512d>(src); + let lanes = _mm512_permutexvar_pd(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes); + let outputs: [__m128d; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + let _ = token; + let inputs: [__m128d; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512d = crate::transmute::checked_transmute_copy(&inputs); + let lanes = _mm512_permutexvar_pd(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), lanes); + crate::transmute::checked_transmute_store::<__m512d, [f64; 8usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] @@ -4726,6 +5168,54 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[i64; 8usize]) -> [i64x2; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[i64; 8usize], __m512i>(src); + let lanes = + _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = + _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), lanes); + crate::transmute::checked_transmute_store::<__m512i, [i64; 8usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5123,7 +5613,55 @@ impl Simd for Avx512 { _mm256_setr_m128i(a.into(), b.into()).simd_into(token) } ); - kernel(self, a, b) + kernel(self, a, b) + } + #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, src: &[u64; 8usize]) -> [u64x2; 4usize] { + let lanes: __m512i = + crate::transmute::checked_transmute_copy::<[u64; 8usize], __m512i>(src); + let lanes = + _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes); + let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(token), + outputs[1].simd_into(token), + outputs[2].simd_into(token), + outputs[3].simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + let _ = token; + let inputs: [__m128i; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: __m512i = crate::transmute::checked_transmute_copy(&inputs); + let lanes = + _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), lanes); + crate::transmute::checked_transmute_store::<__m512i, [u64; 8usize]>(lanes, dest); + } + ); + kernel(self, vectors, dest); } #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { @@ -12245,36 +12783,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[f32; 16usize]) -> f32x16 { - let lanes: __m512 = - crate::transmute::checked_transmute_copy::<[f32; 16usize], __m512>(src); - _mm512_permutexvar_ps( - _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), - lanes, - ) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: f32x16, dest: &mut [f32; 16usize]) -> () { - let lanes = _mm512_permutexvar_ps( - _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), - a.into(), - ); - crate::transmute::checked_transmute_store::<__m512, [f32; 16usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { crate::kernel!( #[inline(always)] @@ -13165,46 +13673,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[i8; 64usize]) -> i8x64 { - let lanes: __m512i = - crate::transmute::checked_transmute_copy::<[i8; 64usize], __m512i>(src); - _mm512_permutexvar_epi8( - _mm512_set_epi8( - 63, 59, 55, 51, 47, 43, 39, 35, 31, 27, 23, 19, 15, 11, 7, 3, 62, 58, 54, - 50, 46, 42, 38, 34, 30, 26, 22, 18, 14, 10, 6, 2, 61, 57, 53, 49, 45, 41, - 37, 33, 29, 25, 21, 17, 13, 9, 5, 1, 60, 56, 52, 48, 44, 40, 36, 32, 28, - 24, 20, 16, 12, 8, 4, 0, - ), - lanes, - ) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: i8x64, dest: &mut [i8; 64usize]) -> () { - let lanes = _mm512_permutexvar_epi8( - _mm512_set_epi8( - 63, 47, 31, 15, 62, 46, 30, 14, 61, 45, 29, 13, 60, 44, 28, 12, 59, 43, 27, - 11, 58, 42, 26, 10, 57, 41, 25, 9, 56, 40, 24, 8, 55, 39, 23, 7, 54, 38, - 22, 6, 53, 37, 21, 5, 52, 36, 20, 4, 51, 35, 19, 3, 50, 34, 18, 2, 49, 33, - 17, 1, 48, 32, 16, 0, - ), - a.into(), - ); - crate::transmute::checked_transmute_store::<__m512i, [i8; 64usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { crate::kernel!( #[inline(always)] @@ -14026,46 +14494,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[u8; 64usize]) -> u8x64 { - let lanes: __m512i = - crate::transmute::checked_transmute_copy::<[u8; 64usize], __m512i>(src); - _mm512_permutexvar_epi8( - _mm512_set_epi8( - 63, 59, 55, 51, 47, 43, 39, 35, 31, 27, 23, 19, 15, 11, 7, 3, 62, 58, 54, - 50, 46, 42, 38, 34, 30, 26, 22, 18, 14, 10, 6, 2, 61, 57, 53, 49, 45, 41, - 37, 33, 29, 25, 21, 17, 13, 9, 5, 1, 60, 56, 52, 48, 44, 40, 36, 32, 28, - 24, 20, 16, 12, 8, 4, 0, - ), - lanes, - ) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: u8x64, dest: &mut [u8; 64usize]) -> () { - let lanes = _mm512_permutexvar_epi8( - _mm512_set_epi8( - 63, 47, 31, 15, 62, 46, 30, 14, 61, 45, 29, 13, 60, 44, 28, 12, 59, 43, 27, - 11, 58, 42, 26, 10, 57, 41, 25, 9, 56, 40, 24, 8, 55, 39, 23, 7, 54, 38, - 22, 6, 53, 37, 21, 5, 52, 36, 20, 4, 51, 35, 19, 3, 50, 34, 18, 2, 49, 33, - 17, 1, 48, 32, 16, 0, - ), - a.into(), - ); - crate::transmute::checked_transmute_store::<__m512i, [u8; 64usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { mask8x64 { val: if val { u64::MAX } else { 0 }, @@ -14853,42 +15281,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[i16; 32usize]) -> i16x32 { - let lanes: __m512i = - crate::transmute::checked_transmute_copy::<[i16; 32usize], __m512i>(src); - _mm512_permutexvar_epi16( - _mm512_set_epi16( - 31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2, 29, 25, 21, 17, - 13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0, - ), - lanes, - ) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: i16x32, dest: &mut [i16; 32usize]) -> () { - let lanes = _mm512_permutexvar_epi16( - _mm512_set_epi16( - 31, 23, 15, 7, 30, 22, 14, 6, 29, 21, 13, 5, 28, 20, 12, 4, 27, 19, 11, 3, - 26, 18, 10, 2, 25, 17, 9, 1, 24, 16, 8, 0, - ), - a.into(), - ); - crate::transmute::checked_transmute_store::<__m512i, [i16; 32usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { crate::kernel!( #[inline(always)] @@ -15525,42 +15917,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[u16; 32usize]) -> u16x32 { - let lanes: __m512i = - crate::transmute::checked_transmute_copy::<[u16; 32usize], __m512i>(src); - _mm512_permutexvar_epi16( - _mm512_set_epi16( - 31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2, 29, 25, 21, 17, - 13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0, - ), - lanes, - ) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: u16x32, dest: &mut [u16; 32usize]) -> () { - let lanes = _mm512_permutexvar_epi16( - _mm512_set_epi16( - 31, 23, 15, 7, 30, 22, 14, 6, 29, 21, 13, 5, 28, 20, 12, 4, 27, 19, 11, 3, - 26, 18, 10, 2, 25, 17, 9, 1, 24, 16, 8, 0, - ), - a.into(), - ); - crate::transmute::checked_transmute_store::<__m512i, [u16; 32usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { crate::kernel!( #[inline(always)] @@ -16276,36 +16632,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[i32; 16usize]) -> i32x16 { - let lanes: __m512i = - crate::transmute::checked_transmute_copy::<[i32; 16usize], __m512i>(src); - _mm512_permutexvar_epi32( - _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), - lanes, - ) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: i32x16, dest: &mut [i32; 16usize]) -> () { - let lanes = _mm512_permutexvar_epi32( - _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), - a.into(), - ); - crate::transmute::checked_transmute_store::<__m512i, [i32; 16usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { crate::kernel!( #[inline(always)] @@ -16870,36 +17196,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[u32; 16usize]) -> u32x16 { - let lanes: __m512i = - crate::transmute::checked_transmute_copy::<[u32; 16usize], __m512i>(src); - _mm512_permutexvar_epi32( - _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), - lanes, - ) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: u32x16, dest: &mut [u32; 16usize]) -> () { - let lanes = _mm512_permutexvar_epi32( - _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15), - a.into(), - ); - crate::transmute::checked_transmute_store::<__m512i, [u32; 16usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { crate::kernel!( #[inline(always)] @@ -17636,31 +17932,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[f64; 8usize]) -> f64x8 { - let lanes: __m512d = - crate::transmute::checked_transmute_copy::<[f64; 8usize], __m512d>(src); - _mm512_permutexvar_pd(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: f64x8, dest: &mut [f64; 8usize]) -> () { - let lanes = - _mm512_permutexvar_pd(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), a.into()); - crate::transmute::checked_transmute_store::<__m512d, [f64; 8usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { crate::kernel!( #[inline(always)] @@ -18167,31 +18438,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[i64; 8usize]) -> i64x8 { - let lanes: __m512i = - crate::transmute::checked_transmute_copy::<[i64; 8usize], __m512i>(src); - _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: i64x8, dest: &mut [i64; 8usize]) -> () { - let lanes = - _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), a.into()); - crate::transmute::checked_transmute_store::<__m512i, [i64; 8usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { crate::kernel!( #[inline(always)] @@ -18688,31 +18934,6 @@ impl Simd for Avx512 { kernel(self, a) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, src: &[u64; 8usize]) -> u64x8 { - let lanes: __m512i = - crate::transmute::checked_transmute_copy::<[u64; 8usize], __m512i>(src); - _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7), lanes) - .simd_into(token) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: u64x8, dest: &mut [u64; 8usize]) -> () { - let lanes = - _mm512_permutexvar_epi64(_mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7), a.into()); - crate::transmute::checked_transmute_store::<__m512i, [u64; 8usize]>(lanes, dest); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { mask64x8 { val: (if val { 255u64 } else { 0 }) as _, diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index 69b5967b..42a15ed5 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -627,6 +627,40 @@ impl Simd for Fallback { result.simd_into(self) } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + [ + [src[0usize], src[4usize], src[8usize], src[12usize]].simd_into(self), + [src[1usize], src[5usize], src[9usize], src[13usize]].simd_into(self), + [src[2usize], src[6usize], src[10usize], src[14usize]].simd_into(self), + [src[3usize], src[7usize], src[11usize], src[15usize]].simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + ]; + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { [ a[0usize] as u32, @@ -1442,6 +1476,160 @@ impl Simd for Fallback { .simd_into(self) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + src[32usize], + src[36usize], + src[40usize], + src[44usize], + src[48usize], + src[52usize], + src[56usize], + src[60usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + src[33usize], + src[37usize], + src[41usize], + src[45usize], + src[49usize], + src[53usize], + src[57usize], + src[61usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + src[34usize], + src[38usize], + src[42usize], + src[46usize], + src[50usize], + src[54usize], + src[58usize], + src[62usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + src[35usize], + src[39usize], + src[43usize], + src[47usize], + src[51usize], + src[55usize], + src[59usize], + src[63usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + vectors[0usize][8usize], + vectors[1usize][8usize], + vectors[2usize][8usize], + vectors[3usize][8usize], + vectors[0usize][9usize], + vectors[1usize][9usize], + vectors[2usize][9usize], + vectors[3usize][9usize], + vectors[0usize][10usize], + vectors[1usize][10usize], + vectors[2usize][10usize], + vectors[3usize][10usize], + vectors[0usize][11usize], + vectors[1usize][11usize], + vectors[2usize][11usize], + vectors[3usize][11usize], + vectors[0usize][12usize], + vectors[1usize][12usize], + vectors[2usize][12usize], + vectors[3usize][12usize], + vectors[0usize][13usize], + vectors[1usize][13usize], + vectors[2usize][13usize], + vectors[3usize][13usize], + vectors[0usize][14usize], + vectors[1usize][14usize], + vectors[2usize][14usize], + vectors[3usize][14usize], + vectors[0usize][15usize], + vectors[1usize][15usize], + vectors[2usize][15usize], + vectors[3usize][15usize], + ]; + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { [val; 16usize].simd_into(self) } @@ -2195,6 +2383,160 @@ impl Simd for Fallback { result.simd_into(self) } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + src[32usize], + src[36usize], + src[40usize], + src[44usize], + src[48usize], + src[52usize], + src[56usize], + src[60usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + src[33usize], + src[37usize], + src[41usize], + src[45usize], + src[49usize], + src[53usize], + src[57usize], + src[61usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + src[34usize], + src[38usize], + src[42usize], + src[46usize], + src[50usize], + src[54usize], + src[58usize], + src[62usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + src[35usize], + src[39usize], + src[43usize], + src[47usize], + src[51usize], + src[55usize], + src[59usize], + src[63usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + vectors[0usize][8usize], + vectors[1usize][8usize], + vectors[2usize][8usize], + vectors[3usize][8usize], + vectors[0usize][9usize], + vectors[1usize][9usize], + vectors[2usize][9usize], + vectors[3usize][9usize], + vectors[0usize][10usize], + vectors[1usize][10usize], + vectors[2usize][10usize], + vectors[3usize][10usize], + vectors[0usize][11usize], + vectors[1usize][11usize], + vectors[2usize][11usize], + vectors[3usize][11usize], + vectors[0usize][12usize], + vectors[1usize][12usize], + vectors[2usize][12usize], + vectors[3usize][12usize], + vectors[0usize][13usize], + vectors[1usize][13usize], + vectors[2usize][13usize], + vectors[3usize][13usize], + vectors[0usize][14usize], + vectors[1usize][14usize], + vectors[2usize][14usize], + vectors[3usize][14usize], + vectors[0usize][15usize], + vectors[1usize][15usize], + vectors[2usize][15usize], + vectors[3usize][15usize], + ]; + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { [ a[0usize] as u16, @@ -3107,6 +3449,96 @@ impl Simd for Fallback { .simd_into(self) } #[inline(always)] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + ]; + } + #[inline(always)] fn splat_u16x8(self, val: u16) -> u16x8 { [val; 8usize].simd_into(self) } @@ -3637,6 +4069,96 @@ impl Simd for Fallback { result.simd_into(self) } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + ]; + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { let val: i16 = if val { !0 } else { 0 }; [val; 8usize].simd_into(self) @@ -4283,6 +4805,40 @@ impl Simd for Fallback { .simd_into(self) } #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + [ + [src[0usize], src[4usize], src[8usize], src[12usize]].simd_into(self), + [src[1usize], src[5usize], src[9usize], src[13usize]].simd_into(self), + [src[2usize], src[6usize], src[10usize], src[14usize]].simd_into(self), + [src[3usize], src[7usize], src[11usize], src[15usize]].simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + ]; + } + #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { [ a[0usize] as f32, @@ -4703,6 +5259,40 @@ impl Simd for Fallback { result.simd_into(self) } #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + [ + [src[0usize], src[4usize], src[8usize], src[12usize]].simd_into(self), + [src[1usize], src[5usize], src[9usize], src[13usize]].simd_into(self), + [src[2usize], src[6usize], src[10usize], src[14usize]].simd_into(self), + [src[3usize], src[7usize], src[11usize], src[15usize]].simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + ]; + } + #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { [ a[0usize] as f32, @@ -5239,6 +5829,32 @@ impl Simd for Fallback { result.simd_into(self) } #[inline(always)] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { + [ + [src[0usize], src[4usize]].simd_into(self), + [src[1usize], src[5usize]].simd_into(self), + [src[2usize], src[6usize]].simd_into(self), + [src[3usize], src[7usize]].simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + ]; + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { [val; 2usize].simd_into(self) } @@ -5595,6 +6211,32 @@ impl Simd for Fallback { [i64::wrapping_neg(a[0usize]), i64::wrapping_neg(a[1usize])].simd_into(self) } #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + [ + [src[0usize], src[4usize]].simd_into(self), + [src[1usize], src[5usize]].simd_into(self), + [src[2usize], src[6usize]].simd_into(self), + [src[3usize], src[7usize]].simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + ]; + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { [val; 2usize].simd_into(self) } @@ -5947,6 +6589,32 @@ impl Simd for Fallback { result.simd_into(self) } #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + [ + [src[0usize], src[4usize]].simd_into(self), + [src[1usize], src[5usize]].simd_into(self), + [src[2usize], src[6usize]].simd_into(self), + [src[3usize], src[7usize]].simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + ]; + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { let val: i64 = if val { !0 } else { 0 }; [val; 2usize].simd_into(self) @@ -10446,36 +11114,6 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - *dest = [ - a[0usize], a[4usize], a[8usize], a[12usize], a[1usize], a[5usize], a[9usize], - a[13usize], a[2usize], a[6usize], a[10usize], a[14usize], a[3usize], a[7usize], - a[11usize], a[15usize], - ]; - } - #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) @@ -11033,91 +11671,6 @@ impl Simd for Fallback { self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) } #[inline(always)] - fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[16usize], - src[20usize], - src[24usize], - src[28usize], - src[32usize], - src[36usize], - src[40usize], - src[44usize], - src[48usize], - src[52usize], - src[56usize], - src[60usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[17usize], - src[21usize], - src[25usize], - src[29usize], - src[33usize], - src[37usize], - src[41usize], - src[45usize], - src[49usize], - src[53usize], - src[57usize], - src[61usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[18usize], - src[22usize], - src[26usize], - src[30usize], - src[34usize], - src[38usize], - src[42usize], - src[46usize], - src[50usize], - src[54usize], - src[58usize], - src[62usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - src[19usize], - src[23usize], - src[27usize], - src[31usize], - src[35usize], - src[39usize], - src[43usize], - src[47usize], - src[51usize], - src[55usize], - src[59usize], - src[63usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { - *dest = [ - a[0usize], a[16usize], a[32usize], a[48usize], a[1usize], a[17usize], a[33usize], - a[49usize], a[2usize], a[18usize], a[34usize], a[50usize], a[3usize], a[19usize], - a[35usize], a[51usize], a[4usize], a[20usize], a[36usize], a[52usize], a[5usize], - a[21usize], a[37usize], a[53usize], a[6usize], a[22usize], a[38usize], a[54usize], - a[7usize], a[23usize], a[39usize], a[55usize], a[8usize], a[24usize], a[40usize], - a[56usize], a[9usize], a[25usize], a[41usize], a[57usize], a[10usize], a[26usize], - a[42usize], a[58usize], a[11usize], a[27usize], a[43usize], a[59usize], a[12usize], - a[28usize], a[44usize], a[60usize], a[13usize], a[29usize], a[45usize], a[61usize], - a[14usize], a[30usize], a[46usize], a[62usize], a[15usize], a[31usize], a[47usize], - a[63usize], - ]; - } - #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { let half = self.splat_u8x32(val); self.combine_u8x32(half, half) @@ -11644,91 +12197,6 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[16usize], - src[20usize], - src[24usize], - src[28usize], - src[32usize], - src[36usize], - src[40usize], - src[44usize], - src[48usize], - src[52usize], - src[56usize], - src[60usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[17usize], - src[21usize], - src[25usize], - src[29usize], - src[33usize], - src[37usize], - src[41usize], - src[45usize], - src[49usize], - src[53usize], - src[57usize], - src[61usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[18usize], - src[22usize], - src[26usize], - src[30usize], - src[34usize], - src[38usize], - src[42usize], - src[46usize], - src[50usize], - src[54usize], - src[58usize], - src[62usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - src[19usize], - src[23usize], - src[27usize], - src[31usize], - src[35usize], - src[39usize], - src[43usize], - src[47usize], - src[51usize], - src[55usize], - src[59usize], - src[63usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - *dest = [ - a[0usize], a[16usize], a[32usize], a[48usize], a[1usize], a[17usize], a[33usize], - a[49usize], a[2usize], a[18usize], a[34usize], a[50usize], a[3usize], a[19usize], - a[35usize], a[51usize], a[4usize], a[20usize], a[36usize], a[52usize], a[5usize], - a[21usize], a[37usize], a[53usize], a[6usize], a[22usize], a[38usize], a[54usize], - a[7usize], a[23usize], a[39usize], a[55usize], a[8usize], a[24usize], a[40usize], - a[56usize], a[9usize], a[25usize], a[41usize], a[57usize], a[10usize], a[26usize], - a[42usize], a[58usize], a[11usize], a[27usize], a[43usize], a[59usize], a[12usize], - a[28usize], a[44usize], a[60usize], a[13usize], a[29usize], a[45usize], a[61usize], - a[14usize], a[30usize], a[46usize], a[62usize], a[15usize], a[31usize], a[47usize], - a[63usize], - ]; - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { let half = self.splat_mask8x32(val); self.combine_mask8x32(half, half) @@ -12254,54 +12722,6 @@ impl Simd for Fallback { self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) } #[inline(always)] - fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[16usize], - src[20usize], - src[24usize], - src[28usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[17usize], - src[21usize], - src[25usize], - src[29usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[18usize], - src[22usize], - src[26usize], - src[30usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - src[19usize], - src[23usize], - src[27usize], - src[31usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { - *dest = [ - a[0usize], a[8usize], a[16usize], a[24usize], a[1usize], a[9usize], a[17usize], - a[25usize], a[2usize], a[10usize], a[18usize], a[26usize], a[3usize], a[11usize], - a[19usize], a[27usize], a[4usize], a[12usize], a[20usize], a[28usize], a[5usize], - a[13usize], a[21usize], a[29usize], a[6usize], a[14usize], a[22usize], a[30usize], - a[7usize], a[15usize], a[23usize], a[31usize], - ]; - } - #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { let half = self.splat_u16x16(val); self.combine_u16x16(half, half) @@ -12710,54 +13130,6 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[16usize], - src[20usize], - src[24usize], - src[28usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[17usize], - src[21usize], - src[25usize], - src[29usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[18usize], - src[22usize], - src[26usize], - src[30usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - src[19usize], - src[23usize], - src[27usize], - src[31usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - *dest = [ - a[0usize], a[8usize], a[16usize], a[24usize], a[1usize], a[9usize], a[17usize], - a[25usize], a[2usize], a[10usize], a[18usize], a[26usize], a[3usize], a[11usize], - a[19usize], a[27usize], a[4usize], a[12usize], a[20usize], a[28usize], a[5usize], - a[13usize], a[21usize], a[29usize], a[6usize], a[14usize], a[22usize], a[30usize], - a[7usize], a[15usize], a[23usize], a[31usize], - ]; - } - #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { let (a0, a1) = self.split_u16x32(a); self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) @@ -13221,36 +13593,6 @@ impl Simd for Fallback { self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] - fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { - *dest = [ - a[0usize], a[4usize], a[8usize], a[12usize], a[1usize], a[5usize], a[9usize], - a[13usize], a[2usize], a[6usize], a[10usize], a[14usize], a[3usize], a[7usize], - a[11usize], a[15usize], - ]; - } - #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { let (a0, a1) = self.split_i32x16(a); self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) @@ -13594,36 +13936,6 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - *dest = [ - a[0usize], a[4usize], a[8usize], a[12usize], a[1usize], a[5usize], a[9usize], - a[13usize], a[2usize], a[6usize], a[10usize], a[14usize], a[3usize], a[7usize], - a[11usize], a[15usize], - ]; - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { let (a0, a1) = self.split_u32x16(a); self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) @@ -14099,26 +14411,6 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] - fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { - [ - src[0usize], - src[4usize], - src[1usize], - src[5usize], - src[2usize], - src[6usize], - src[3usize], - src[7usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { - *dest = [ - a[0usize], a[2usize], a[4usize], a[6usize], a[1usize], a[3usize], a[5usize], a[7usize], - ]; - } - #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) @@ -14426,26 +14718,6 @@ impl Simd for Fallback { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] - fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { - [ - src[0usize], - src[4usize], - src[1usize], - src[5usize], - src[2usize], - src[6usize], - src[3usize], - src[7usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { - *dest = [ - a[0usize], a[2usize], a[4usize], a[6usize], a[1usize], a[3usize], a[5usize], a[7usize], - ]; - } - #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) @@ -14748,26 +15020,6 @@ impl Simd for Fallback { (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - [ - src[0usize], - src[4usize], - src[1usize], - src[5usize], - src[2usize], - src[6usize], - src[3usize], - src[7usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - *dest = [ - a[0usize], a[2usize], a[4usize], a[6usize], a[1usize], a[3usize], a[5usize], a[7usize], - ]; - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index 3e646ae2..dcca19fe 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -566,6 +566,32 @@ impl Simd for Neon { } } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + unsafe { + let native = vld4q_f32(src.as_ptr()); + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + unsafe { + let v0: float32x4_t = vectors[0usize].into(); + let v1: float32x4_t = vectors[1usize].into(); + let v2: float32x4_t = vectors[2usize].into(); + let v3: float32x4_t = vectors[3usize].into(); + vst4q_f32(dest.as_mut_ptr(), float32x4x4_t(v0, v1, v2, v3)) + } + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -1040,6 +1066,32 @@ impl Simd for Neon { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + unsafe { + let native = vld4q_s8(src.as_ptr()); + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + unsafe { + let v0: int8x16_t = vectors[0usize].into(); + let v1: int8x16_t = vectors[1usize].into(); + let v2: int8x16_t = vectors[2usize].into(); + let v3: int8x16_t = vectors[3usize].into(); + vst4q_s8(dest.as_mut_ptr(), int8x16x4_t(v0, v1, v2, v3)) + } + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1476,6 +1528,32 @@ impl Simd for Neon { } } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + unsafe { + let native = vld4q_u8(src.as_ptr()); + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + unsafe { + let v0: uint8x16_t = vectors[0usize].into(); + let v1: uint8x16_t = vectors[1usize].into(); + let v2: uint8x16_t = vectors[2usize].into(); + let v3: uint8x16_t = vectors[3usize].into(); + vst4q_u8(dest.as_mut_ptr(), uint8x16x4_t(v0, v1, v2, v3)) + } + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -2089,6 +2167,32 @@ impl Simd for Neon { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { + unsafe { + let native = vld4q_s16(src.as_ptr()); + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + } + #[inline(always)] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + unsafe { + let v0: int16x8_t = vectors[0usize].into(); + let v1: int16x8_t = vectors[1usize].into(); + let v2: int16x8_t = vectors[2usize].into(); + let v3: int16x8_t = vectors[3usize].into(); + vst4q_s16(dest.as_mut_ptr(), int16x8x4_t(v0, v1, v2, v3)) + } + } + #[inline(always)] fn splat_u16x8(self, val: u16) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2493,6 +2597,32 @@ impl Simd for Neon { } } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + unsafe { + let native = vld4q_u16(src.as_ptr()); + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + unsafe { + let v0: uint16x8_t = vectors[0usize].into(); + let v1: uint16x8_t = vectors[1usize].into(); + let v2: uint16x8_t = vectors[2usize].into(); + let v3: uint16x8_t = vectors[3usize].into(); + vst4q_u16(dest.as_mut_ptr(), uint16x8x4_t(v0, v1, v2, v3)) + } + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { crate::kernel!( #[inline(always)] @@ -3070,6 +3200,32 @@ impl Simd for Neon { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + unsafe { + let native = vld4q_s32(src.as_ptr()); + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + unsafe { + let v0: int32x4_t = vectors[0usize].into(); + let v1: int32x4_t = vectors[1usize].into(); + let v2: int32x4_t = vectors[2usize].into(); + let v3: int32x4_t = vectors[3usize].into(); + vst4q_s32(dest.as_mut_ptr(), int32x4x4_t(v0, v1, v2, v3)) + } + } + #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -3468,6 +3624,32 @@ impl Simd for Neon { } } #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + unsafe { + let native = vld4q_u32(src.as_ptr()); + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + } + #[inline(always)] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + unsafe { + let v0: uint32x4_t = vectors[0usize].into(); + let v1: uint32x4_t = vectors[1usize].into(); + let v2: uint32x4_t = vectors[2usize].into(); + let v3: uint32x4_t = vectors[3usize].into(); + vst4q_u32(dest.as_mut_ptr(), uint32x4x4_t(v0, v1, v2, v3)) + } + } + #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -4109,6 +4291,32 @@ impl Simd for Neon { } } #[inline(always)] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { + unsafe { + let native = vld4q_f64(src.as_ptr()); + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + unsafe { + let v0: float64x2_t = vectors[0usize].into(); + let v1: float64x2_t = vectors[1usize].into(); + let v2: float64x2_t = vectors[2usize].into(); + let v3: float64x2_t = vectors[3usize].into(); + vst4q_f64(dest.as_mut_ptr(), float64x2x4_t(v0, v1, v2, v3)) + } + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { crate::kernel!( #[inline(always)] @@ -4493,6 +4701,32 @@ impl Simd for Neon { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + unsafe { + let native = vld4q_s64(src.as_ptr()); + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + unsafe { + let v0: int64x2_t = vectors[0usize].into(); + let v1: int64x2_t = vectors[1usize].into(); + let v2: int64x2_t = vectors[2usize].into(); + let v3: int64x2_t = vectors[3usize].into(); + vst4q_s64(dest.as_mut_ptr(), int64x2x4_t(v0, v1, v2, v3)) + } + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -4867,6 +5101,32 @@ impl Simd for Neon { } } #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + unsafe { + let native = vld4q_u64(src.as_ptr()); + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] + } + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + unsafe { + let v0: uint64x2_t = vectors[0usize].into(); + let v1: uint64x2_t = vectors[1usize].into(); + let v2: uint64x2_t = vectors[2usize].into(); + let v3: uint64x2_t = vectors[3usize].into(); + vst4q_u64(dest.as_mut_ptr(), uint64x2x4_t(v0, v1, v2, v3)) + } + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { crate::kernel!( #[inline(always)] @@ -9857,14 +10117,6 @@ impl Simd for Neon { ) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - unsafe { vld4q_f32(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - unsafe { vst4q_f32(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) @@ -10475,14 +10727,6 @@ impl Simd for Neon { self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) } #[inline(always)] - fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { - unsafe { vld4q_s8(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { - unsafe { vst4q_s8(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { let half = self.splat_u8x32(val); self.combine_u8x32(half, half) @@ -11062,14 +11306,6 @@ impl Simd for Neon { ) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - unsafe { vld4q_u8(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - unsafe { vst4q_u8(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { let half = self.splat_mask8x32(val); self.combine_mask8x32(half, half) @@ -11653,14 +11889,6 @@ impl Simd for Neon { self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) } #[inline(always)] - fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { - unsafe { vld4q_s16(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { - unsafe { vst4q_s16(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { let half = self.splat_u16x16(val); self.combine_u16x16(half, half) @@ -12122,14 +12350,6 @@ impl Simd for Neon { ) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - unsafe { vld4q_u16(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - unsafe { vst4q_u16(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { let (a0, a1) = self.split_u16x32(a); self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) @@ -12651,14 +12871,6 @@ impl Simd for Neon { self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] - fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { - unsafe { vld4q_s32(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { - unsafe { vst4q_s32(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { let (a0, a1) = self.split_i32x16(a); self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) @@ -13055,14 +13267,6 @@ impl Simd for Neon { ) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - unsafe { vld4q_u32(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - unsafe { vst4q_u32(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { let (a0, a1) = self.split_u32x16(a); self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) @@ -13596,14 +13800,6 @@ impl Simd for Neon { ) } #[inline(always)] - fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { - unsafe { vld4q_f64(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { - unsafe { vst4q_f64(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) @@ -13964,14 +14160,6 @@ impl Simd for Neon { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] - fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { - unsafe { vld4q_s64(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { - unsafe { vst4q_s64(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) @@ -14327,14 +14515,6 @@ impl Simd for Neon { ) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - unsafe { vld4q_u64(src.as_ptr()).simd_into(self) } - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - unsafe { vst4q_u64(dest.as_mut_ptr(), a.into()) } - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index b450a482..a9c3fd67 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -267,6 +267,16 @@ pub trait Simd: fn select_f32x4(self, a: mask32x4, b: f32x4, c: f32x4) -> f32x4; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_f32x4(self, a: f32x4, b: f32x4) -> f32x8; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> (); #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms below AVX-512, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32`.\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4; #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] @@ -369,6 +379,16 @@ pub trait Simd: fn combine_i8x16(self, a: i8x16, b: i8x16) -> i8x32; #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i8x16(self, a: i8x16) -> i8x16; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u8x16(self, val: u8) -> u8x16; #[doc = "Create a SIMD vector from an array of the same length."] @@ -461,6 +481,16 @@ pub trait Simd: fn max_u8x16(self, a: u8x16, b: u8x16) -> u8x16; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> (); #[doc = "Zero-extend each element to a wider integer type.\n\nThe number of elements in the result is half that of the input."] fn widen_u8x16(self, a: u8x16) -> u16x16; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] @@ -596,6 +626,16 @@ pub trait Simd: fn combine_i16x8(self, a: i16x8, b: i16x8) -> i16x16; #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i16x8(self, a: i16x8) -> i16x8; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u16x8(self, val: u16) -> u16x8; #[doc = "Create a SIMD vector from an array of the same length."] @@ -688,6 +728,16 @@ pub trait Simd: fn max_u16x8(self, a: u16x8, b: u16x8) -> u16x8; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_u16x8(self, a: u16x8, b: u16x8) -> u16x16; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> (); #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask16x8(self, val: bool) -> mask16x8; #[doc = "Create a SIMD mask from signed integer mask lanes."] @@ -821,6 +871,16 @@ pub trait Simd: fn combine_i32x4(self, a: i32x4, b: i32x4) -> i32x8; #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i32x4(self, a: i32x4) -> i32x4; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> (); #[doc = "Convert each signed 32-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4; #[doc = "Create a SIMD vector with all elements set to the given value."] @@ -915,6 +975,16 @@ pub trait Simd: fn max_u32x4(self, a: u32x4, b: u32x4) -> u32x4; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_u32x4(self, a: u32x4, b: u32x4) -> u32x8; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> (); #[doc = "Convert each unsigned 32-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] @@ -1062,6 +1132,16 @@ pub trait Simd: fn select_f64x2(self, a: mask64x2, b: f64x2, c: f64x2) -> f64x2; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_f64x2(self, a: f64x2, b: f64x2) -> f64x4; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_i64x2(self, val: i64) -> i64x2; #[doc = "Create a SIMD vector from an array of the same length."] @@ -1156,6 +1236,16 @@ pub trait Simd: fn combine_i64x2(self, a: i64x2, b: i64x2) -> i64x4; #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i64x2(self, a: i64x2) -> i64x2; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u64x2(self, val: u64) -> u64x2; #[doc = "Create a SIMD vector from an array of the same length."] @@ -1248,6 +1338,16 @@ pub trait Simd: fn max_u64x2(self, a: u64x2, b: u64x2) -> u64x2; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_u64x2(self, a: u64x2, b: u64x2) -> u64x4; + #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] + #[doc(alias = "load_interleaved")] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize]; + #[doc = "Store four 128-bit vectors to an array with 4-way interleaving.\n\nThis is the inverse of `load_four_interleaved`.\n\nThis is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\nFor example, with 32-bit lanes, vectors containing `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`."] + #[doc(alias = "store_interleaved")] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> (); #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask64x2(self, val: bool) -> mask64x2; #[doc = "Create a SIMD mask from signed integer mask lanes."] @@ -2561,10 +2661,6 @@ pub trait Simd: fn select_f32x16(self, a: mask32x16, b: f32x16, c: f32x16) -> f32x16; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_f32x16(self, a: f32x16) -> (f32x8, f32x8); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> (); #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values or NaN will produce implementation-defined results.\n\nOn x86 platforms below AVX-512, this operation will still be slower than converting to `i32`, because there is no native instruction for converting to `u32`.\nIf you know your values fit within range of an `i32`, you should convert to an `i32` and cast to your desired datatype afterwards."] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16; #[doc = "Convert each floating-point element to an unsigned 32-bit integer, truncating towards zero.\n\nOut-of-range values are saturated to the closest in-range value. NaN becomes 0."] @@ -2667,10 +2763,6 @@ pub trait Simd: fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32); #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i8x64(self, a: i8x64) -> i8x64; - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u8x64(self, val: u8) -> u8x64; #[doc = "Create a SIMD vector from an array of the same length."] @@ -2763,10 +2855,6 @@ pub trait Simd: fn max_u8x64(self, a: u8x64, b: u8x64) -> u8x64; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> (); #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask8x64(self, val: bool) -> mask8x64; #[doc = "Create a SIMD mask from signed integer mask lanes."] @@ -2904,10 +2992,6 @@ pub trait Simd: fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16); #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i16x32(self, a: i16x32) -> i16x32; - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u16x32(self, val: u16) -> u16x32; #[doc = "Create a SIMD vector from an array of the same length."] @@ -3004,10 +3088,6 @@ pub trait Simd: fn max_u16x32(self, a: u16x32, b: u16x32) -> u16x32; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> (); #[doc = "Truncate each element to a narrower integer type.\n\nThe number of elements in the result is twice that of the input."] fn narrow_u16x32(self, a: u16x32) -> u8x32; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] @@ -3147,10 +3227,6 @@ pub trait Simd: fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8); #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i32x16(self, a: i32x16) -> i32x16; - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> (); #[doc = "Convert each signed 32-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16; #[doc = "Create a SIMD vector with all elements set to the given value."] @@ -3249,10 +3325,6 @@ pub trait Simd: fn max_u32x16(self, a: u32x16, b: u32x16) -> u32x16; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> (); #[doc = "Convert each unsigned 32-bit integer element to a floating-point value.\n\nValues that cannot be exactly represented are rounded to the nearest representable value."] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16; #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] @@ -3400,10 +3472,6 @@ pub trait Simd: fn select_f64x8(self, a: mask64x8, b: f64x8, c: f64x8) -> f64x8; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_f64x8(self, a: f64x8) -> (f64x4, f64x4); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_i64x8(self, val: i64) -> i64x8; #[doc = "Create a SIMD vector from an array of the same length."] @@ -3498,10 +3566,6 @@ pub trait Simd: fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4); #[doc = "Negate each element of the vector, wrapping on overflow."] fn neg_i64x8(self, a: i64x8) -> i64x8; - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> (); #[doc = "Create a SIMD vector with all elements set to the given value."] fn splat_u64x8(self, val: u64) -> u64x8; #[doc = "Create a SIMD vector from an array of the same length."] @@ -3594,10 +3658,6 @@ pub trait Simd: fn max_u64x8(self, a: u64x8, b: u64x8) -> u64x8; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4); - #[doc = "Load elements from an array with 4-way interleaving.\n\nThis is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them into one vector.\n\nFor example, with 32-bit lanes, memory laid out as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`."] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8; - #[doc = "Store elements to an array with 4-way interleaving.\n\nThis is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit vectors into lane-interleaved memory.\n\nFor example, with 32-bit lanes, a vector containing `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`."] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> (); #[doc = "Create a SIMD mask with all lanes set from the given boolean value."] fn splat_mask64x8(self, val: bool) -> mask64x8; #[doc = "Create a SIMD mask from signed integer mask lanes."] diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index 4f1c47e8..8d5c4733 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -689,6 +689,89 @@ impl Simd for Sse2 { } } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[f32; 16usize]) -> [f32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); + let v1: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); + let v2: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); + let v3: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse2, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + let _ = token; + let v0: __m128 = vectors[0].into(); + let v1: __m128 = vectors[1].into(); + let v2: __m128 = vectors[2].into(); + let v3: __m128 = vectors[3].into(); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { [ a[0usize] as u32, @@ -1288,6 +1371,160 @@ impl Simd for Sse2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + src[32usize], + src[36usize], + src[40usize], + src[44usize], + src[48usize], + src[52usize], + src[56usize], + src[60usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + src[33usize], + src[37usize], + src[41usize], + src[45usize], + src[49usize], + src[53usize], + src[57usize], + src[61usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + src[34usize], + src[38usize], + src[42usize], + src[46usize], + src[50usize], + src[54usize], + src[58usize], + src[62usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + src[35usize], + src[39usize], + src[43usize], + src[47usize], + src[51usize], + src[55usize], + src[59usize], + src[63usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + vectors[0usize][8usize], + vectors[1usize][8usize], + vectors[2usize][8usize], + vectors[3usize][8usize], + vectors[0usize][9usize], + vectors[1usize][9usize], + vectors[2usize][9usize], + vectors[3usize][9usize], + vectors[0usize][10usize], + vectors[1usize][10usize], + vectors[2usize][10usize], + vectors[3usize][10usize], + vectors[0usize][11usize], + vectors[1usize][11usize], + vectors[2usize][11usize], + vectors[3usize][11usize], + vectors[0usize][12usize], + vectors[1usize][12usize], + vectors[2usize][12usize], + vectors[3usize][12usize], + vectors[0usize][13usize], + vectors[1usize][13usize], + vectors[2usize][13usize], + vectors[3usize][13usize], + vectors[0usize][14usize], + vectors[1usize][14usize], + vectors[2usize][14usize], + vectors[3usize][14usize], + vectors[0usize][15usize], + vectors[1usize][15usize], + vectors[2usize][15usize], + vectors[3usize][15usize], + ]; + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1847,6 +2084,160 @@ impl Simd for Sse2 { } } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + src[32usize], + src[36usize], + src[40usize], + src[44usize], + src[48usize], + src[52usize], + src[56usize], + src[60usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + src[33usize], + src[37usize], + src[41usize], + src[45usize], + src[49usize], + src[53usize], + src[57usize], + src[61usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + src[34usize], + src[38usize], + src[42usize], + src[46usize], + src[50usize], + src[54usize], + src[58usize], + src[62usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + src[35usize], + src[39usize], + src[43usize], + src[47usize], + src[51usize], + src[55usize], + src[59usize], + src[63usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + vectors[0usize][8usize], + vectors[1usize][8usize], + vectors[2usize][8usize], + vectors[3usize][8usize], + vectors[0usize][9usize], + vectors[1usize][9usize], + vectors[2usize][9usize], + vectors[3usize][9usize], + vectors[0usize][10usize], + vectors[1usize][10usize], + vectors[2usize][10usize], + vectors[3usize][10usize], + vectors[0usize][11usize], + vectors[1usize][11usize], + vectors[2usize][11usize], + vectors[3usize][11usize], + vectors[0usize][12usize], + vectors[1usize][12usize], + vectors[2usize][12usize], + vectors[3usize][12usize], + vectors[0usize][13usize], + vectors[1usize][13usize], + vectors[2usize][13usize], + vectors[3usize][13usize], + vectors[0usize][14usize], + vectors[1usize][14usize], + vectors[2usize][14usize], + vectors[3usize][14usize], + vectors[0usize][15usize], + vectors[1usize][15usize], + vectors[2usize][15usize], + vectors[3usize][15usize], + ]; + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -2513,6 +2904,96 @@ impl Simd for Sse2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + ]; + } + #[inline(always)] fn splat_u16x8(self, val: u16) -> u16x8 { crate::kernel!( #[inline(always)] @@ -3025,6 +3506,96 @@ impl Simd for Sse2 { } } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + [ + [ + src[0usize], + src[4usize], + src[8usize], + src[12usize], + src[16usize], + src[20usize], + src[24usize], + src[28usize], + ] + .simd_into(self), + [ + src[1usize], + src[5usize], + src[9usize], + src[13usize], + src[17usize], + src[21usize], + src[25usize], + src[29usize], + ] + .simd_into(self), + [ + src[2usize], + src[6usize], + src[10usize], + src[14usize], + src[18usize], + src[22usize], + src[26usize], + src[30usize], + ] + .simd_into(self), + [ + src[3usize], + src[7usize], + src[11usize], + src[15usize], + src[19usize], + src[23usize], + src[27usize], + src[31usize], + ] + .simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + *dest = [ + vectors[0usize][0usize], + vectors[1usize][0usize], + vectors[2usize][0usize], + vectors[3usize][0usize], + vectors[0usize][1usize], + vectors[1usize][1usize], + vectors[2usize][1usize], + vectors[3usize][1usize], + vectors[0usize][2usize], + vectors[1usize][2usize], + vectors[2usize][2usize], + vectors[3usize][2usize], + vectors[0usize][3usize], + vectors[1usize][3usize], + vectors[2usize][3usize], + vectors[3usize][3usize], + vectors[0usize][4usize], + vectors[1usize][4usize], + vectors[2usize][4usize], + vectors[3usize][4usize], + vectors[0usize][5usize], + vectors[1usize][5usize], + vectors[2usize][5usize], + vectors[3usize][5usize], + vectors[0usize][6usize], + vectors[1usize][6usize], + vectors[2usize][6usize], + vectors[3usize][6usize], + vectors[0usize][7usize], + vectors[1usize][7usize], + vectors[2usize][7usize], + vectors[3usize][7usize], + ]; + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { crate::kernel!( #[inline(always)] @@ -3671,6 +4242,89 @@ impl Simd for Sse2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[i32; 16usize]) -> [i32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse2, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -4179,6 +4833,89 @@ impl Simd for Sse2 { } } #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[u32; 16usize]) -> [u32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse2, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { [ a[0usize] as f32, @@ -4840,28 +5577,99 @@ impl Simd for Sse2 { } } #[inline(always)] - fn splat_i64x2(self, val: i64) -> i64x2 { + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { crate::kernel!( #[inline(always)] - fn kernel(token: Sse2, val: i64) -> i64x2 { - _mm_set1_epi64x(val).simd_into(token) + fn kernel(token: Sse2, src: &[f64; 8usize]) -> [f64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[0]); + let v1: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[1]); + let v2: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[2]); + let v3: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[3]); + let out0 = _mm_unpacklo_pd(v0, v2); + let out1 = _mm_unpackhi_pd(v0, v2); + let out2 = _mm_unpacklo_pd(v1, v3); + let out3 = _mm_unpackhi_pd(v1, v3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] } ); - kernel(self, val) - } - #[inline(always)] - fn load_array_i64x2(self, val: [i64; 2usize]) -> i64x2 { - i64x2 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } - } - #[inline(always)] - fn load_array_ref_i64x2(self, val: &[i64; 2usize]) -> i64x2 { - i64x2 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, vectors: [f64x2; 4usize], dest: &mut [f64; 8usize]) -> () { + let _ = token; + let v0: __m128d = vectors[0].into(); + let v1: __m128d = vectors[1].into(); + let v2: __m128d = vectors[2].into(); + let v3: __m128d = vectors[3].into(); + let out0 = _mm_unpacklo_pd(v0, v1); + let out1 = _mm_unpacklo_pd(v2, v3); + let out2 = _mm_unpackhi_pd(v0, v1); + let out3 = _mm_unpackhi_pd(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] + fn splat_i64x2(self, val: i64) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, val: i64) -> i64x2 { + _mm_set1_epi64x(val).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_i64x2(self, val: [i64; 2usize]) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_i64x2(self, val: &[i64; 2usize]) -> i64x2 { + i64x2 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } } #[inline(always)] fn as_array_i64x2(self, a: i64x2) -> [i64; 2usize] { @@ -5263,6 +6071,77 @@ impl Simd for Sse2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[i64; 8usize]) -> [i64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[3]); + let out0 = _mm_unpacklo_epi64(v0, v2); + let out1 = _mm_unpackhi_epi64(v0, v2); + let out2 = _mm_unpacklo_epi64(v1, v3); + let out3 = _mm_unpackhi_epi64(v1, v3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, vectors: [i64x2; 4usize], dest: &mut [i64; 8usize]) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let out0 = _mm_unpacklo_epi64(v0, v1); + let out1 = _mm_unpacklo_epi64(v2, v3); + let out2 = _mm_unpackhi_epi64(v0, v1); + let out3 = _mm_unpackhi_epi64(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5678,6 +6557,77 @@ impl Simd for Sse2 { } } #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, src: &[u64; 8usize]) -> [u64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[3]); + let out0 = _mm_unpacklo_epi64(v0, v2); + let out1 = _mm_unpackhi_epi64(v0, v2); + let out2 = _mm_unpacklo_epi64(v1, v3); + let out3 = _mm_unpackhi_epi64(v1, v3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, vectors: [u64x2; 4usize], dest: &mut [u64; 8usize]) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let out0 = _mm_unpacklo_epi64(v0, v1); + let out1 = _mm_unpacklo_epi64(v2, v3); + let out2 = _mm_unpackhi_epi64(v0, v1); + let out3 = _mm_unpackhi_epi64(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { crate::kernel!( #[inline(always)] @@ -10393,81 +11343,6 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, src: &[f32; 16usize]) -> f32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); - let v1: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); - let v2: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); - let v3: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - token.combine_f32x8( - token.combine_f32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_f32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, a: f32x16, dest: &mut [f32; 16usize]) -> () { - let (v01, v23) = token.split_f32x16(a); - let (v0, v1) = token.split_f32x8(v01); - let (v2, v3) = token.split_f32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) @@ -11039,91 +11914,6 @@ impl Simd for Sse2 { self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) } #[inline(always)] - fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[16usize], - src[20usize], - src[24usize], - src[28usize], - src[32usize], - src[36usize], - src[40usize], - src[44usize], - src[48usize], - src[52usize], - src[56usize], - src[60usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[17usize], - src[21usize], - src[25usize], - src[29usize], - src[33usize], - src[37usize], - src[41usize], - src[45usize], - src[49usize], - src[53usize], - src[57usize], - src[61usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[18usize], - src[22usize], - src[26usize], - src[30usize], - src[34usize], - src[38usize], - src[42usize], - src[46usize], - src[50usize], - src[54usize], - src[58usize], - src[62usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - src[19usize], - src[23usize], - src[27usize], - src[31usize], - src[35usize], - src[39usize], - src[43usize], - src[47usize], - src[51usize], - src[55usize], - src[59usize], - src[63usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { - *dest = [ - a[0usize], a[16usize], a[32usize], a[48usize], a[1usize], a[17usize], a[33usize], - a[49usize], a[2usize], a[18usize], a[34usize], a[50usize], a[3usize], a[19usize], - a[35usize], a[51usize], a[4usize], a[20usize], a[36usize], a[52usize], a[5usize], - a[21usize], a[37usize], a[53usize], a[6usize], a[22usize], a[38usize], a[54usize], - a[7usize], a[23usize], a[39usize], a[55usize], a[8usize], a[24usize], a[40usize], - a[56usize], a[9usize], a[25usize], a[41usize], a[57usize], a[10usize], a[26usize], - a[42usize], a[58usize], a[11usize], a[27usize], a[43usize], a[59usize], a[12usize], - a[28usize], a[44usize], a[60usize], a[13usize], a[29usize], a[45usize], a[61usize], - a[14usize], a[30usize], a[46usize], a[62usize], a[15usize], a[31usize], a[47usize], - a[63usize], - ]; - } - #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { let half = self.splat_u8x32(val); self.combine_u8x32(half, half) @@ -11664,91 +12454,6 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[16usize], - src[20usize], - src[24usize], - src[28usize], - src[32usize], - src[36usize], - src[40usize], - src[44usize], - src[48usize], - src[52usize], - src[56usize], - src[60usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[17usize], - src[21usize], - src[25usize], - src[29usize], - src[33usize], - src[37usize], - src[41usize], - src[45usize], - src[49usize], - src[53usize], - src[57usize], - src[61usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[18usize], - src[22usize], - src[26usize], - src[30usize], - src[34usize], - src[38usize], - src[42usize], - src[46usize], - src[50usize], - src[54usize], - src[58usize], - src[62usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - src[19usize], - src[23usize], - src[27usize], - src[31usize], - src[35usize], - src[39usize], - src[43usize], - src[47usize], - src[51usize], - src[55usize], - src[59usize], - src[63usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - *dest = [ - a[0usize], a[16usize], a[32usize], a[48usize], a[1usize], a[17usize], a[33usize], - a[49usize], a[2usize], a[18usize], a[34usize], a[50usize], a[3usize], a[19usize], - a[35usize], a[51usize], a[4usize], a[20usize], a[36usize], a[52usize], a[5usize], - a[21usize], a[37usize], a[53usize], a[6usize], a[22usize], a[38usize], a[54usize], - a[7usize], a[23usize], a[39usize], a[55usize], a[8usize], a[24usize], a[40usize], - a[56usize], a[9usize], a[25usize], a[41usize], a[57usize], a[10usize], a[26usize], - a[42usize], a[58usize], a[11usize], a[27usize], a[43usize], a[59usize], a[12usize], - a[28usize], a[44usize], a[60usize], a[13usize], a[29usize], a[45usize], a[61usize], - a[14usize], a[30usize], a[46usize], a[62usize], a[15usize], a[31usize], a[47usize], - a[63usize], - ]; - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { let half = self.splat_mask8x32(val); self.combine_mask8x32(half, half) @@ -12293,54 +12998,6 @@ impl Simd for Sse2 { self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) } #[inline(always)] - fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[16usize], - src[20usize], - src[24usize], - src[28usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[17usize], - src[21usize], - src[25usize], - src[29usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[18usize], - src[22usize], - src[26usize], - src[30usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - src[19usize], - src[23usize], - src[27usize], - src[31usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { - *dest = [ - a[0usize], a[8usize], a[16usize], a[24usize], a[1usize], a[9usize], a[17usize], - a[25usize], a[2usize], a[10usize], a[18usize], a[26usize], a[3usize], a[11usize], - a[19usize], a[27usize], a[4usize], a[12usize], a[20usize], a[28usize], a[5usize], - a[13usize], a[21usize], a[29usize], a[6usize], a[14usize], a[22usize], a[30usize], - a[7usize], a[15usize], a[23usize], a[31usize], - ]; - } - #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { let half = self.splat_u16x16(val); self.combine_u16x16(half, half) @@ -12763,54 +13420,6 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - [ - src[0usize], - src[4usize], - src[8usize], - src[12usize], - src[16usize], - src[20usize], - src[24usize], - src[28usize], - src[1usize], - src[5usize], - src[9usize], - src[13usize], - src[17usize], - src[21usize], - src[25usize], - src[29usize], - src[2usize], - src[6usize], - src[10usize], - src[14usize], - src[18usize], - src[22usize], - src[26usize], - src[30usize], - src[3usize], - src[7usize], - src[11usize], - src[15usize], - src[19usize], - src[23usize], - src[27usize], - src[31usize], - ] - .simd_into(self) - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - *dest = [ - a[0usize], a[8usize], a[16usize], a[24usize], a[1usize], a[9usize], a[17usize], - a[25usize], a[2usize], a[10usize], a[18usize], a[26usize], a[3usize], a[11usize], - a[19usize], a[27usize], a[4usize], a[12usize], a[20usize], a[28usize], a[5usize], - a[13usize], a[21usize], a[29usize], a[6usize], a[14usize], a[22usize], a[30usize], - a[7usize], a[15usize], a[23usize], a[31usize], - ]; - } - #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { let (a0, a1) = self.split_u16x32(a); self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) @@ -13302,81 +13911,6 @@ impl Simd for Sse2 { self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] - fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, src: &[i32; 16usize]) -> i32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_i32x8( - token.combine_i32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_i32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, a: i32x16, dest: &mut [i32; 16usize]) -> () { - let (v01, v23) = token.split_i32x16(a); - let (v0, v1) = token.split_i32x8(v01); - let (v2, v3) = token.split_i32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { let (a0, a1) = self.split_i32x16(a); self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) @@ -13734,81 +14268,6 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, src: &[u32; 16usize]) -> u32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u32x8( - token.combine_u32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_u32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, a: u32x16, dest: &mut [u32; 16usize]) -> () { - let (v01, v23) = token.split_u32x16(a); - let (v0, v1) = token.split_u32x8(v01); - let (v2, v3) = token.split_u32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { let (a0, a1) = self.split_u32x16(a); self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) @@ -14303,73 +14762,6 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, src: &[f64; 8usize]) -> f64x8 { - let (chunks, []) = src.as_chunks::<2usize>() else { - unreachable!() - }; - let v0: __m128d = - crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[0]); - let v1: __m128d = - crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[1]); - let v2: __m128d = - crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[2]); - let v3: __m128d = - crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[3]); - let out0 = _mm_unpacklo_pd(v0, v2); - let out1 = _mm_unpackhi_pd(v0, v2); - let out2 = _mm_unpacklo_pd(v1, v3); - let out3 = _mm_unpackhi_pd(v1, v3); - token.combine_f64x4( - token.combine_f64x2(out0.simd_into(token), out1.simd_into(token)), - token.combine_f64x2(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, a: f64x8, dest: &mut [f64; 8usize]) -> () { - let (v01, v23) = token.split_f64x8(a); - let (v0, v1) = token.split_f64x4(v01); - let (v2, v3) = token.split_f64x4(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let out0 = _mm_unpacklo_pd(v0, v1); - let out1 = _mm_unpacklo_pd(v2, v3); - let out2 = _mm_unpackhi_pd(v0, v1); - let out3 = _mm_unpackhi_pd(v2, v3); - let (chunks, []) = dest.as_chunks_mut::<2usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) @@ -14691,73 +15083,6 @@ impl Simd for Sse2 { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] - fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, src: &[i64; 8usize]) -> i64x8 { - let (chunks, []) = src.as_chunks::<2usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[3]); - let out0 = _mm_unpacklo_epi64(v0, v2); - let out1 = _mm_unpackhi_epi64(v0, v2); - let out2 = _mm_unpacklo_epi64(v1, v3); - let out3 = _mm_unpackhi_epi64(v1, v3); - token.combine_i64x4( - token.combine_i64x2(out0.simd_into(token), out1.simd_into(token)), - token.combine_i64x2(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, a: i64x8, dest: &mut [i64; 8usize]) -> () { - let (v01, v23) = token.split_i64x8(a); - let (v0, v1) = token.split_i64x4(v01); - let (v2, v3) = token.split_i64x4(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let out0 = _mm_unpacklo_epi64(v0, v1); - let out1 = _mm_unpacklo_epi64(v2, v3); - let out2 = _mm_unpackhi_epi64(v0, v1); - let out3 = _mm_unpackhi_epi64(v2, v3); - let (chunks, []) = dest.as_chunks_mut::<2usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) @@ -15074,73 +15399,6 @@ impl Simd for Sse2 { ) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, src: &[u64; 8usize]) -> u64x8 { - let (chunks, []) = src.as_chunks::<2usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[3]); - let out0 = _mm_unpacklo_epi64(v0, v2); - let out1 = _mm_unpackhi_epi64(v0, v2); - let out2 = _mm_unpacklo_epi64(v1, v3); - let out3 = _mm_unpackhi_epi64(v1, v3); - token.combine_u64x4( - token.combine_u64x2(out0.simd_into(token), out1.simd_into(token)), - token.combine_u64x2(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse2, a: u64x8, dest: &mut [u64; 8usize]) -> () { - let (v01, v23) = token.split_u64x8(a); - let (v0, v1) = token.split_u64x4(v01); - let (v2, v3) = token.split_u64x4(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let out0 = _mm_unpacklo_epi64(v0, v1); - let out1 = _mm_unpacklo_epi64(v2, v3); - let out2 = _mm_unpackhi_epi64(v0, v1); - let out3 = _mm_unpackhi_epi64(v2, v3); - let (chunks, []) = dest.as_chunks_mut::<2usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 69581ab2..590b1c9d 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -552,6 +552,89 @@ impl Simd for Sse4_2 { } } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[f32; 16usize]) -> [f32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); + let v1: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); + let v2: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); + let v3: __m128 = + crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + let _ = token; + let v0: __m128 = vectors[0].into(); + let v1: __m128 = vectors[1].into(); + let v2: __m128 = vectors[2].into(); + let v3: __m128 = vectors[3].into(); + let tmp0 = _mm_unpacklo_ps(v0, v1); + let tmp1 = _mm_unpackhi_ps(v0, v1); + let tmp2 = _mm_unpacklo_ps(v2, v3); + let tmp3 = _mm_unpackhi_ps(v2, v3); + let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); + let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -1114,6 +1197,99 @@ impl Simd for Sse4_2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[i8; 64usize]) -> [i8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1595,6 +1771,99 @@ impl Simd for Sse4_2 { } } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[u8; 64usize]) -> [u8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -2196,37 +2465,130 @@ impl Simd for Sse4_2 { kernel(self, a) } #[inline(always)] - fn splat_u16x8(self, val: u16) -> u16x8 { + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { crate::kernel!( #[inline(always)] - fn kernel(token: Sse4_2, val: u16) -> u16x8 { - _mm_set1_epi16(val.cast_signed()).simd_into(token) + fn kernel(token: Sse4_2, src: &[i16; 32usize]) -> [i16x8; 4usize] { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] } ); - kernel(self, val) - } - #[inline(always)] - fn load_array_u16x8(self, val: [u16; 8usize]) -> u16x8 { - u16x8 { - val: crate::transmute::checked_transmute_copy(&val), - simd: self, - } - } - #[inline(always)] - fn load_array_ref_u16x8(self, val: &[u16; 8usize]) -> u16x8 { - u16x8 { - val: crate::transmute::checked_transmute_copy(val), - simd: self, - } - } - #[inline(always)] - fn as_array_u16x8(self, a: u16x8) -> [u16; 8usize] { - crate::transmute::checked_transmute_copy::<__m128i, [u16; 8usize]>(&a.val.0) + kernel(self, src) } #[inline(always)] - fn as_array_ref_u16x8(self, a: &u16x8) -> &[u16; 8usize] { - crate::transmute::checked_cast_ref::<__m128i, [u16; 8usize]>(&a.val.0) - } + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] + fn splat_u16x8(self, val: u16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, val: u16) -> u16x8 { + _mm_set1_epi16(val.cast_signed()).simd_into(token) + } + ); + kernel(self, val) + } + #[inline(always)] + fn load_array_u16x8(self, val: [u16; 8usize]) -> u16x8 { + u16x8 { + val: crate::transmute::checked_transmute_copy(&val), + simd: self, + } + } + #[inline(always)] + fn load_array_ref_u16x8(self, val: &[u16; 8usize]) -> u16x8 { + u16x8 { + val: crate::transmute::checked_transmute_copy(val), + simd: self, + } + } + #[inline(always)] + fn as_array_u16x8(self, a: u16x8) -> [u16; 8usize] { + crate::transmute::checked_transmute_copy::<__m128i, [u16; 8usize]>(&a.val.0) + } + #[inline(always)] + fn as_array_ref_u16x8(self, a: &u16x8) -> &[u16; 8usize] { + crate::transmute::checked_cast_ref::<__m128i, [u16; 8usize]>(&a.val.0) + } #[inline(always)] fn as_array_mut_u16x8(self, a: &mut u16x8) -> &mut [u16; 8usize] { crate::transmute::checked_cast_mut::<__m128i, [u16; 8usize]>(&mut a.val.0) @@ -2612,6 +2974,99 @@ impl Simd for Sse4_2 { } } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[u16; 32usize]) -> [u16x8; 4usize] { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + let out0 = _mm_shuffle_epi8(out0, mask); + let out1 = _mm_shuffle_epi8(out1, mask); + let out2 = _mm_shuffle_epi8(out2, mask); + let out3 = _mm_shuffle_epi8(out3, mask); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { crate::kernel!( #[inline(always)] @@ -3172,6 +3627,89 @@ impl Simd for Sse4_2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[i32; 16usize]) -> [i32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -3572,6 +4110,89 @@ impl Simd for Sse4_2 { } } #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[u32; 16usize]) -> [u32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -4178,14 +4799,89 @@ impl Simd for Sse4_2 { _mm_blendv_pd(c.into(), b.into(), _mm_castsi128_pd(a.into())).simd_into(token) } ); - kernel(self, a, b, c) - } - #[inline(always)] - fn combine_f64x2(self, a: f64x2, b: f64x2) -> f64x4 { - f64x4 { - val: crate::support::Aligned256([a.val.0, b.val.0]), - simd: self, - } + kernel(self, a, b, c) + } + #[inline(always)] + fn combine_f64x2(self, a: f64x2, b: f64x2) -> f64x4 { + f64x4 { + val: crate::support::Aligned256([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[f64; 8usize]) -> [f64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[0]); + let v1: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[1]); + let v2: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[2]); + let v3: __m128d = + crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[3]); + let out0 = _mm_unpacklo_pd(v0, v2); + let out1 = _mm_unpackhi_pd(v0, v2); + let out2 = _mm_unpacklo_pd(v1, v3); + let out3 = _mm_unpackhi_pd(v1, v3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + let _ = token; + let v0: __m128d = vectors[0].into(); + let v1: __m128d = vectors[1].into(); + let v2: __m128d = vectors[2].into(); + let v3: __m128d = vectors[3].into(); + let out0 = _mm_unpacklo_pd(v0, v1); + let out1 = _mm_unpacklo_pd(v2, v3); + let out2 = _mm_unpackhi_pd(v0, v1); + let out3 = _mm_unpackhi_pd(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); } #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { @@ -4544,6 +5240,81 @@ impl Simd for Sse4_2 { kernel(self, a) } #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[i64; 8usize]) -> [i64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[3]); + let out0 = _mm_unpacklo_epi64(v0, v2); + let out1 = _mm_unpackhi_epi64(v0, v2); + let out2 = _mm_unpacklo_epi64(v1, v3); + let out3 = _mm_unpackhi_epi64(v1, v3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let out0 = _mm_unpacklo_epi64(v0, v1); + let out1 = _mm_unpacklo_epi64(v2, v3); + let out2 = _mm_unpackhi_epi64(v0, v1); + let out3 = _mm_unpackhi_epi64(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { crate::kernel!( #[inline(always)] @@ -4892,6 +5663,81 @@ impl Simd for Sse4_2 { } } #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, src: &[u64; 8usize]) -> [u64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[3]); + let out0 = _mm_unpacklo_epi64(v0, v2); + let out1 = _mm_unpackhi_epi64(v0, v2); + let out2 = _mm_unpacklo_epi64(v1, v3); + let out3 = _mm_unpackhi_epi64(v1, v3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] + } + ); + kernel(self, src) + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + let _ = token; + let v0: __m128i = vectors[0].into(); + let v1: __m128i = vectors[1].into(); + let v2: __m128i = vectors[2].into(); + let v3: __m128i = vectors[3].into(); + let out0 = _mm_unpacklo_epi64(v0, v1); + let out1 = _mm_unpacklo_epi64(v2, v3); + let out2 = _mm_unpackhi_epi64(v0, v1); + let out3 = _mm_unpackhi_epi64(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out0, + &mut chunks[0], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out1, + &mut chunks[1], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out2, + &mut chunks[2], + ); + crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( + out3, + &mut chunks[3], + ); + } + ); + kernel(self, vectors, dest); + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { crate::kernel!( #[inline(always)] @@ -9605,81 +10451,6 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[f32; 16usize]) -> f32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]); - let v1: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]); - let v2: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]); - let v3: __m128 = - crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - token.combine_f32x8( - token.combine_f32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_f32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: f32x16, dest: &mut [f32; 16usize]) -> () { - let (v01, v23) = token.split_f32x16(a); - let (v0, v1) = token.split_f32x8(v01); - let (v2, v3) = token.split_f32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_ps(v0, v1); - let tmp1 = _mm_unpackhi_ps(v0, v1); - let tmp2 = _mm_unpacklo_ps(v2, v3); - let tmp3 = _mm_unpackhi_ps(v2, v3); - let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2))); - let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3))); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) @@ -10241,99 +11012,14 @@ impl Simd for Sse4_2 { }, i8x32 { val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), - simd: self, - }, - ) - } - #[inline(always)] - fn neg_i8x64(self, a: i8x64) -> i8x64 { - let (a0, a1) = self.split_i8x64(a); - self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) - } - #[inline(always)] - fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[i8; 64usize]) -> i8x64 { - let (chunks, []) = src.as_chunks::<16usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_i8x32( - token.combine_i8x16(out0.simd_into(token), out1.simd_into(token)), - token.combine_i8x16(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: i8x64, dest: &mut [i8; 64usize]) -> () { - let (v01, v23) = token.split_i8x64(a); - let (v0, v1) = token.split_i8x32(v01); - let (v2, v3) = token.split_i8x32(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let out0 = _mm_shuffle_epi8(out0, mask); - let out1 = _mm_shuffle_epi8(out1, mask); - let out2 = _mm_shuffle_epi8(out2, mask); - let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<16usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); + simd: self, + }, + ) + } + #[inline(always)] + fn neg_i8x64(self, a: i8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) } #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { @@ -10876,91 +11562,6 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[u8; 64usize]) -> u8x64 { - let (chunks, []) = src.as_chunks::<16usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u8x32( - token.combine_u8x16(out0.simd_into(token), out1.simd_into(token)), - token.combine_u8x16(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: u8x64, dest: &mut [u8; 64usize]) -> () { - let (v01, v23) = token.split_u8x64(a); - let (v0, v1) = token.split_u8x32(v01); - let (v2, v3) = token.split_u8x32(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let out0 = _mm_shuffle_epi8(out0, mask); - let out1 = _mm_shuffle_epi8(out1, mask); - let out2 = _mm_shuffle_epi8(out2, mask); - let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<16usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { let half = self.splat_mask8x32(val); self.combine_mask8x32(half, half) @@ -11546,91 +12147,6 @@ impl Simd for Sse4_2 { self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) } #[inline(always)] - fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[i16; 32usize]) -> i16x32 { - let (chunks, []) = src.as_chunks::<8usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_i16x16( - token.combine_i16x8(out0.simd_into(token), out1.simd_into(token)), - token.combine_i16x8(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: i16x32, dest: &mut [i16; 32usize]) -> () { - let (v01, v23) = token.split_i16x32(a); - let (v0, v1) = token.split_i16x16(v01); - let (v2, v3) = token.split_i16x16(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); - let out0 = _mm_shuffle_epi8(out0, mask); - let out1 = _mm_shuffle_epi8(out1, mask); - let out2 = _mm_shuffle_epi8(out2, mask); - let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<8usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { let half = self.splat_u16x16(val); self.combine_u16x16(half, half) @@ -12053,91 +12569,6 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[u16; 32usize]) -> u16x32 { - let (chunks, []) = src.as_chunks::<8usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u16x16( - token.combine_u16x8(out0.simd_into(token), out1.simd_into(token)), - token.combine_u16x8(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: u16x32, dest: &mut [u16; 32usize]) -> () { - let (v01, v23) = token.split_u16x32(a); - let (v0, v1) = token.split_u16x16(v01); - let (v2, v3) = token.split_u16x16(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); - let out0 = _mm_shuffle_epi8(out0, mask); - let out1 = _mm_shuffle_epi8(out1, mask); - let out2 = _mm_shuffle_epi8(out2, mask); - let out3 = _mm_shuffle_epi8(out3, mask); - let (chunks, []) = dest.as_chunks_mut::<8usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { let (a0, a1) = self.split_u16x32(a); self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) @@ -12616,92 +13047,17 @@ impl Simd for Sse4_2 { i32x8 { val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]), simd: self, - }, - i32x8 { - val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), - simd: self, - }, - ) - } - #[inline(always)] - fn neg_i32x16(self, a: i32x16) -> i32x16 { - let (a0, a1) = self.split_i32x16(a); - self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) - } - #[inline(always)] - fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[i32; 16usize]) -> i32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_i32x8( - token.combine_i32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_i32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) + }, + i32x8 { + val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]), + simd: self, + }, + ) } #[inline(always)] - fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: i32x16, dest: &mut [i32; 16usize]) -> () { - let (v01, v23) = token.split_i32x16(a); - let (v0, v1) = token.split_i32x8(v01); - let (v2, v3) = token.split_i32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); + fn neg_i32x16(self, a: i32x16) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { @@ -13061,81 +13417,6 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[u32; 16usize]) -> u32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - token.combine_u32x8( - token.combine_u32x4(out0.simd_into(token), out1.simd_into(token)), - token.combine_u32x4(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: u32x16, dest: &mut [u32; 16usize]) -> () { - let (v01, v23) = token.split_u32x16(a); - let (v0, v1) = token.split_u32x8(v01); - let (v2, v3) = token.split_u32x8(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { let (a0, a1) = self.split_u32x16(a); self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) @@ -13630,73 +13911,6 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[f64; 8usize]) -> f64x8 { - let (chunks, []) = src.as_chunks::<2usize>() else { - unreachable!() - }; - let v0: __m128d = - crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[0]); - let v1: __m128d = - crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[1]); - let v2: __m128d = - crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[2]); - let v3: __m128d = - crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[3]); - let out0 = _mm_unpacklo_pd(v0, v2); - let out1 = _mm_unpackhi_pd(v0, v2); - let out2 = _mm_unpacklo_pd(v1, v3); - let out3 = _mm_unpackhi_pd(v1, v3); - token.combine_f64x4( - token.combine_f64x2(out0.simd_into(token), out1.simd_into(token)), - token.combine_f64x2(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: f64x8, dest: &mut [f64; 8usize]) -> () { - let (v01, v23) = token.split_f64x8(a); - let (v0, v1) = token.split_f64x4(v01); - let (v2, v3) = token.split_f64x4(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let out0 = _mm_unpacklo_pd(v0, v1); - let out1 = _mm_unpacklo_pd(v2, v3); - let out2 = _mm_unpackhi_pd(v0, v1); - let out3 = _mm_unpackhi_pd(v2, v3); - let (chunks, []) = dest.as_chunks_mut::<2usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) @@ -14018,73 +14232,6 @@ impl Simd for Sse4_2 { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] - fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[i64; 8usize]) -> i64x8 { - let (chunks, []) = src.as_chunks::<2usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[3]); - let out0 = _mm_unpacklo_epi64(v0, v2); - let out1 = _mm_unpackhi_epi64(v0, v2); - let out2 = _mm_unpacklo_epi64(v1, v3); - let out3 = _mm_unpackhi_epi64(v1, v3); - token.combine_i64x4( - token.combine_i64x2(out0.simd_into(token), out1.simd_into(token)), - token.combine_i64x2(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: i64x8, dest: &mut [i64; 8usize]) -> () { - let (v01, v23) = token.split_i64x8(a); - let (v0, v1) = token.split_i64x4(v01); - let (v2, v3) = token.split_i64x4(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let out0 = _mm_unpacklo_epi64(v0, v1); - let out1 = _mm_unpacklo_epi64(v2, v3); - let out2 = _mm_unpackhi_epi64(v0, v1); - let out3 = _mm_unpackhi_epi64(v2, v3); - let (chunks, []) = dest.as_chunks_mut::<2usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) @@ -14401,73 +14548,6 @@ impl Simd for Sse4_2 { ) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[u64; 8usize]) -> u64x8 { - let (chunks, []) = src.as_chunks::<2usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[3]); - let out0 = _mm_unpacklo_epi64(v0, v2); - let out1 = _mm_unpackhi_epi64(v0, v2); - let out2 = _mm_unpacklo_epi64(v1, v3); - let out3 = _mm_unpackhi_epi64(v1, v3); - token.combine_u64x4( - token.combine_u64x2(out0.simd_into(token), out1.simd_into(token)), - token.combine_u64x2(out2.simd_into(token), out3.simd_into(token)), - ) - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, a: u64x8, dest: &mut [u64; 8usize]) -> () { - let (v01, v23) = token.split_u64x8(a); - let (v0, v1) = token.split_u64x4(v01); - let (v2, v3) = token.split_u64x4(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); - let out0 = _mm_unpacklo_epi64(v0, v1); - let out1 = _mm_unpacklo_epi64(v2, v3); - let out2 = _mm_unpackhi_epi64(v0, v1); - let out3 = _mm_unpackhi_epi64(v2, v3); - let (chunks, []) = dest.as_chunks_mut::<2usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out0, - &mut chunks[0], - ); - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out1, - &mut chunks[1], - ); - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out2, - &mut chunks[2], - ); - crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>( - out3, - &mut chunks[3], - ); - } - ); - kernel(self, a, dest); - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index 55ddfa6f..667beeeb 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -394,6 +394,56 @@ impl Simd for WasmSimd128 { } } #[inline(always)] + fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[3]); + let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); + let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); + let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); + let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); + let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); + let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); + let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); + let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_f32x4( + self, + vectors: [f32x4; 4usize], + dest: &mut [f32; 16usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); + let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); + let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); + let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); + let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); + let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); + let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); + let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn cvt_u32_f32x4(self, a: f32x4) -> u32x4 { #[cfg(target_feature = "relaxed-simd")] { @@ -755,6 +805,80 @@ impl Simd for WasmSimd128 { i8x16_neg(a.into()).simd_into(self) } #[inline(always)] + fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[3]); + let v01_lower = + u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v0, v1); + let v23_lower = + u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v2, v3); + let v01_upper = + u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v0, v1); + let v23_upper = + u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v2, v3); + let out0 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( + v01_lower, v23_lower, + ); + let out1 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( + v01_lower, v23_lower, + ); + let out2 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( + v01_upper, v23_upper, + ); + let out3 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( + v01_upper, v23_upper, + ); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i8x16( + self, + vectors: [i8x16; 4usize], + dest: &mut [i8; 64usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = + u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v0, v2); + let v13_lower = + u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v1, v3); + let v02_upper = + u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v0, v2); + let v13_upper = + u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v1, v3); + let out0 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( + v02_lower, v13_lower, + ); + let out1 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( + v02_lower, v13_lower, + ); + let out2 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( + v02_upper, v13_upper, + ); + let out3 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( + v02_upper, v13_upper, + ); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_u8x16(self, val: u8) -> u8x16 { u8x16_splat(val).simd_into(self) } @@ -1082,6 +1206,80 @@ impl Simd for WasmSimd128 { } } #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[3]); + let v01_lower = + u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v0, v1); + let v23_lower = + u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v2, v3); + let v01_upper = + u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v0, v1); + let v23_upper = + u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v2, v3); + let out0 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( + v01_lower, v23_lower, + ); + let out1 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( + v01_lower, v23_lower, + ); + let out2 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( + v01_upper, v23_upper, + ); + let out3 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( + v01_upper, v23_upper, + ); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = + u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v0, v2); + let v13_lower = + u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v1, v3); + let v02_upper = + u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v0, v2); + let v13_upper = + u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v1, v3); + let out0 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( + v02_lower, v13_lower, + ); + let out1 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( + v02_lower, v13_lower, + ); + let out2 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( + v02_upper, v13_upper, + ); + let out3 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( + v02_upper, v13_upper, + ); + let (chunks, []) = dest.as_chunks_mut::<16usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn widen_u8x16(self, a: u8x16) -> u16x16 { let low = u16x8_extend_low_u8x16(a.into()); let high = u16x8_extend_high_u8x16(a.into()); @@ -1458,6 +1656,56 @@ impl Simd for WasmSimd128 { i16x8_neg(a.into()).simd_into(self) } #[inline(always)] + fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[3]); + let v01_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v0, v1); + let v23_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v2, v3); + let v01_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v0, v1); + let v23_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v2, v3); + let out0 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_lower, v23_lower); + let out1 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_lower, v23_lower); + let out2 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_upper, v23_upper); + let out3 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_upper, v23_upper); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i16x8( + self, + vectors: [i16x8; 4usize], + dest: &mut [i16; 32usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v0, v2); + let v13_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v1, v3); + let v02_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v0, v2); + let v13_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v1, v3); + let out0 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_lower, v13_lower); + let out1 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_lower, v13_lower); + let out2 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_upper, v13_upper); + let out3 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_u16x8(self, val: u16) -> u16x8 { u16x8_splat(val).simd_into(self) } @@ -1725,6 +1973,56 @@ impl Simd for WasmSimd128 { } } #[inline(always)] + fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { + let (chunks, []) = src.as_chunks::<8usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[3]); + let v01_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v0, v1); + let v23_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v2, v3); + let v01_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v0, v1); + let v23_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v2, v3); + let out0 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_lower, v23_lower); + let out1 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_lower, v23_lower); + let out2 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_upper, v23_upper); + let out3 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_upper, v23_upper); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u16x8( + self, + vectors: [u16x8; 4usize], + dest: &mut [u16; 32usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v0, v2); + let v13_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v1, v3); + let v02_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v0, v2); + let v13_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v1, v3); + let out0 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_lower, v13_lower); + let out1 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_lower, v13_lower); + let out2 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_upper, v13_upper); + let out3 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<8usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_mask16x8(self, val: bool) -> mask16x8 { let val: i16 = if val { !0 } else { 0 }; i16x8_splat(val).simd_into(self) @@ -2069,6 +2367,56 @@ impl Simd for WasmSimd128 { i32x4_neg(a.into()).simd_into(self) } #[inline(always)] + fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[3]); + let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); + let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); + let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); + let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); + let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); + let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); + let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); + let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i32x4( + self, + vectors: [i32x4; 4usize], + dest: &mut [i32; 16usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); + let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); + let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); + let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); + let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); + let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); + let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); + let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn cvt_f32_i32x4(self, a: i32x4) -> f32x4 { f32x4_convert_i32x4(a.into()).simd_into(self) } @@ -2316,6 +2664,56 @@ impl Simd for WasmSimd128 { } } #[inline(always)] + fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { + let (chunks, []) = src.as_chunks::<4usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[3]); + let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); + let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); + let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); + let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); + let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); + let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); + let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); + let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u32x4( + self, + vectors: [u32x4; 4usize], + dest: &mut [u32; 16usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); + let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); + let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); + let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); + let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); + let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); + let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); + let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); + let (chunks, []) = dest.as_chunks_mut::<4usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn cvt_f32_u32x4(self, a: u32x4) -> f32x4 { f32x4_convert_u32x4(a.into()).simd_into(self) } @@ -2703,6 +3101,48 @@ impl Simd for WasmSimd128 { } } #[inline(always)] + fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[3]); + let out0 = u64x2_shuffle::<0, 2>(v0, v2); + let out1 = u64x2_shuffle::<1, 3>(v0, v2); + let out2 = u64x2_shuffle::<0, 2>(v1, v3); + let out3 = u64x2_shuffle::<1, 3>(v1, v3); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_f64x2( + self, + vectors: [f64x2; 4usize], + dest: &mut [f64; 8usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let out0 = u64x2_shuffle::<0, 2>(v0, v1); + let out1 = u64x2_shuffle::<0, 2>(v2, v3); + let out2 = u64x2_shuffle::<1, 3>(v0, v1); + let out3 = u64x2_shuffle::<1, 3>(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { i64x2_splat(val).simd_into(self) } @@ -2946,6 +3386,48 @@ impl Simd for WasmSimd128 { i64x2_neg(a.into()).simd_into(self) } #[inline(always)] + fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[3]); + let out0 = u64x2_shuffle::<0, 2>(v0, v2); + let out1 = u64x2_shuffle::<1, 3>(v0, v2); + let out2 = u64x2_shuffle::<0, 2>(v1, v3); + let out3 = u64x2_shuffle::<1, 3>(v1, v3); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_i64x2( + self, + vectors: [i64x2; 4usize], + dest: &mut [i64; 8usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let out0 = u64x2_shuffle::<0, 2>(v0, v1); + let out1 = u64x2_shuffle::<0, 2>(v2, v3); + let out2 = u64x2_shuffle::<1, 3>(v0, v1); + let out3 = u64x2_shuffle::<1, 3>(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { u64x2_splat(val).simd_into(self) } @@ -3205,6 +3687,48 @@ impl Simd for WasmSimd128 { } } #[inline(always)] + fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { + let (chunks, []) = src.as_chunks::<2usize>() else { + unreachable!() + }; + let v0: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[0]); + let v1: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[1]); + let v2: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[2]); + let v3: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[3]); + let out0 = u64x2_shuffle::<0, 2>(v0, v2); + let out1 = u64x2_shuffle::<1, 3>(v0, v2); + let out2 = u64x2_shuffle::<0, 2>(v1, v3); + let out3 = u64x2_shuffle::<1, 3>(v1, v3); + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] + } + #[inline(always)] + fn store_four_interleaved_u64x2( + self, + vectors: [u64x2; 4usize], + dest: &mut [u64; 8usize], + ) -> () { + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); + let out0 = u64x2_shuffle::<0, 2>(v0, v1); + let out1 = u64x2_shuffle::<0, 2>(v2, v3); + let out2 = u64x2_shuffle::<1, 3>(v0, v1); + let out3 = u64x2_shuffle::<1, 3>(v2, v3); + let (chunks, []) = dest.as_chunks_mut::<2usize>() else { + unreachable!() + }; + crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); + crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); + crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); + crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); + } + #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { let val: i64 = if val { !0 } else { 0 }; i64x2_splat(val).simd_into(self) @@ -7817,52 +8341,6 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[3]); - let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); - let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); - let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); - let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); - let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); - let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); - let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); - let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); - let combined_lower = self.combine_f32x4(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_f32x4(out2.simd_into(self), out3.simd_into(self)); - self.combine_f32x8(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_f32x16(self, a: f32x16, dest: &mut [f32; 16usize]) -> () { - let (lower, upper) = self.split_f32x16(a); - let (v0_vec, v1_vec) = self.split_f32x8(lower); - let (v2_vec, v3_vec) = self.split_f32x8(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); - let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); - let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); - let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); - let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); - let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); - let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); - let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn cvt_u32_f32x16(self, a: f32x16) -> u32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_u32x8(self.cvt_u32_f32x8(a0), self.cvt_u32_f32x8(a1)) @@ -8430,76 +8908,6 @@ impl Simd for WasmSimd128 { self.combine_i8x32(self.neg_i8x32(a0), self.neg_i8x32(a1)) } #[inline(always)] - fn load_interleaved_128_i8x64(self, src: &[i8; 64usize]) -> i8x64 { - let (chunks, []) = src.as_chunks::<16usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[3]); - let v01_lower = - u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v0, v1); - let v23_lower = - u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v2, v3); - let v01_upper = - u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v0, v1); - let v23_upper = - u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v2, v3); - let out0 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( - v01_lower, v23_lower, - ); - let out1 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( - v01_lower, v23_lower, - ); - let out2 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( - v01_upper, v23_upper, - ); - let out3 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( - v01_upper, v23_upper, - ); - let combined_lower = self.combine_i8x16(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_i8x16(out2.simd_into(self), out3.simd_into(self)); - self.combine_i8x32(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_i8x64(self, a: i8x64, dest: &mut [i8; 64usize]) -> () { - let (lower, upper) = self.split_i8x64(a); - let (v0_vec, v1_vec) = self.split_i8x32(lower); - let (v2_vec, v3_vec) = self.split_i8x32(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let v02_lower = - u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v0, v2); - let v13_lower = - u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v1, v3); - let v02_upper = - u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v0, v2); - let v13_upper = - u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v1, v3); - let out0 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( - v02_lower, v13_lower, - ); - let out1 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( - v02_lower, v13_lower, - ); - let out2 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( - v02_upper, v13_upper, - ); - let out3 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( - v02_upper, v13_upper, - ); - let (chunks, []) = dest.as_chunks_mut::<16usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn splat_u8x64(self, val: u8) -> u8x64 { let half = self.splat_u8x32(val); self.combine_u8x32(half, half) @@ -9036,76 +9444,6 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64 { - let (chunks, []) = src.as_chunks::<16usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[3]); - let v01_lower = - u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v0, v1); - let v23_lower = - u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v2, v3); - let v01_upper = - u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v0, v1); - let v23_upper = - u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v2, v3); - let out0 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( - v01_lower, v23_lower, - ); - let out1 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( - v01_lower, v23_lower, - ); - let out2 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>( - v01_upper, v23_upper, - ); - let out3 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>( - v01_upper, v23_upper, - ); - let combined_lower = self.combine_u8x16(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_u8x16(out2.simd_into(self), out3.simd_into(self)); - self.combine_u8x32(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_u8x64(self, a: u8x64, dest: &mut [u8; 64usize]) -> () { - let (lower, upper) = self.split_u8x64(a); - let (v0_vec, v1_vec) = self.split_u8x32(lower); - let (v2_vec, v3_vec) = self.split_u8x32(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let v02_lower = - u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v0, v2); - let v13_lower = - u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v1, v3); - let v02_upper = - u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v0, v2); - let v13_upper = - u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v1, v3); - let out0 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( - v02_lower, v13_lower, - ); - let out1 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( - v02_lower, v13_lower, - ); - let out2 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( - v02_upper, v13_upper, - ); - let out3 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( - v02_upper, v13_upper, - ); - let (chunks, []) = dest.as_chunks_mut::<16usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn splat_mask8x64(self, val: bool) -> mask8x64 { let half = self.splat_mask8x32(val); self.combine_mask8x32(half, half) @@ -9649,52 +9987,6 @@ impl Simd for WasmSimd128 { self.combine_i16x16(self.neg_i16x16(a0), self.neg_i16x16(a1)) } #[inline(always)] - fn load_interleaved_128_i16x32(self, src: &[i16; 32usize]) -> i16x32 { - let (chunks, []) = src.as_chunks::<8usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[3]); - let v01_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v0, v1); - let v23_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v2, v3); - let v01_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v0, v1); - let v23_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v2, v3); - let out0 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_lower, v23_lower); - let out1 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_lower, v23_lower); - let out2 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_upper, v23_upper); - let out3 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_upper, v23_upper); - let combined_lower = self.combine_i16x8(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_i16x8(out2.simd_into(self), out3.simd_into(self)); - self.combine_i16x16(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_i16x32(self, a: i16x32, dest: &mut [i16; 32usize]) -> () { - let (lower, upper) = self.split_i16x32(a); - let (v0_vec, v1_vec) = self.split_i16x16(lower); - let (v2_vec, v3_vec) = self.split_i16x16(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let v02_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v0, v2); - let v13_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v1, v3); - let v02_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v0, v2); - let v13_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v1, v3); - let out0 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_lower, v13_lower); - let out1 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_lower, v13_lower); - let out2 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_upper, v13_upper); - let out3 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_upper, v13_upper); - let (chunks, []) = dest.as_chunks_mut::<8usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn splat_u16x32(self, val: u16) -> u16x32 { let half = self.splat_u16x16(val); self.combine_u16x16(half, half) @@ -10116,52 +10408,6 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32 { - let (chunks, []) = src.as_chunks::<8usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[3]); - let v01_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v0, v1); - let v23_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v2, v3); - let v01_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v0, v1); - let v23_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v2, v3); - let out0 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_lower, v23_lower); - let out1 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_lower, v23_lower); - let out2 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_upper, v23_upper); - let out3 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_upper, v23_upper); - let combined_lower = self.combine_u16x8(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_u16x8(out2.simd_into(self), out3.simd_into(self)); - self.combine_u16x16(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_u16x32(self, a: u16x32, dest: &mut [u16; 32usize]) -> () { - let (lower, upper) = self.split_u16x32(a); - let (v0_vec, v1_vec) = self.split_u16x16(lower); - let (v2_vec, v3_vec) = self.split_u16x16(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let v02_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v0, v2); - let v13_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v1, v3); - let v02_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v0, v2); - let v13_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v1, v3); - let out0 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_lower, v13_lower); - let out1 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_lower, v13_lower); - let out2 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_upper, v13_upper); - let out3 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_upper, v13_upper); - let (chunks, []) = dest.as_chunks_mut::<8usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn narrow_u16x32(self, a: u16x32) -> u8x32 { let (a0, a1) = self.split_u16x32(a); self.combine_u8x16(self.narrow_u16x16(a0), self.narrow_u16x16(a1)) @@ -10643,52 +10889,6 @@ impl Simd for WasmSimd128 { self.combine_i32x8(self.neg_i32x8(a0), self.neg_i32x8(a1)) } #[inline(always)] - fn load_interleaved_128_i32x16(self, src: &[i32; 16usize]) -> i32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[3]); - let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); - let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); - let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); - let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); - let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); - let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); - let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); - let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); - let combined_lower = self.combine_i32x4(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_i32x4(out2.simd_into(self), out3.simd_into(self)); - self.combine_i32x8(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_i32x16(self, a: i32x16, dest: &mut [i32; 16usize]) -> () { - let (lower, upper) = self.split_i32x16(a); - let (v0_vec, v1_vec) = self.split_i32x8(lower); - let (v2_vec, v3_vec) = self.split_i32x8(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); - let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); - let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); - let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); - let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); - let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); - let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); - let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn cvt_f32_i32x16(self, a: i32x16) -> f32x16 { let (a0, a1) = self.split_i32x16(a); self.combine_f32x8(self.cvt_f32_i32x8(a0), self.cvt_f32_i32x8(a1)) @@ -11045,52 +11245,6 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16 { - let (chunks, []) = src.as_chunks::<4usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[3]); - let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1); - let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3); - let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1); - let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3); - let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower); - let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower); - let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper); - let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper); - let combined_lower = self.combine_u32x4(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_u32x4(out2.simd_into(self), out3.simd_into(self)); - self.combine_u32x8(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_u32x16(self, a: u32x16, dest: &mut [u32; 16usize]) -> () { - let (lower, upper) = self.split_u32x16(a); - let (v0_vec, v1_vec) = self.split_u32x8(lower); - let (v2_vec, v3_vec) = self.split_u32x8(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2); - let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3); - let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2); - let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3); - let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower); - let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower); - let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper); - let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper); - let (chunks, []) = dest.as_chunks_mut::<4usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn cvt_f32_u32x16(self, a: u32x16) -> f32x16 { let (a0, a1) = self.split_u32x16(a); self.combine_f32x8(self.cvt_f32_u32x8(a0), self.cvt_f32_u32x8(a1)) @@ -11584,44 +11738,6 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn load_interleaved_128_f64x8(self, src: &[f64; 8usize]) -> f64x8 { - let (chunks, []) = src.as_chunks::<2usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[3]); - let out0 = u64x2_shuffle::<0, 2>(v0, v2); - let out1 = u64x2_shuffle::<1, 3>(v0, v2); - let out2 = u64x2_shuffle::<0, 2>(v1, v3); - let out3 = u64x2_shuffle::<1, 3>(v1, v3); - let combined_lower = self.combine_f64x2(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_f64x2(out2.simd_into(self), out3.simd_into(self)); - self.combine_f64x4(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_f64x8(self, a: f64x8, dest: &mut [f64; 8usize]) -> () { - let (lower, upper) = self.split_f64x8(a); - let (v0_vec, v1_vec) = self.split_f64x4(lower); - let (v2_vec, v3_vec) = self.split_f64x4(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let out0 = u64x2_shuffle::<0, 2>(v0, v1); - let out1 = u64x2_shuffle::<0, 2>(v2, v3); - let out2 = u64x2_shuffle::<1, 3>(v0, v1); - let out3 = u64x2_shuffle::<1, 3>(v2, v3); - let (chunks, []) = dest.as_chunks_mut::<2usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn splat_i64x8(self, val: i64) -> i64x8 { let half = self.splat_i64x4(val); self.combine_i64x4(half, half) @@ -11942,44 +12058,6 @@ impl Simd for WasmSimd128 { self.combine_i64x4(self.neg_i64x4(a0), self.neg_i64x4(a1)) } #[inline(always)] - fn load_interleaved_128_i64x8(self, src: &[i64; 8usize]) -> i64x8 { - let (chunks, []) = src.as_chunks::<2usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[3]); - let out0 = u64x2_shuffle::<0, 2>(v0, v2); - let out1 = u64x2_shuffle::<1, 3>(v0, v2); - let out2 = u64x2_shuffle::<0, 2>(v1, v3); - let out3 = u64x2_shuffle::<1, 3>(v1, v3); - let combined_lower = self.combine_i64x2(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_i64x2(out2.simd_into(self), out3.simd_into(self)); - self.combine_i64x4(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_i64x8(self, a: i64x8, dest: &mut [i64; 8usize]) -> () { - let (lower, upper) = self.split_i64x8(a); - let (v0_vec, v1_vec) = self.split_i64x4(lower); - let (v2_vec, v3_vec) = self.split_i64x4(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let out0 = u64x2_shuffle::<0, 2>(v0, v1); - let out1 = u64x2_shuffle::<0, 2>(v2, v3); - let out2 = u64x2_shuffle::<1, 3>(v0, v1); - let out3 = u64x2_shuffle::<1, 3>(v2, v3); - let (chunks, []) = dest.as_chunks_mut::<2usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn splat_u64x8(self, val: u64) -> u64x8 { let half = self.splat_u64x4(val); self.combine_u64x4(half, half) @@ -12295,44 +12373,6 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] - fn load_interleaved_128_u64x8(self, src: &[u64; 8usize]) -> u64x8 { - let (chunks, []) = src.as_chunks::<2usize>() else { - unreachable!() - }; - let v0: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[0]); - let v1: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[1]); - let v2: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[2]); - let v3: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[3]); - let out0 = u64x2_shuffle::<0, 2>(v0, v2); - let out1 = u64x2_shuffle::<1, 3>(v0, v2); - let out2 = u64x2_shuffle::<0, 2>(v1, v3); - let out3 = u64x2_shuffle::<1, 3>(v1, v3); - let combined_lower = self.combine_u64x2(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.combine_u64x2(out2.simd_into(self), out3.simd_into(self)); - self.combine_u64x4(combined_lower, combined_upper) - } - #[inline(always)] - fn store_interleaved_128_u64x8(self, a: u64x8, dest: &mut [u64; 8usize]) -> () { - let (lower, upper) = self.split_u64x8(a); - let (v0_vec, v1_vec) = self.split_u64x4(lower); - let (v2_vec, v3_vec) = self.split_u64x4(upper); - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); - let out0 = u64x2_shuffle::<0, 2>(v0, v1); - let out1 = u64x2_shuffle::<0, 2>(v2, v3); - let out2 = u64x2_shuffle::<1, 3>(v0, v1); - let out3 = u64x2_shuffle::<1, 3>(v2, v3); - let (chunks, []) = dest.as_chunks_mut::<2usize>() else { - unreachable!() - }; - crate::transmute::checked_transmute_store::(out0, &mut chunks[0]); - crate::transmute::checked_transmute_store::(out1, &mut chunks[1]); - crate::transmute::checked_transmute_store::(out2, &mut chunks[2]); - crate::transmute::checked_transmute_store::(out3, &mut chunks[3]); - } - #[inline(always)] fn splat_mask64x8(self, val: bool) -> mask64x8 { let half = self.splat_mask64x4(val); self.combine_mask64x4(half, half) diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index b53de485..22d75760 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -226,26 +226,8 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { OpSig::MaskSet => { panic!("Mask set must operate on the full mask vector") } - OpSig::LoadInterleaved { - block_size, - block_count, - } => { - let split_len = (block_size * block_count) as usize / (ty.scalar_bits * 2); - let ty_rust = ty.rust(); - quote! { - #method_sig { - let (chunks, _) = src.as_chunks::<#split_len>(); - let lo = self.#do_half(&chunks[0]); - let hi = self.#do_half(&chunks[1]); - #ty_rust { - val: crate::transmute::checked_transmute_copy(&[lo.val, hi.val]), - simd: self, - } - } - } - } - OpSig::StoreInterleaved { .. } => { - panic!("The generic fallback is not implemented for this operation") + OpSig::LoadInterleaved { .. } | OpSig::StoreInterleaved { .. } => { + panic!("Interleaved memory operations must operate on full 128-bit vectors") } OpSig::Split { .. } | OpSig::Combine { .. } diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index aa4f5359..a4c3b2d0 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -495,13 +495,23 @@ impl Level for Fallback { block_size, block_count, } => { - let len = (block_size * block_count) as usize / vec_ty.scalar_bits; - let items = - interleave_indices(len, block_count as usize, |idx| quote! { src[#idx] }); + let block_count = block_count as usize; + let elems_per_vec = block_size as usize / vec_ty.scalar_bits; + let vectors = (0..block_count).map(|channel| { + let items = make_list( + (0..elems_per_vec) + .map(|lane| { + let idx = lane * block_count + channel; + quote! { src[#idx] } + }) + .collect(), + ); + quote! { #items.simd_into(self) } + }); quote! { #method_sig { - #items.simd_into(self) + [#(#vectors),*] } } } @@ -509,9 +519,17 @@ impl Level for Fallback { block_size, block_count, } => { - let len = (block_size * block_count) as usize / vec_ty.scalar_bits; - let items = - interleave_indices(len, len / block_count as usize, |idx| quote! { a[#idx] }); + let block_count = block_count as usize; + let elems_per_vec = block_size as usize / vec_ty.scalar_bits; + let items = make_list( + (0..elems_per_vec) + .flat_map(|lane| { + (0..block_count).map(move |channel| { + quote! { vectors[#channel][#lane] } + }) + }) + .collect(), + ); quote! { #method_sig { @@ -573,19 +591,6 @@ impl Level for Fallback { } } -fn interleave_indices( - len: usize, - stride: usize, - func: impl FnMut(usize) -> TokenStream, -) -> TokenStream { - let indices = { - let indices = (0..len).collect::>(); - interleave(&indices, stride) - }; - - make_list(indices.into_iter().map(func).collect::>()) -} - fn lane(value: TokenStream, vec_ty: &VecType, idx: usize) -> TokenStream { if vec_ty.scalar == ScalarType::Mask { quote! { #value.val.0[#idx] } @@ -612,15 +617,3 @@ fn rhs_reference(method: &str) -> bool { fn make_list(items: Vec) -> TokenStream { quote!([#( #items, )*]) } - -fn interleave(input: &[usize], width: usize) -> Vec { - let height = input.len() / width; - - let mut output = Vec::with_capacity(input.len()); - for col in 0..width { - for row in 0..height { - output.push(input[row * width + col]); - } - } - output -} diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index 9a9ba2ca..5138f7bb 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -19,6 +19,19 @@ use crate::{ #[derive(Clone, Copy)] pub(crate) struct Neon; +fn neon_multi_vector_ty(vec_ty: &VecType, count: u16) -> Ident { + let scalar = match vec_ty.scalar { + ScalarType::Float => "float", + ScalarType::Unsigned => "uint", + ScalarType::Int => "int", + ScalarType::Mask => unreachable!("interleaved operations are not defined for masks"), + }; + Ident::new( + &format!("{scalar}{}x{}x{count}_t", vec_ty.scalar_bits, vec_ty.len), + Span::call_site(), + ) +} + impl Level for Neon { fn name(&self) -> &'static str { "Neon" @@ -138,21 +151,19 @@ impl Level for Neon { block_size, block_count, } => { - assert_eq!(block_count, 4, "only count of 4 is currently supported"); - let intrinsic = { - // The function expects 64-bit or 128-bit - let ty = VecType::new( - vec_ty.scalar, - vec_ty.scalar_bits, - block_size as usize / vec_ty.scalar_bits, - ); - simple_intrinsic("vld4", &ty) - }; + assert_eq!( + vec_ty.n_bits(), + block_size as usize, + "NEON interleaved loads require full block-sized vectors" + ); + let intrinsic = simple_intrinsic(&format!("vld{block_count}"), vec_ty); + let fields = (0..block_count).map(Literal::u16_unsuffixed); quote! { #method_sig { unsafe { - #intrinsic(src.as_ptr()).simd_into(self) + let native = #intrinsic(src.as_ptr()); + [#(native.#fields.simd_into(self)),*] } } } @@ -161,21 +172,27 @@ impl Level for Neon { block_size, block_count, } => { - assert_eq!(block_count, 4, "only count of 4 is currently supported"); - let intrinsic = { - // The function expects 64-bit or 128-bit - let ty = VecType::new( - vec_ty.scalar, - vec_ty.scalar_bits, - block_size as usize / vec_ty.scalar_bits, - ); - simple_intrinsic("vst4", &ty) - }; + assert_eq!( + vec_ty.n_bits(), + block_size as usize, + "NEON interleaved stores require full block-sized vectors" + ); + let intrinsic = simple_intrinsic(&format!("vst{block_count}"), vec_ty); + let aggregate_ty = neon_multi_vector_ty(vec_ty, block_count); + let indices = 0..block_count as usize; + let native_ty = self.arch_ty(vec_ty); + let native_values = (0..block_count as usize) + .map(|idx| format_ident!("v{idx}")) + .collect::>(); + let native_decls = native_values.iter().zip(indices).map(|(value, idx)| { + quote! { let #value: #native_ty = vectors[#idx].into(); } + }); quote! { #method_sig { unsafe { - #intrinsic(dest.as_mut_ptr(), a.into()) + #(#native_decls)* + #intrinsic(dest.as_mut_ptr(), #aggregate_ty(#(#native_values),*)) } } } diff --git a/fearless_simd_gen/src/mk_simd_trait.rs b/fearless_simd_gen/src/mk_simd_trait.rs index 4e4cdd95..515676e1 100644 --- a/fearless_simd_gen/src/mk_simd_trait.rs +++ b/fearless_simd_gen/src/mk_simd_trait.rs @@ -20,8 +20,12 @@ pub(crate) fn mk_simd_trait() -> TokenStream { for op in ops_for_type(vec_ty) { let method_sig = op.simd_trait_method_sig(vec_ty); let doc = op.format_docstring(TyFlavor::SimdTrait); + let doc_alias = op + .doc_alias() + .map(|alias| quote! { #[doc(alias = #alias)] }); methods.extend(quote! { #[doc = #doc] + #doc_alias #method_sig; }); } diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index 3624c59f..eb3457be 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -671,19 +671,6 @@ impl Level for WasmSimd128 { _ => panic!("unsupported scalar_bits"), }; - let block_ty = vec_ty.block_ty(); - let block_ty_2x = - VecType::new(block_ty.scalar, block_ty.scalar_bits, block_ty.len * 2); - - let combine_method = generic_op_name("combine", &block_ty); - let combine_method_2x = generic_op_name("combine", &block_ty_2x); - - let combine_code = quote! { - let combined_lower = self.#combine_method(out0.simd_into(self), out1.simd_into(self)); - let combined_upper = self.#combine_method(out2.simd_into(self), out3.simd_into(self)); - self.#combine_method_2x(combined_lower, combined_upper) - }; - let shuffle_code = if vec_ty.scalar_bits == 64 { quote! { let out0 = #shuffle_fn::<#i1>(v0, v2); @@ -728,7 +715,12 @@ impl Level for WasmSimd128 { ); #shuffle_code - #combine_code + [ + out0.simd_into(self), + out1.simd_into(self), + out2.simd_into(self), + out3.simd_into(self), + ] } } } @@ -760,24 +752,11 @@ impl Level for WasmSimd128 { _ => panic!("unsupported scalar_bits"), }; - let block_ty = vec_ty.block_ty(); - let block_ty_2x = - VecType::new(block_ty.scalar, block_ty.scalar_bits, block_ty.len * 2); - let block_ty_4x = - VecType::new(block_ty.scalar, block_ty.scalar_bits, block_ty.len * 4); - - let split_method = generic_op_name("split", &block_ty_2x); - let split_method_2x = generic_op_name("split", &block_ty_4x); - let split_code = quote! { - let (lower, upper) = self.#split_method_2x(a); - let (v0_vec, v1_vec) = self.#split_method(lower); - let (v2_vec, v3_vec) = self.#split_method(upper); - - let v0: v128 = v0_vec.into(); - let v1: v128 = v1_vec.into(); - let v2: v128 = v2_vec.into(); - let v3: v128 = v3_vec.into(); + let v0: v128 = vectors[0].into(); + let v1: v128 = vectors[1].into(); + let v2: v128 = vectors[2].into(); + let v3: v128 = vectors[3].into(); }; let shuffle_code = if vec_ty.scalar_bits == 64 { diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 857d7fc1..a7a98243 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -3193,7 +3193,7 @@ impl X86 { "only 128-bit blocks are currently supported" ); assert_eq!(block_count, 4, "only count of 4 is currently supported"); - if *self == Self::Avx512 && vec_ty.n_bits() == 512 { + if *self == Self::Avx512 { return self.handle_avx512_load_interleaved(op, vec_ty, block_size, block_count); } if *self == Self::Sse2 && matches!(vec_ty.scalar_bits, 8 | 16) { @@ -3210,6 +3210,7 @@ impl X86 { let block_ty = VecType::new(vec_ty.scalar, vec_ty.scalar_bits, block_len); let scalar_ty = block_ty.scalar.rust(block_ty.scalar_bits); let native_ty = self.arch_ty(&block_ty); + let native_block_ty = self.arch_ty(vec_ty); let vec_32 = block_ty.reinterpret(block_ty.scalar, 32); let unpacklo_32 = simple_sign_unaware_intrinsic("unpacklo", &vec_32); let unpackhi_32 = simple_sign_unaware_intrinsic("unpackhi", &vec_32); @@ -3225,16 +3226,6 @@ impl X86 { 256, ); - let vec_combined = - VecType::new(block_ty.scalar, block_ty.scalar_bits, block_ty.len * 2); - let combine_half = Ident::new( - &format!("combine_{}", block_ty.rust_name()), - Span::call_site(), - ); - let combine_full = Ident::new( - &format!("combine_{}", vec_combined.rust_name()), - Span::call_site(), - ); if avx2_64 { return self.kernel_method(op, vec_ty, |token| { quote! { @@ -3248,8 +3239,15 @@ impl X86 { let hi = #unpackhi_64(v0, v1); // [1,5,3,7] let out0 = #permute_128::<0x20>(lo, hi); // [0,4,1,5] let out1 = #permute_128::<0x31>(lo, hi); // [2,6,3,7] - - #token.#combine_half(out0.simd_into(#token), out1.simd_into(#token)) + let outputs: [#native_block_ty; 4] = + crate::transmute::checked_transmute_copy(&[out0, out1]); + + [ + outputs[0].simd_into(#token), + outputs[1].simd_into(#token), + outputs[2].simd_into(#token), + outputs[3].simd_into(#token), + ] } }); } @@ -3357,10 +3355,12 @@ impl X86 { #initial_unpack #final_unpack - #token.#combine_full( - #token.#combine_half(out0.simd_into(#token), out1.simd_into(#token)), - #token.#combine_half(out2.simd_into(#token), out3.simd_into(#token)), - ) + [ + out0.simd_into(#token), + out1.simd_into(#token), + out2.simd_into(#token), + out3.simd_into(#token), + ] } }) } @@ -3382,16 +3382,22 @@ impl X86 { assert_eq!(block_count, 4, "only count of 4 is currently supported"); assert_eq!( vec_ty.n_bits(), - 512, - "AVX-512 interleaved loads only specialize 512-bit vectors" + 128, + "AVX-512 interleaved loads return 128-bit vectors" + ); + let total_ty = VecType::new( + vec_ty.scalar, + vec_ty.scalar_bits, + vec_ty.len * block_count as usize, ); let scalar_ty = vec_ty.scalar.rust(vec_ty.scalar_bits); - let native_ty = self.arch_ty(vec_ty); - let len = vec_ty.len; - let permute = avx512_permutexvar_intrinsic(vec_ty); + let native_ty = self.arch_ty(&total_ty); + let native_block_ty = self.arch_ty(vec_ty); + let len = total_ty.len; + let permute = avx512_permutexvar_intrinsic(&total_ty); let indices = avx512_index_vector( - vec_ty, - interleaved_load_indices(vec_ty.len, block_count as usize), + &total_ty, + interleaved_load_indices(total_ty.len, block_count as usize), ); self.kernel_method(op, vec_ty, |token| { @@ -3400,7 +3406,15 @@ impl X86 { crate::transmute::checked_transmute_copy::<[#scalar_ty; #len], #native_ty>( src, ); - #permute(#indices, lanes).simd_into(#token) + let lanes = #permute(#indices, lanes); + let outputs: [#native_block_ty; 4] = + crate::transmute::checked_transmute_copy(&lanes); + [ + outputs[0].simd_into(#token), + outputs[1].simd_into(#token), + outputs[2].simd_into(#token), + outputs[3].simd_into(#token), + ] } }) } @@ -3417,7 +3431,7 @@ impl X86 { "only 128-bit blocks are currently supported" ); assert_eq!(block_count, 4, "only count of 4 is currently supported"); - if *self == Self::Avx512 && vec_ty.n_bits() == 512 { + if *self == Self::Avx512 { return self.handle_avx512_store_interleaved(op, vec_ty, block_size, block_count); } if *self == Self::Sse2 && matches!(vec_ty.scalar_bits, 8 | 16) { @@ -3434,6 +3448,7 @@ impl X86 { let block_ty = VecType::new(vec_ty.scalar, vec_ty.scalar_bits, block_len); let scalar_ty = block_ty.scalar.rust(block_ty.scalar_bits); let native_ty = self.arch_ty(&block_ty); + let native_block_ty = self.arch_ty(vec_ty); let vec_32 = block_ty.reinterpret(block_ty.scalar, 32); let unpacklo_32 = simple_sign_unaware_intrinsic("unpacklo", &vec_32); let unpackhi_32 = simple_sign_unaware_intrinsic("unpackhi", &vec_32); @@ -3449,24 +3464,23 @@ impl X86 { 256, ); - let vec_combined = - VecType::new(block_ty.scalar, block_ty.scalar_bits, block_ty.len * 2); - let split_half = Ident::new( - &format!("split_{}", vec_combined.rust_name()), - Span::call_site(), - ); - let split_full = - Ident::new(&format!("split_{}", vec_ty.rust_name()), Span::call_site()); - // For 64-bit values, AVX2 permits a more efficient implementation. // It is special-cased here as a full early return because plumbing it // through the rest of the logic in this function complicates it significantly. if avx2_64 { return self.kernel_method(op, vec_ty, |token| { quote! { - let (v0, v1) = #token.#split_full(a); - let v0: #native_ty = v0.into(); - let v1: #native_ty = v1.into(); + let _ = #token; + let inputs: [#native_block_ty; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let wide_inputs: [#native_ty; 2] = + crate::transmute::checked_transmute_copy(&inputs); + let v0 = wide_inputs[0]; + let v1 = wide_inputs[1]; let lo = #permute_128::<0x20>(v0, v1); // [0,1,4,5] let hi = #permute_128::<0x31>(v0, v1); // [2,3,6,7] @@ -3565,13 +3579,11 @@ impl X86 { self.kernel_method(op, vec_ty, |token| { quote! { - let (v01, v23) = #token.#split_full(a); - let (v0, v1) = #token.#split_half(v01); - let (v2, v3) = #token.#split_half(v23); - let v0 = v0.into(); - let v1 = v1.into(); - let v2 = v2.into(); - let v3 = v3.into(); + let _ = #token; + let v0: #native_ty = vectors[0].into(); + let v1: #native_ty = vectors[1].into(); + let v2: #native_ty = vectors[2].into(); + let v3: #native_ty = vectors[3].into(); #initial_unpack #final_unpack @@ -3607,21 +3619,35 @@ impl X86 { assert_eq!(block_count, 4, "only count of 4 is currently supported"); assert_eq!( vec_ty.n_bits(), - 512, - "AVX-512 interleaved stores only specialize 512-bit vectors" + 128, + "AVX-512 interleaved stores accept 128-bit vectors" + ); + let total_ty = VecType::new( + vec_ty.scalar, + vec_ty.scalar_bits, + vec_ty.len * block_count as usize, ); let scalar_ty = vec_ty.scalar.rust(vec_ty.scalar_bits); - let native_ty = self.arch_ty(vec_ty); - let len = vec_ty.len; - let permute = avx512_permutexvar_intrinsic(vec_ty); + let native_ty = self.arch_ty(&total_ty); + let native_block_ty = self.arch_ty(vec_ty); + let len = total_ty.len; + let permute = avx512_permutexvar_intrinsic(&total_ty); let indices = avx512_index_vector( - vec_ty, - interleaved_store_indices(vec_ty.len, block_count as usize), + &total_ty, + interleaved_store_indices(total_ty.len, block_count as usize), ); - self.kernel_method(op, vec_ty, |_| { + self.kernel_method(op, vec_ty, |token| { quote! { - let lanes = #permute(#indices, a.into()); + let _ = #token; + let inputs: [#native_block_ty; 4] = [ + vectors[0].into(), + vectors[1].into(), + vectors[2].into(), + vectors[3].into(), + ]; + let lanes: #native_ty = crate::transmute::checked_transmute_copy(&inputs); + let lanes = #permute(#indices, lanes); crate::transmute::checked_transmute_store::<#native_ty, [#scalar_ty; #len]>( lanes, dest, diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index e968129b..b75114b4 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -121,14 +121,15 @@ pub(crate) enum OpSig { /// Takes a mutable mask vector, a lane index, and a boolean, and updates the lane in place. MaskSet, /// Takes an argument of an array of a certain scalar type, with the length (`block_size` * `block_count`) / [scalar - /// type's byte size]. Returns a vector type of that scalar type and length. + /// type's byte size]. Returns `block_count` vectors whose width is `block_size`. /// /// First argument is the base block size (i.e. 128), second argument is how many blocks. For example, /// `LoadInterleaved(128, 4)` would correspond to the NEON instructions `vld4q_f32`, while `LoadInterleaved(64, 4)` /// would correspond to `vld4_f32`. LoadInterleaved { block_size: u16, block_count: u16 }, - /// The inverse of [`OpSig::LoadInterleaved`]. Takes a vector argument with the length (`block_size` * `block_count`) / - /// [scalar type's byte size], and a mutable reference to a scalar array of the same length, and returns nothing. + /// The inverse of [`OpSig::LoadInterleaved`]. Takes `block_count` vectors whose width is `block_size` and a mutable + /// reference to a scalar array with the length (`block_size` * `block_count`) / [scalar type's byte size], and + /// returns nothing. StoreInterleaved { block_size: u16, block_count: u16 }, /// Takes a single argument of the array type corresponding to the vector type (e.g. `[f32; 4]` for `f32x4`), or /// a reference to it, and returns that vector type. @@ -197,6 +198,14 @@ impl Op { } } + pub(crate) fn doc_alias(&self) -> Option<&'static str> { + match self.method { + "load_four_interleaved" => Some("load_interleaved"), + "store_four_interleaved" => Some("store_interleaved"), + _ => None, + } + } + pub(crate) fn simd_trait_method_sig(&self, vec_ty: &VecType) -> TokenStream { let method_ident = generic_op_name(self.method, vec_ty); let sig = self.simd_trait_sig_parts(vec_ty, quote! { Self }); @@ -280,14 +289,16 @@ impl Op { block_count, } => { let arg_ty = load_interleaved_arg_ty(*block_size, *block_count, vec_ty); - (vec![arg_ty], vec) + let block_count = *block_count as usize; + (vec![arg_ty], quote! { [#vec; #block_count] }) } OpSig::StoreInterleaved { block_size, block_count, } => { let arg_ty = store_interleaved_arg_ty(*block_size, *block_count, vec_ty); - (vec![vec.clone(), arg_ty], quote! { () }) + let block_count = *block_count as usize; + (vec![quote! { [#vec; #block_count] }, arg_ty], quote! { () }) } OpSig::Compare => { let result = vec_ty.mask_ty().rust(); @@ -1365,44 +1376,41 @@ pub(crate) fn ops_for_type(ty: &VecType) -> Vec { if matches!( ty.scalar, ScalarType::Int | ScalarType::Unsigned | ScalarType::Float - ) && ty.n_bits() == 512 + ) && ty.n_bits() == 128 { ops.push(Op::new( - "load_interleaved_128", + "load_four_interleaved", OpKind::AssociatedOnly, OpSig::LoadInterleaved { block_size: 128, block_count: 4, }, - "Load elements from an array with 4-way interleaving.\n\n\ - This is different from loading a vector and calling `interleave`: `interleave` combines two already-loaded \ - vectors, while this operation treats memory as four interleaved 128-bit vectors and deinterleaves them \ - into one vector.\n\n\ - For example, with 32-bit lanes, memory laid out as \ - `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]` loads as \ - `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]`.", + "Load four 128-bit vectors from an array with 4-way interleaving.\n\n\ + This is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\n\ + For example, with 32-bit lanes, memory laid out as\ + `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as\ + `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`.", )); } if matches!( ty.scalar, ScalarType::Int | ScalarType::Unsigned | ScalarType::Float - ) && ty.n_bits() == 512 + ) && ty.n_bits() == 128 { ops.push(Op::new( - "store_interleaved_128", + "store_four_interleaved", OpKind::AssociatedOnly, OpSig::StoreInterleaved { block_size: 128, block_count: 4, }, - "Store elements to an array with 4-way interleaving.\n\n\ - This is the inverse of `load_interleaved_128`. It is different from calling `interleave` and then storing: \ - `interleave` combines two already-loaded vectors, while this operation stores four consecutive 128-bit \ - vectors into lane-interleaved memory.\n\n\ - For example, with 32-bit lanes, a vector containing \ - `[a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3]` stores as \ - `[a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, b3, c3, d3]`.", + "Store four 128-bit vectors to an array with 4-way interleaving.\n\n\ + This is the inverse of `load_four_interleaved`.\n\n\ + This is useful e.g. in image processing to turn vectors of each color component into interleaved RGBA pixels.\n\n\ + For example, with 32-bit lanes, vectors containing \ + `[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]` get stored as \ + `[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]`.", )); } @@ -1619,7 +1627,8 @@ impl OpSig { Self::Ternary | Self::Select => &["a", "b", "c"], Self::Shift => &["a", "shift"], Self::LoadInterleaved { .. } => &["src"], - Self::StoreInterleaved { .. } | Self::StoreArray => &["a", "dest"], + Self::StoreInterleaved { .. } => &["vectors", "dest"], + Self::StoreArray => &["a", "dest"], } } fn vec_trait_arg_names(&self) -> &'static [&'static str] { diff --git a/fearless_simd_tests/tests/harness/ops/load_four_interleaved.rs b/fearless_simd_tests/tests/harness/ops/load_four_interleaved.rs new file mode 100644 index 00000000..0fe8bd3f --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/load_four_interleaved.rs @@ -0,0 +1,325 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +// One concrete test row per supported vector type. + +#[simd_test] +fn load_four_interleaved_f32x4(simd: S) { + let data = [ + 0.0, + 4.0, + 8.0, + f32::MIN, + f32::NAN, + -0.0, + 9.0, + 13.0, + f32::INFINITY, + 6.0, + -10.0, + f32::MAX, + -3.0, + f32::NEG_INFINITY, + 11.0, + 15.0, + ]; + let expected = [ + [0.0, f32::NAN, f32::INFINITY, -3.0], + [4.0, -0.0, 6.0, f32::NEG_INFINITY], + [8.0, 9.0, -10.0, 11.0], + [f32::MIN, 13.0, f32::MAX, 15.0], + ]; + + // Note: f32::NAN != f32::NAN hence we compare the bit pattern. + let result = simd.load_four_interleaved_f32x4(&data); + assert_eq!( + result.map(|vector| (*vector).map(f32::to_bits)), + expected.map(|vector| vector.map(f32::to_bits)), + ); +} + +#[simd_test] +fn load_four_interleaved_f64x2(simd: S) { + let data = [ + 0.0, + f64::NAN, + f64::INFINITY, + -3.0, + -0.0, + f64::MIN, + f64::NEG_INFINITY, + f64::MAX, + ]; + let expected = [ + [0.0, -0.0], + [f64::NAN, f64::MIN], + [f64::INFINITY, f64::NEG_INFINITY], + [-3.0, f64::MAX], + ]; + + // Note: f64::NAN != f64::NAN hence we compare the bit pattern. + let result = simd.load_four_interleaved_f64x2(&data); + assert_eq!( + result.map(|vector| (*vector).map(f64::to_bits)), + expected.map(|vector| vector.map(f64::to_bits)), + ); +} + +#[simd_test] +fn load_four_interleaved_i8x16(simd: S) { + #[rustfmt::skip] + let data: [i8; 64] = [ + i8::MIN, -31, -30, -29, -28, -27, -26, -25, + -24, -23, -22, -21, -20, -19, -18, -17, + -16, -15, -14, -13, -12, -11, -10, -9, + -8, -7, -6, -5, -4, -3, -2, -1, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, i8::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_i8x16(&data) + .map(|vector| *vector), + [ + [ + i8::MIN, + -28, + -24, + -20, + -16, + -12, + -8, + -4, + 0, + 4, + 8, + 12, + 16, + 20, + 24, + 28 + ], + [ + -31, -27, -23, -19, -15, -11, -7, -3, 1, 5, 9, 13, 17, 21, 25, 29 + ], + [ + -30, -26, -22, -18, -14, -10, -6, -2, 2, 6, 10, 14, 18, 22, 26, 30 + ], + [ + -29, + -25, + -21, + -17, + -13, + -9, + -5, + -1, + 3, + 7, + 11, + 15, + 19, + 23, + 27, + i8::MAX + ], + ] + ); +} + +#[simd_test] +fn load_four_interleaved_u8x16(simd: S) { + #[rustfmt::skip] + let data: [u8; 64] = [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, u8::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_u8x16(&data) + .map(|vector| *vector), + [ + [0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60], + [1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61], + [2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62], + [ + 3, + 7, + 11, + 15, + 19, + 23, + 27, + 31, + 35, + 39, + 43, + 47, + 51, + 55, + 59, + u8::MAX + ], + ] + ); +} + +#[simd_test] +fn load_four_interleaved_i16x8(simd: S) { + let data: [i16; 32] = [ + i16::MIN, + -15, + -14, + -13, + -12, + -11, + -10, + -9, + -8, + -7, + -6, + -5, + -4, + -3, + -2, + -1, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + i16::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_i16x8(&data) + .map(|vector| *vector), + [ + [i16::MIN, -12, -8, -4, 0, 4, 8, 12], + [-15, -11, -7, -3, 1, 5, 9, 13], + [-14, -10, -6, -2, 2, 6, 10, 14], + [-13, -9, -5, -1, 3, 7, 11, i16::MAX], + ] + ); +} + +#[simd_test] +fn load_four_interleaved_u16x8(simd: S) { + #[rustfmt::skip] + let data: [u16; 32] = [ + u16::MIN, 2, 3, 4, 5, 6, 7, 8, + 10, 20, 30, 40, 50, 60, 70, 80, + 100, 200, 300, 400, 500, 600, 700, 800, + 1000, 2000, 3000, 4000, 5000, 6000, 7000, u16::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_u16x8(&data) + .map(|vector| *vector), + [ + [u16::MIN, 5, 10, 50, 100, 500, 1000, 5000], + [2, 6, 20, 60, 200, 600, 2000, 6000], + [3, 7, 30, 70, 300, 700, 3000, 7000], + [4, 8, 40, 80, 400, 800, 4000, u16::MAX], + ] + ); +} + +#[simd_test] +fn load_four_interleaved_i32x4(simd: S) { + let data: [i32; 16] = [ + i32::MIN, + -7, + -6, + -5, + -4, + -3, + -2, + -1, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + i32::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_i32x4(&data) + .map(|vector| *vector), + [ + [i32::MIN, -4, 0, 4], + [-7, -3, 1, 5], + [-6, -2, 2, 6], + [-5, -1, 3, i32::MAX] + ] + ); +} + +#[simd_test] +fn load_four_interleaved_u32x4(simd: S) { + #[rustfmt::skip] + let data: [u32; 16] = [ + 1, 2, 3, 4, + 10, 20, 30, 40, + 100, 200, 300, 400, + 1000, 2000, 3000, u32::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_u32x4(&data) + .map(|vector| *vector), + [ + [1, 10, 100, 1000], + [2, 20, 200, 2000], + [3, 30, 300, 3000], + [4, 40, 400, u32::MAX], + ] + ); +} + +#[simd_test] +fn load_four_interleaved_i64x2(simd: S) { + let data = [ + i64::MIN, + -3_i64, + -2_i64, + -1_i64, + 0_i64, + 1_i64, + 2_i64, + i64::MAX, + ]; + assert_eq!( + simd.load_four_interleaved_i64x2(&data) + .map(|vector| *vector), + [[i64::MIN, 0], [-3, 1], [-2, 2], [-1, i64::MAX]] + ); +} + +#[simd_test] +fn load_four_interleaved_u64x2(simd: S) { + let data = [u64::MIN, 3_u64, 5_u64, 7_u64, 2_u64, 4_u64, 6_u64, u64::MAX]; + assert_eq!( + simd.load_four_interleaved_u64x2(&data) + .map(|vector| *vector), + [[u64::MIN, 2], [3, 4], [5, 6], [7, u64::MAX]] + ); +} diff --git a/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs b/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs deleted file mode 100644 index 523c4dae..00000000 --- a/fearless_simd_tests/tests/harness/ops/load_interleaved_128.rs +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2026 the Fearless_SIMD Authors -// SPDX-License-Identifier: Apache-2.0 OR MIT - -use fearless_simd::*; -use fearless_simd_dev_macros::simd_test; - -// One concrete test row per supported vector type. - -#[simd_test] -fn load_interleaved_128_f32x16(simd: S) { - let data = [ - 0.0, - 4.0, - 8.0, - f32::MIN, - f32::NAN, - -0.0, - 9.0, - 13.0, - f32::INFINITY, - 6.0, - -10.0, - f32::MAX, - -3.0, - f32::NEG_INFINITY, - 11.0, - 15.0, - ]; - let expected = [ - 0.0, - f32::NAN, - f32::INFINITY, - -3.0, - 4.0, - -0.0, - 6.0, - f32::NEG_INFINITY, - 8.0, - 9.0, - -10.0, - 11.0, - f32::MIN, - 13.0, - f32::MAX, - 15.0, - ]; - - // Note: f32::NAN != f32::NAN hence we compare the bit pattern. - let result = simd.load_interleaved_128_f32x16(&data); - assert_eq!((*result).map(f32::to_bits), expected.map(f32::to_bits),); -} - -#[simd_test] -fn load_interleaved_128_f64x8(simd: S) { - let data = [ - 0.0, - f64::NAN, - f64::INFINITY, - -3.0, - -0.0, - f64::MIN, - f64::NEG_INFINITY, - f64::MAX, - ]; - let expected = [ - 0.0, - -0.0, - f64::NAN, - f64::MIN, - f64::INFINITY, - f64::NEG_INFINITY, - -3.0, - f64::MAX, - ]; - - // Note: f64::NAN != f64::NAN hence we compare the bit pattern. - let result = simd.load_interleaved_128_f64x8(&data); - assert_eq!((*result).map(f64::to_bits), expected.map(f64::to_bits)); -} - -#[simd_test] -fn load_interleaved_128_i32x16(simd: S) { - #[rustfmt::skip] - let data: [i32; 16] = [ - -8, -7, -6, -5, - -4, -3, -2, -1, - 0, 1, 2, 3, - 4, 5, 6, 7, - ]; - assert_eq!( - *simd.load_interleaved_128_i32x16(&data), - [-8, -4, 0, 4, -7, -3, 1, 5, -6, -2, 2, 6, -5, -1, 3, 7] - ); -} - -#[simd_test] -fn load_interleaved_128_u32x16(simd: S) { - #[rustfmt::skip] - let data: [u32; 16] = [ - 1, 2, 3, 4, - 10, 20, 30, 40, - 100, 200, 300, 400, - 1000, 2000, 3000, 4000, - ]; - assert_eq!( - *simd.load_interleaved_128_u32x16(&data), - [ - 1, 10, 100, 1000, 2, 20, 200, 2000, 3, 30, 300, 3000, 4, 40, 400, 4000 - ] - ); -} - -#[simd_test] -fn load_interleaved_128_u16x32(simd: S) { - #[rustfmt::skip] - let data: [u16; 32] = [ - 1, 2, 3, 4, - 5, 6, 7, 8, - - 10, 20, 30, 40, - 50, 60, 70, 80, - - 100, 200, 300, 400, - 500, 600, 700, 800, - - 1000, 2000, 3000, 4000, - 5000, 6000, 7000, 8000, - ]; - assert_eq!( - *simd.load_interleaved_128_u16x32(&data), - [ - 1, 5, 10, 50, 100, 500, 1000, 5000, 2, 6, 20, 60, 200, 600, 2000, 6000, 3, 7, 30, 70, - 300, 700, 3000, 7000, 4, 8, 40, 80, 400, 800, 4000, 8000 - ] - ); -} - -#[simd_test] -fn load_interleaved_128_i16x32(simd: S) { - #[rustfmt::skip] - let data: [i16; 32] = [ - -16, -15, -14, -13, - -12, -11, -10, -9, - -8, -7, -6, -5, - -4, -3, -2, -1, - 0, 1, 2, 3, - 4, 5, 6, 7, - 8, 9, 10, 11, - 12, 13, 14, 15, - ]; - assert_eq!( - *simd.load_interleaved_128_i16x32(&data), - [ - -16, -12, -8, -4, 0, 4, 8, 12, -15, -11, -7, -3, 1, 5, 9, 13, -14, -10, -6, -2, 2, 6, - 10, 14, -13, -9, -5, -1, 3, 7, 11, 15 - ] - ); -} - -#[simd_test] -fn load_interleaved_128_u8x64(simd: S) { - #[rustfmt::skip] - let data: [u8; 64] = [ - 0, 1, 2, 3, - 4, 5, 6, 7, - 8, 9, 10, 11, - 12, 13, 14, 15, - - 16, 17, 18, 19, - 20, 21, 22, 23, - 24, 25, 26, 27, - 28, 29, 30, 31, - - 32, 33, 34, 35, - 36, 37, 38, 39, - 40, 41, 42, 43, - 44, 45, 46, 47, - - 48, 49, 50, 51, - 52, 53, 54, 55, - 56, 57, 58, 59, - 60, 61, 62, 63, - ]; - assert_eq!( - *simd.load_interleaved_128_u8x64(&data), - [ - 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 1, 5, 9, 13, 17, 21, 25, - 29, 33, 37, 41, 45, 49, 53, 57, 61, 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, - 54, 58, 62, 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 - ] - ); -} - -#[simd_test] -fn load_interleaved_128_i8x64(simd: S) { - #[rustfmt::skip] - let data: [i8; 64] = [ - -32, -31, -30, -29, - -28, -27, -26, -25, - -24, -23, -22, -21, - -20, -19, -18, -17, - -16, -15, -14, -13, - -12, -11, -10, -9, - -8, -7, -6, -5, - -4, -3, -2, -1, - 0, 1, 2, 3, - 4, 5, 6, 7, - 8, 9, 10, 11, - 12, 13, 14, 15, - 16, 17, 18, 19, - 20, 21, 22, 23, - 24, 25, 26, 27, - 28, 29, 30, 31, - ]; - assert_eq!( - *simd.load_interleaved_128_i8x64(&data), - [ - -32, -28, -24, -20, -16, -12, -8, -4, 0, 4, 8, 12, 16, 20, 24, 28, -31, -27, -23, -19, - -15, -11, -7, -3, 1, 5, 9, 13, 17, 21, 25, 29, -30, -26, -22, -18, -14, -10, -6, -2, 2, - 6, 10, 14, 18, 22, 26, 30, -29, -25, -21, -17, -13, -9, -5, -1, 3, 7, 11, 15, 19, 23, - 27, 31 - ] - ); -} - -// Additional concrete rows for this operation. - -#[simd_test] -fn load_interleaved_128_u64x8(simd: S) { - let data = [1_u64, 3_u64, 5_u64, 7_u64, 2_u64, 4_u64, 6_u64, 8_u64]; - let a = simd.load_interleaved_128_u64x8(&data); - assert_eq!(*a, [1_u64, 2_u64, 3_u64, 4_u64, 5_u64, 6_u64, 7_u64, 8_u64]); -} - -#[simd_test] -fn load_interleaved_128_i64x8(simd: S) { - let data = [-4_i64, -3_i64, -2_i64, -1_i64, 0_i64, 1_i64, 2_i64, 3_i64]; - let a = simd.load_interleaved_128_i64x8(&data); - assert_eq!( - *a, - [-4_i64, 0_i64, -3_i64, 1_i64, -2_i64, 2_i64, -1_i64, 3_i64] - ); -} diff --git a/fearless_simd_tests/tests/harness/ops/mod.rs b/fearless_simd_tests/tests/harness/ops/mod.rs index a7ecf411..45002f76 100644 --- a/fearless_simd_tests/tests/harness/ops/mod.rs +++ b/fearless_simd_tests/tests/harness/ops/mod.rs @@ -36,7 +36,7 @@ mod index; mod interleave; mod load_array; mod load_array_ref; -mod load_interleaved_128; +mod load_four_interleaved; mod max; mod max_precise; mod min; @@ -72,7 +72,7 @@ mod splat; mod split; mod sqrt; mod store_array; -mod store_interleaved_128; +mod store_four_interleaved; mod store_slice; mod sub; mod swizzle_dyn_within_blocks; diff --git a/fearless_simd_tests/tests/harness/ops/store_four_interleaved.rs b/fearless_simd_tests/tests/harness/ops/store_four_interleaved.rs new file mode 100644 index 00000000..d32fe1ca --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/store_four_interleaved.rs @@ -0,0 +1,500 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +// One concrete test row per supported vector type. + +#[simd_test] +fn store_four_interleaved_f32x4(simd: S) { + let vectors = [ + f32x4::from_slice(simd, &[0.0, f32::NAN, f32::INFINITY, -3.0]), + f32x4::from_slice(simd, &[4.0, -0.0, 6.0, f32::NEG_INFINITY]), + f32x4::from_slice(simd, &[8.0, 9.0, -10.0, 11.0]), + f32x4::from_slice(simd, &[f32::MIN, 13.0, f32::MAX, 15.0]), + ]; + let mut dest = [0.0_f32; 16]; + simd.store_four_interleaved_f32x4(vectors, &mut dest); + + let expected = [ + 0.0, + 4.0, + 8.0, + f32::MIN, + f32::NAN, + -0.0, + 9.0, + 13.0, + f32::INFINITY, + 6.0, + -10.0, + f32::MAX, + -3.0, + f32::NEG_INFINITY, + 11.0, + 15.0, + ]; + + // Note: f32::NAN != f32::NAN hence we compare the bit pattern. + assert_eq!(dest.map(f32::to_bits), expected.map(f32::to_bits)); +} + +#[simd_test] +fn store_four_interleaved_f64x2(simd: S) { + let vectors = [ + f64x2::from_slice(simd, &[0.0, -0.0]), + f64x2::from_slice(simd, &[f64::NAN, f64::MIN]), + f64x2::from_slice(simd, &[f64::INFINITY, f64::NEG_INFINITY]), + f64x2::from_slice(simd, &[-3.0, f64::MAX]), + ]; + let mut dest = [0.0_f64; 8]; + simd.store_four_interleaved_f64x2(vectors, &mut dest); + + let expected = [ + 0.0, + f64::NAN, + f64::INFINITY, + -3.0, + -0.0, + f64::MIN, + f64::NEG_INFINITY, + f64::MAX, + ]; + + // Note: f64::NAN != f64::NAN hence we compare the bit pattern. + assert_eq!(dest.map(f64::to_bits), expected.map(f64::to_bits)); +} + +#[simd_test] +fn store_four_interleaved_i8x16(simd: S) { + let vectors = [ + i8x16::from_slice( + simd, + &[ + i8::MIN, + -31, + -30, + -29, + -28, + -27, + -26, + -25, + -24, + -23, + -22, + -21, + -20, + -19, + -18, + -17, + ], + ), + i8x16::from_slice( + simd, + &[ + -16, -15, -14, -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, + ], + ), + i8x16::from_slice( + simd, + &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + ), + i8x16::from_slice( + simd, + &[ + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + i8::MAX, + ], + ), + ]; + let mut dest = [0_i8; 64]; + simd.store_four_interleaved_i8x16(vectors, &mut dest); + + let expected = [ + i8::MIN, + -16, + 0, + 16, + -31, + -15, + 1, + 17, + -30, + -14, + 2, + 18, + -29, + -13, + 3, + 19, + -28, + -12, + 4, + 20, + -27, + -11, + 5, + 21, + -26, + -10, + 6, + 22, + -25, + -9, + 7, + 23, + -24, + -8, + 8, + 24, + -23, + -7, + 9, + 25, + -22, + -6, + 10, + 26, + -21, + -5, + 11, + 27, + -20, + -4, + 12, + 28, + -19, + -3, + 13, + 29, + -18, + -2, + 14, + 30, + -17, + -1, + 15, + i8::MAX, + ]; + + assert_eq!(dest, expected); +} + +#[simd_test] +fn store_four_interleaved_u8x16(simd: S) { + let vectors = [ + u8x16::from_slice( + simd, + &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + ), + u8x16::from_slice( + simd, + &[ + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ], + ), + u8x16::from_slice( + simd, + &[ + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + ], + ), + u8x16::from_slice( + simd, + &[ + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + u8::MAX, + ], + ), + ]; + let mut dest = [0_u8; 64]; + simd.store_four_interleaved_u8x16(vectors, &mut dest); + + let expected = [ + 0, + 16, + 32, + 48, + 1, + 17, + 33, + 49, + 2, + 18, + 34, + 50, + 3, + 19, + 35, + 51, + 4, + 20, + 36, + 52, + 5, + 21, + 37, + 53, + 6, + 22, + 38, + 54, + 7, + 23, + 39, + 55, + 8, + 24, + 40, + 56, + 9, + 25, + 41, + 57, + 10, + 26, + 42, + 58, + 11, + 27, + 43, + 59, + 12, + 28, + 44, + 60, + 13, + 29, + 45, + 61, + 14, + 30, + 46, + 62, + 15, + 31, + 47, + u8::MAX, + ]; + + assert_eq!(dest, expected); +} + +#[simd_test] +fn store_four_interleaved_i16x8(simd: S) { + let vectors = [ + i16x8::from_slice(simd, &[i16::MIN, -15, -14, -13, -12, -11, -10, -9]), + i16x8::from_slice(simd, &[-8, -7, -6, -5, -4, -3, -2, -1]), + i16x8::from_slice(simd, &[0, 1, 2, 3, 4, 5, 6, 7]), + i16x8::from_slice(simd, &[8, 9, 10, 11, 12, 13, 14, i16::MAX]), + ]; + let mut dest = [0_i16; 32]; + simd.store_four_interleaved_i16x8(vectors, &mut dest); + + let expected = [ + i16::MIN, + -8, + 0, + 8, + -15, + -7, + 1, + 9, + -14, + -6, + 2, + 10, + -13, + -5, + 3, + 11, + -12, + -4, + 4, + 12, + -11, + -3, + 5, + 13, + -10, + -2, + 6, + 14, + -9, + -1, + 7, + i16::MAX, + ]; + + assert_eq!(dest, expected); +} + +#[simd_test] +fn store_four_interleaved_u16x8(simd: S) { + let vectors = [ + u16x8::from_slice(simd, &[0, 1, 2, 3, 4, 5, 6, 7]), + u16x8::from_slice(simd, &[100, 101, 102, 103, 104, 105, 106, 107]), + u16x8::from_slice(simd, &[200, 201, 202, 203, 204, 205, 206, 207]), + u16x8::from_slice(simd, &[300, 301, 302, 303, 304, 305, 306, u16::MAX]), + ]; + let mut dest = [0_u16; 32]; + simd.store_four_interleaved_u16x8(vectors, &mut dest); + + let expected = [ + 0, + 100, + 200, + 300, + 1, + 101, + 201, + 301, + 2, + 102, + 202, + 302, + 3, + 103, + 203, + 303, + 4, + 104, + 204, + 304, + 5, + 105, + 205, + 305, + 6, + 106, + 206, + 306, + 7, + 107, + 207, + u16::MAX, + ]; + + assert_eq!(dest, expected); +} + +#[simd_test] +fn store_four_interleaved_i32x4(simd: S) { + let vectors = [ + i32x4::from_slice(simd, &[i32::MIN, -7, -6, -5]), + i32x4::from_slice(simd, &[-4, -3, -2, -1]), + i32x4::from_slice(simd, &[0, 1, 2, 3]), + i32x4::from_slice(simd, &[4, 5, 6, i32::MAX]), + ]; + let mut dest = [0_i32; 16]; + simd.store_four_interleaved_i32x4(vectors, &mut dest); + assert_eq!( + dest, + [ + i32::MIN, + -4, + 0, + 4, + -7, + -3, + 1, + 5, + -6, + -2, + 2, + 6, + -5, + -1, + 3, + i32::MAX + ] + ); +} + +#[simd_test] +fn store_four_interleaved_u32x4(simd: S) { + let vectors = [ + u32x4::from_slice(simd, &[0, 1, u32::MAX, 3]), + u32x4::from_slice(simd, &[1000, 1001, 1002, 1003]), + u32x4::from_slice(simd, &[2000, 2001, 2002, 2003]), + u32x4::from_slice(simd, &[u32::MIN, 3001, 3002, u32::MAX - 1]), + ]; + let mut dest = [0_u32; 16]; + simd.store_four_interleaved_u32x4(vectors, &mut dest); + assert_eq!( + dest, + [ + 0, + 1000, + 2000, + u32::MIN, + 1, + 1001, + 2001, + 3001, + u32::MAX, + 1002, + 2002, + 3002, + 3, + 1003, + 2003, + u32::MAX - 1, + ] + ); +} + +#[simd_test] +fn store_four_interleaved_i64x2(simd: S) { + let vectors = [ + i64x2::from_slice(simd, &[i64::MIN, -3]), + i64x2::from_slice(simd, &[-2, -1]), + i64x2::from_slice(simd, &[0, 1]), + i64x2::from_slice(simd, &[2, i64::MAX]), + ]; + let mut dest = [0_i64; 8]; + simd.store_four_interleaved_i64x2(vectors, &mut dest); + assert_eq!(dest, [i64::MIN, -2, 0, 2, -3, -1, 1, i64::MAX]); +} + +#[simd_test] +fn store_four_interleaved_u64x2(simd: S) { + let vectors = [ + u64x2::from_slice(simd, &[u64::MIN, 2]), + u64x2::from_slice(simd, &[3, 4]), + u64x2::from_slice(simd, &[5, 6]), + u64x2::from_slice(simd, &[7, u64::MAX]), + ]; + let mut dest = [0_u64; 8]; + simd.store_four_interleaved_u64x2(vectors, &mut dest); + assert_eq!(dest, [u64::MIN, 3, 5, 7, 2, 4, 6, u64::MAX]); +} diff --git a/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs b/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs deleted file mode 100644 index 22e67a04..00000000 --- a/fearless_simd_tests/tests/harness/ops/store_interleaved_128.rs +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright 2026 the Fearless_SIMD Authors -// SPDX-License-Identifier: Apache-2.0 OR MIT - -use fearless_simd::*; -use fearless_simd_dev_macros::simd_test; - -// One concrete test row per supported vector type. - -#[simd_test] -fn store_interleaved_128_f32x16(simd: S) { - let input = [ - 0.0, - f32::NAN, - f32::INFINITY, - -3.0, - 4.0, - -0.0, - 6.0, - f32::NEG_INFINITY, - 8.0, - 9.0, - -10.0, - 11.0, - f32::MIN, - 13.0, - f32::MAX, - 15.0, - ]; - let a = f32x16::from_slice(simd, &input); - let mut dest = [0.0_f32; 16]; - simd.store_interleaved_128_f32x16(a, &mut dest); - - let expected = [ - 0.0, - 4.0, - 8.0, - f32::MIN, - f32::NAN, - -0.0, - 9.0, - 13.0, - f32::INFINITY, - 6.0, - -10.0, - f32::MAX, - -3.0, - f32::NEG_INFINITY, - 11.0, - 15.0, - ]; - - // Note: f32::NAN != f32::NAN hence we compare the bit pattern. - assert_eq!(dest.map(f32::to_bits), expected.map(f32::to_bits)); -} - -#[simd_test] -fn store_interleaved_128_f64x8(simd: S) { - let input = [ - 0.0, - -0.0, - f64::NAN, - f64::MIN, - f64::INFINITY, - f64::NEG_INFINITY, - -3.0, - f64::MAX, - ]; - let a = f64x8::from_slice(simd, &input); - let mut dest = [0.0_f64; 8]; - simd.store_interleaved_128_f64x8(a, &mut dest); - - let expected = [ - 0.0, - f64::NAN, - f64::INFINITY, - -3.0, - -0.0, - f64::MIN, - f64::NEG_INFINITY, - f64::MAX, - ]; - - // Note: f64::NAN != f64::NAN hence we compare the bit pattern. - assert_eq!(dest.map(f64::to_bits), expected.map(f64::to_bits)); -} - -#[simd_test] -fn store_interleaved_128_i8x64(simd: S) { - let input: [i8; 64] = [ - -32, -31, -30, -29, -28, -27, -26, -25, -24, -23, -22, -21, -20, -19, -18, -17, -16, -15, - -14, -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - ]; - let a = i8x64::from_slice(simd, &input); - let mut dest = [0_i8; 64]; - simd.store_interleaved_128_i8x64(a, &mut dest); - - let expected = [ - -32, -16, 0, 16, -31, -15, 1, 17, -30, -14, 2, 18, -29, -13, 3, 19, -28, -12, 4, 20, -27, - -11, 5, 21, -26, -10, 6, 22, -25, -9, 7, 23, -24, -8, 8, 24, -23, -7, 9, 25, -22, -6, 10, - 26, -21, -5, 11, 27, -20, -4, 12, 28, -19, -3, 13, 29, -18, -2, 14, 30, -17, -1, 15, 31, - ]; - - assert_eq!(dest, expected); -} - -#[simd_test] -fn store_interleaved_128_u8x64(simd: S) { - let input: [u8; 64] = [ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, - ]; - let a = u8x64::from_slice(simd, &input); - let mut dest = [0_u8; 64]; - simd.store_interleaved_128_u8x64(a, &mut dest); - - let expected = [ - 0, 16, 32, 48, 1, 17, 33, 49, 2, 18, 34, 50, 3, 19, 35, 51, 4, 20, 36, 52, 5, 21, 37, 53, - 6, 22, 38, 54, 7, 23, 39, 55, 8, 24, 40, 56, 9, 25, 41, 57, 10, 26, 42, 58, 11, 27, 43, 59, - 12, 28, 44, 60, 13, 29, 45, 61, 14, 30, 46, 62, 15, 31, 47, 63, - ]; - - assert_eq!(dest, expected); -} - -#[simd_test] -fn store_interleaved_128_i16x32(simd: S) { - let input: [i16; 32] = [ - -16, -15, -14, -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, - 7, 8, 9, 10, 11, 12, 13, 14, 15, - ]; - let a = i16x32::from_slice(simd, &input); - let mut dest = [0_i16; 32]; - simd.store_interleaved_128_i16x32(a, &mut dest); - - let expected = [ - -16, -8, 0, 8, -15, -7, 1, 9, -14, -6, 2, 10, -13, -5, 3, 11, -12, -4, 4, 12, -11, -3, 5, - 13, -10, -2, 6, 14, -9, -1, 7, 15, - ]; - - assert_eq!(dest, expected); -} - -#[simd_test] -fn store_interleaved_128_u16x32(simd: S) { - let input: [u16; 32] = [ - 0, 1, 2, 3, 4, 5, 6, 7, 100, 101, 102, 103, 104, 105, 106, 107, 200, 201, 202, 203, 204, - 205, 206, 207, 300, 301, 302, 303, 304, 305, 306, 307, - ]; - let a = u16x32::from_slice(simd, &input); - let mut dest = [0_u16; 32]; - simd.store_interleaved_128_u16x32(a, &mut dest); - - let expected = [ - 0, 100, 200, 300, 1, 101, 201, 301, 2, 102, 202, 302, 3, 103, 203, 303, 4, 104, 204, 304, - 5, 105, 205, 305, 6, 106, 206, 306, 7, 107, 207, 307, - ]; - - assert_eq!(dest, expected); -} - -#[simd_test] -fn store_interleaved_128_i32x16(simd: S) { - let input: [i32; 16] = [-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7]; - let a = i32x16::from_slice(simd, &input); - let mut dest = [0_i32; 16]; - simd.store_interleaved_128_i32x16(a, &mut dest); - - let expected = [-8, -4, 0, 4, -7, -3, 1, 5, -6, -2, 2, 6, -5, -1, 3, 7]; - - assert_eq!(dest, expected); -} - -#[simd_test] -fn store_interleaved_128_u32x16(simd: S) { - let input: [u32; 16] = [ - 0, - 1, - u32::MAX, - 3, - 1000, - 1001, - 1002, - 1003, - 2000, - 2001, - 2002, - 2003, - u32::MIN, - 3001, - 3002, - u32::MAX - 1, - ]; - let a = u32x16::from_slice(simd, &input); - let mut dest = [0_u32; 16]; - simd.store_interleaved_128_u32x16(a, &mut dest); - - let expected = [ - 0, - 1000, - 2000, - u32::MIN, - 1, - 1001, - 2001, - 3001, - u32::MAX, - 1002, - 2002, - 3002, - 3, - 1003, - 2003, - u32::MAX - 1, - ]; - - assert_eq!(dest, expected); -} - -// Additional concrete rows for this operation. - -#[simd_test] -fn store_interleaved_128_u64x8(simd: S) { - let a = u64x8::from_slice( - simd, - &[1_u64, 2_u64, 3_u64, 4_u64, 5_u64, 6_u64, 7_u64, 8_u64], - ); - let mut out = [0_u64; 8]; - simd.store_interleaved_128_u64x8(a, &mut out); - assert_eq!( - out, - [1_u64, 3_u64, 5_u64, 7_u64, 2_u64, 4_u64, 6_u64, 8_u64] - ); -} - -#[simd_test] -fn store_interleaved_128_i64x8(simd: S) { - let a = i64x8::from_slice( - simd, - &[-4_i64, -3_i64, -2_i64, -1_i64, 0_i64, 1_i64, 2_i64, 3_i64], - ); - let mut out = [0_i64; 8]; - simd.store_interleaved_128_i64x8(a, &mut out); - assert_eq!( - out, - [-4_i64, -2_i64, 0_i64, 2_i64, -3_i64, -1_i64, 1_i64, 3_i64] - ); -} From f62652d90cbc423dc967a5ea6b05158da388519a Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Mon, 3 Aug 2026 12:23:18 +0100 Subject: [PATCH 4/5] Placate clippy --- fearless_simd/src/generated/neon.rs | 20 ++++++++++---------- fearless_simd_gen/src/mk_neon.rs | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index dcca19fe..48403d65 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -588,7 +588,7 @@ impl Simd for Neon { let v1: float32x4_t = vectors[1usize].into(); let v2: float32x4_t = vectors[2usize].into(); let v3: float32x4_t = vectors[3usize].into(); - vst4q_f32(dest.as_mut_ptr(), float32x4x4_t(v0, v1, v2, v3)) + vst4q_f32(dest.as_mut_ptr(), float32x4x4_t(v0, v1, v2, v3)); } } #[inline(always)] @@ -1088,7 +1088,7 @@ impl Simd for Neon { let v1: int8x16_t = vectors[1usize].into(); let v2: int8x16_t = vectors[2usize].into(); let v3: int8x16_t = vectors[3usize].into(); - vst4q_s8(dest.as_mut_ptr(), int8x16x4_t(v0, v1, v2, v3)) + vst4q_s8(dest.as_mut_ptr(), int8x16x4_t(v0, v1, v2, v3)); } } #[inline(always)] @@ -1550,7 +1550,7 @@ impl Simd for Neon { let v1: uint8x16_t = vectors[1usize].into(); let v2: uint8x16_t = vectors[2usize].into(); let v3: uint8x16_t = vectors[3usize].into(); - vst4q_u8(dest.as_mut_ptr(), uint8x16x4_t(v0, v1, v2, v3)) + vst4q_u8(dest.as_mut_ptr(), uint8x16x4_t(v0, v1, v2, v3)); } } #[inline(always)] @@ -2189,7 +2189,7 @@ impl Simd for Neon { let v1: int16x8_t = vectors[1usize].into(); let v2: int16x8_t = vectors[2usize].into(); let v3: int16x8_t = vectors[3usize].into(); - vst4q_s16(dest.as_mut_ptr(), int16x8x4_t(v0, v1, v2, v3)) + vst4q_s16(dest.as_mut_ptr(), int16x8x4_t(v0, v1, v2, v3)); } } #[inline(always)] @@ -2619,7 +2619,7 @@ impl Simd for Neon { let v1: uint16x8_t = vectors[1usize].into(); let v2: uint16x8_t = vectors[2usize].into(); let v3: uint16x8_t = vectors[3usize].into(); - vst4q_u16(dest.as_mut_ptr(), uint16x8x4_t(v0, v1, v2, v3)) + vst4q_u16(dest.as_mut_ptr(), uint16x8x4_t(v0, v1, v2, v3)); } } #[inline(always)] @@ -3222,7 +3222,7 @@ impl Simd for Neon { let v1: int32x4_t = vectors[1usize].into(); let v2: int32x4_t = vectors[2usize].into(); let v3: int32x4_t = vectors[3usize].into(); - vst4q_s32(dest.as_mut_ptr(), int32x4x4_t(v0, v1, v2, v3)) + vst4q_s32(dest.as_mut_ptr(), int32x4x4_t(v0, v1, v2, v3)); } } #[inline(always)] @@ -3646,7 +3646,7 @@ impl Simd for Neon { let v1: uint32x4_t = vectors[1usize].into(); let v2: uint32x4_t = vectors[2usize].into(); let v3: uint32x4_t = vectors[3usize].into(); - vst4q_u32(dest.as_mut_ptr(), uint32x4x4_t(v0, v1, v2, v3)) + vst4q_u32(dest.as_mut_ptr(), uint32x4x4_t(v0, v1, v2, v3)); } } #[inline(always)] @@ -4313,7 +4313,7 @@ impl Simd for Neon { let v1: float64x2_t = vectors[1usize].into(); let v2: float64x2_t = vectors[2usize].into(); let v3: float64x2_t = vectors[3usize].into(); - vst4q_f64(dest.as_mut_ptr(), float64x2x4_t(v0, v1, v2, v3)) + vst4q_f64(dest.as_mut_ptr(), float64x2x4_t(v0, v1, v2, v3)); } } #[inline(always)] @@ -4723,7 +4723,7 @@ impl Simd for Neon { let v1: int64x2_t = vectors[1usize].into(); let v2: int64x2_t = vectors[2usize].into(); let v3: int64x2_t = vectors[3usize].into(); - vst4q_s64(dest.as_mut_ptr(), int64x2x4_t(v0, v1, v2, v3)) + vst4q_s64(dest.as_mut_ptr(), int64x2x4_t(v0, v1, v2, v3)); } } #[inline(always)] @@ -5123,7 +5123,7 @@ impl Simd for Neon { let v1: uint64x2_t = vectors[1usize].into(); let v2: uint64x2_t = vectors[2usize].into(); let v3: uint64x2_t = vectors[3usize].into(); - vst4q_u64(dest.as_mut_ptr(), uint64x2x4_t(v0, v1, v2, v3)) + vst4q_u64(dest.as_mut_ptr(), uint64x2x4_t(v0, v1, v2, v3)); } } #[inline(always)] diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index 5138f7bb..71a1efa6 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -192,7 +192,7 @@ impl Level for Neon { #method_sig { unsafe { #(#native_decls)* - #intrinsic(dest.as_mut_ptr(), #aggregate_ty(#(#native_values),*)) + #intrinsic(dest.as_mut_ptr(), #aggregate_ty(#(#native_values),*)); } } } From 9a2d556ed8f8c615a8d65a67ae41050c5c8d3c70 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Tue, 4 Aug 2026 23:27:55 +0100 Subject: [PATCH 5/5] Scope unsafe more tightly in NEON interleaved loads/stores --- fearless_simd/src/generated/neon.rs | 240 +++++++++++++--------------- fearless_simd_gen/src/mk_neon.rs | 12 +- 2 files changed, 114 insertions(+), 138 deletions(-) diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index 6730117b..54207fdc 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -551,15 +551,13 @@ impl Simd for Neon { } #[inline(always)] fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4; 4usize] { - unsafe { - let native = vld4q_f32(src.as_ptr()); - [ - native.0.simd_into(self), - native.1.simd_into(self), - native.2.simd_into(self), - native.3.simd_into(self), - ] - } + let native = unsafe { vld4q_f32(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] } #[inline(always)] fn store_four_interleaved_f32x4( @@ -567,11 +565,11 @@ impl Simd for Neon { vectors: [f32x4; 4usize], dest: &mut [f32; 16usize], ) -> () { + let v0: float32x4_t = vectors[0usize].into(); + let v1: float32x4_t = vectors[1usize].into(); + let v2: float32x4_t = vectors[2usize].into(); + let v3: float32x4_t = vectors[3usize].into(); unsafe { - let v0: float32x4_t = vectors[0usize].into(); - let v1: float32x4_t = vectors[1usize].into(); - let v2: float32x4_t = vectors[2usize].into(); - let v3: float32x4_t = vectors[3usize].into(); vst4q_f32(dest.as_mut_ptr(), float32x4x4_t(v0, v1, v2, v3)); } } @@ -1035,15 +1033,13 @@ impl Simd for Neon { } #[inline(always)] fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16; 4usize] { - unsafe { - let native = vld4q_s8(src.as_ptr()); - [ - native.0.simd_into(self), - native.1.simd_into(self), - native.2.simd_into(self), - native.3.simd_into(self), - ] - } + let native = unsafe { vld4q_s8(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] } #[inline(always)] fn store_four_interleaved_i8x16( @@ -1051,11 +1047,11 @@ impl Simd for Neon { vectors: [i8x16; 4usize], dest: &mut [i8; 64usize], ) -> () { + let v0: int8x16_t = vectors[0usize].into(); + let v1: int8x16_t = vectors[1usize].into(); + let v2: int8x16_t = vectors[2usize].into(); + let v3: int8x16_t = vectors[3usize].into(); unsafe { - let v0: int8x16_t = vectors[0usize].into(); - let v1: int8x16_t = vectors[1usize].into(); - let v2: int8x16_t = vectors[2usize].into(); - let v3: int8x16_t = vectors[3usize].into(); vst4q_s8(dest.as_mut_ptr(), int8x16x4_t(v0, v1, v2, v3)); } } @@ -1481,15 +1477,13 @@ impl Simd for Neon { } #[inline(always)] fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { - unsafe { - let native = vld4q_u8(src.as_ptr()); - [ - native.0.simd_into(self), - native.1.simd_into(self), - native.2.simd_into(self), - native.3.simd_into(self), - ] - } + let native = unsafe { vld4q_u8(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] } #[inline(always)] fn store_four_interleaved_u8x16( @@ -1497,11 +1491,11 @@ impl Simd for Neon { vectors: [u8x16; 4usize], dest: &mut [u8; 64usize], ) -> () { + let v0: uint8x16_t = vectors[0usize].into(); + let v1: uint8x16_t = vectors[1usize].into(); + let v2: uint8x16_t = vectors[2usize].into(); + let v3: uint8x16_t = vectors[3usize].into(); unsafe { - let v0: uint8x16_t = vectors[0usize].into(); - let v1: uint8x16_t = vectors[1usize].into(); - let v2: uint8x16_t = vectors[2usize].into(); - let v3: uint8x16_t = vectors[3usize].into(); vst4q_u8(dest.as_mut_ptr(), uint8x16x4_t(v0, v1, v2, v3)); } } @@ -2093,15 +2087,13 @@ impl Simd for Neon { } #[inline(always)] fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8; 4usize] { - unsafe { - let native = vld4q_s16(src.as_ptr()); - [ - native.0.simd_into(self), - native.1.simd_into(self), - native.2.simd_into(self), - native.3.simd_into(self), - ] - } + let native = unsafe { vld4q_s16(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] } #[inline(always)] fn store_four_interleaved_i16x8( @@ -2109,11 +2101,11 @@ impl Simd for Neon { vectors: [i16x8; 4usize], dest: &mut [i16; 32usize], ) -> () { + let v0: int16x8_t = vectors[0usize].into(); + let v1: int16x8_t = vectors[1usize].into(); + let v2: int16x8_t = vectors[2usize].into(); + let v3: int16x8_t = vectors[3usize].into(); unsafe { - let v0: int16x8_t = vectors[0usize].into(); - let v1: int16x8_t = vectors[1usize].into(); - let v2: int16x8_t = vectors[2usize].into(); - let v3: int16x8_t = vectors[3usize].into(); vst4q_s16(dest.as_mut_ptr(), int16x8x4_t(v0, v1, v2, v3)); } } @@ -2507,15 +2499,13 @@ impl Simd for Neon { } #[inline(always)] fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8; 4usize] { - unsafe { - let native = vld4q_u16(src.as_ptr()); - [ - native.0.simd_into(self), - native.1.simd_into(self), - native.2.simd_into(self), - native.3.simd_into(self), - ] - } + let native = unsafe { vld4q_u16(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] } #[inline(always)] fn store_four_interleaved_u16x8( @@ -2523,11 +2513,11 @@ impl Simd for Neon { vectors: [u16x8; 4usize], dest: &mut [u16; 32usize], ) -> () { + let v0: uint16x8_t = vectors[0usize].into(); + let v1: uint16x8_t = vectors[1usize].into(); + let v2: uint16x8_t = vectors[2usize].into(); + let v3: uint16x8_t = vectors[3usize].into(); unsafe { - let v0: uint16x8_t = vectors[0usize].into(); - let v1: uint16x8_t = vectors[1usize].into(); - let v2: uint16x8_t = vectors[2usize].into(); - let v3: uint16x8_t = vectors[3usize].into(); vst4q_u16(dest.as_mut_ptr(), uint16x8x4_t(v0, v1, v2, v3)); } } @@ -3083,15 +3073,13 @@ impl Simd for Neon { } #[inline(always)] fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4; 4usize] { - unsafe { - let native = vld4q_s32(src.as_ptr()); - [ - native.0.simd_into(self), - native.1.simd_into(self), - native.2.simd_into(self), - native.3.simd_into(self), - ] - } + let native = unsafe { vld4q_s32(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] } #[inline(always)] fn store_four_interleaved_i32x4( @@ -3099,11 +3087,11 @@ impl Simd for Neon { vectors: [i32x4; 4usize], dest: &mut [i32; 16usize], ) -> () { + let v0: int32x4_t = vectors[0usize].into(); + let v1: int32x4_t = vectors[1usize].into(); + let v2: int32x4_t = vectors[2usize].into(); + let v3: int32x4_t = vectors[3usize].into(); unsafe { - let v0: int32x4_t = vectors[0usize].into(); - let v1: int32x4_t = vectors[1usize].into(); - let v2: int32x4_t = vectors[2usize].into(); - let v3: int32x4_t = vectors[3usize].into(); vst4q_s32(dest.as_mut_ptr(), int32x4x4_t(v0, v1, v2, v3)); } } @@ -3491,15 +3479,13 @@ impl Simd for Neon { } #[inline(always)] fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4; 4usize] { - unsafe { - let native = vld4q_u32(src.as_ptr()); - [ - native.0.simd_into(self), - native.1.simd_into(self), - native.2.simd_into(self), - native.3.simd_into(self), - ] - } + let native = unsafe { vld4q_u32(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] } #[inline(always)] fn store_four_interleaved_u32x4( @@ -3507,11 +3493,11 @@ impl Simd for Neon { vectors: [u32x4; 4usize], dest: &mut [u32; 16usize], ) -> () { + let v0: uint32x4_t = vectors[0usize].into(); + let v1: uint32x4_t = vectors[1usize].into(); + let v2: uint32x4_t = vectors[2usize].into(); + let v3: uint32x4_t = vectors[3usize].into(); unsafe { - let v0: uint32x4_t = vectors[0usize].into(); - let v1: uint32x4_t = vectors[1usize].into(); - let v2: uint32x4_t = vectors[2usize].into(); - let v3: uint32x4_t = vectors[3usize].into(); vst4q_u32(dest.as_mut_ptr(), uint32x4x4_t(v0, v1, v2, v3)); } } @@ -4131,15 +4117,13 @@ impl Simd for Neon { } #[inline(always)] fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2; 4usize] { - unsafe { - let native = vld4q_f64(src.as_ptr()); - [ - native.0.simd_into(self), - native.1.simd_into(self), - native.2.simd_into(self), - native.3.simd_into(self), - ] - } + let native = unsafe { vld4q_f64(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] } #[inline(always)] fn store_four_interleaved_f64x2( @@ -4147,11 +4131,11 @@ impl Simd for Neon { vectors: [f64x2; 4usize], dest: &mut [f64; 8usize], ) -> () { + let v0: float64x2_t = vectors[0usize].into(); + let v1: float64x2_t = vectors[1usize].into(); + let v2: float64x2_t = vectors[2usize].into(); + let v3: float64x2_t = vectors[3usize].into(); unsafe { - let v0: float64x2_t = vectors[0usize].into(); - let v1: float64x2_t = vectors[1usize].into(); - let v2: float64x2_t = vectors[2usize].into(); - let v3: float64x2_t = vectors[3usize].into(); vst4q_f64(dest.as_mut_ptr(), float64x2x4_t(v0, v1, v2, v3)); } } @@ -4525,15 +4509,13 @@ impl Simd for Neon { } #[inline(always)] fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2; 4usize] { - unsafe { - let native = vld4q_s64(src.as_ptr()); - [ - native.0.simd_into(self), - native.1.simd_into(self), - native.2.simd_into(self), - native.3.simd_into(self), - ] - } + let native = unsafe { vld4q_s64(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] } #[inline(always)] fn store_four_interleaved_i64x2( @@ -4541,11 +4523,11 @@ impl Simd for Neon { vectors: [i64x2; 4usize], dest: &mut [i64; 8usize], ) -> () { + let v0: int64x2_t = vectors[0usize].into(); + let v1: int64x2_t = vectors[1usize].into(); + let v2: int64x2_t = vectors[2usize].into(); + let v3: int64x2_t = vectors[3usize].into(); unsafe { - let v0: int64x2_t = vectors[0usize].into(); - let v1: int64x2_t = vectors[1usize].into(); - let v2: int64x2_t = vectors[2usize].into(); - let v3: int64x2_t = vectors[3usize].into(); vst4q_s64(dest.as_mut_ptr(), int64x2x4_t(v0, v1, v2, v3)); } } @@ -4909,15 +4891,13 @@ impl Simd for Neon { } #[inline(always)] fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2; 4usize] { - unsafe { - let native = vld4q_u64(src.as_ptr()); - [ - native.0.simd_into(self), - native.1.simd_into(self), - native.2.simd_into(self), - native.3.simd_into(self), - ] - } + let native = unsafe { vld4q_u64(src.as_ptr()) }; + [ + native.0.simd_into(self), + native.1.simd_into(self), + native.2.simd_into(self), + native.3.simd_into(self), + ] } #[inline(always)] fn store_four_interleaved_u64x2( @@ -4925,11 +4905,11 @@ impl Simd for Neon { vectors: [u64x2; 4usize], dest: &mut [u64; 8usize], ) -> () { + let v0: uint64x2_t = vectors[0usize].into(); + let v1: uint64x2_t = vectors[1usize].into(); + let v2: uint64x2_t = vectors[2usize].into(); + let v3: uint64x2_t = vectors[3usize].into(); unsafe { - let v0: uint64x2_t = vectors[0usize].into(); - let v1: uint64x2_t = vectors[1usize].into(); - let v2: uint64x2_t = vectors[2usize].into(); - let v3: uint64x2_t = vectors[3usize].into(); vst4q_u64(dest.as_mut_ptr(), uint64x2x4_t(v0, v1, v2, v3)); } } diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index 8b6c847c..0412f74d 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -160,10 +160,8 @@ impl Level for Neon { quote! { #method_sig { - unsafe { - let native = #intrinsic(src.as_ptr()); - [#(native.#fields.simd_into(self)),*] - } + let native = unsafe { #intrinsic(src.as_ptr()) }; + [#(native.#fields.simd_into(self)),*] } } } @@ -189,10 +187,8 @@ impl Level for Neon { quote! { #method_sig { - unsafe { - #(#native_decls)* - #intrinsic(dest.as_mut_ptr(), #aggregate_ty(#(#native_values),*)); - } + #(#native_decls)* + unsafe { #intrinsic(dest.as_mut_ptr(), #aggregate_ty(#(#native_values),*)); } } } }