25#include <xvec/detail/config.hpp>
26#include <xvec/detail/utilities.hpp>
27#include <xvec/detail/core.hpp>
29#include <xvec/x86/permute.hpp>
31namespace _XVEC_NAMESPACE::simd {
36#if defined(__AVX512F__)
37template<
typename _Tp,
typename _Abi>
38constexpr basic_vec<_Tp, _Abi>
39select_if_else(x86_avx512_tag,
const typename basic_vec<_Tp, _Abi>::mask_type& if_mask,
40 const basic_vec<_Tp, _Abi>& true_value,
const basic_vec<_Tp, _Abi>& false_value)
42 target_overloads impl {
43 [=]<xmm_register<std::uint8_t> _Vec>(_Vec if_true, _Vec if_false,
auto m)
44 {
return _mm_mask_blend_epi8(__mmask16(m.to_register()), if_false.to_register(), if_true.to_register()); },
45 [=]<ymm_register<std::uint8_t> _Vec>(_Vec if_true, _Vec if_false,
auto m)
46 {
return _mm256_mask_blend_epi8(__mmask32(m.to_register()), if_false.to_register(), if_true.to_register()); },
47 [=]<zmm_register<std::uint8_t> _Vec>(_Vec if_true, _Vec if_false,
auto m)
48 {
return _mm512_mask_blend_epi8(__mmask64(m.to_register()), if_false.to_register(), if_true.to_register()); },
50 [=]<xmm_register<std::uint16_t> _Vec>(_Vec if_true, _Vec if_false,
auto m)
51 {
return _mm_mask_blend_epi16(__mmask8(m.to_register()), if_false.to_register(), if_true.to_register()); },
52 [=]<ymm_register<std::uint16_t> _Vec>(_Vec if_true, _Vec if_false,
auto m)
53 {
return _mm256_mask_blend_epi16(__mmask16(m.to_register()), if_false.to_register(), if_true.to_register()); },
54 [=]<zmm_register<std::uint16_t> _Vec>(_Vec if_true, _Vec if_false,
auto m)
55 {
return _mm512_mask_blend_epi16(__mmask32(m.to_register()), if_false.to_register(), if_true.to_register()); },
57 [=]<xmm_register<std::uint32_t> _Vec>(_Vec if_true, _Vec if_false,
auto m)
58 {
return _mm_mask_blend_epi32(__mmask8(m.to_register()), if_false.to_register(), if_true.to_register()); },
59 [=]<ymm_register<std::uint32_t> _Vec>(_Vec if_true, _Vec if_false,
auto m)
60 {
return _mm256_mask_blend_epi32(__mmask8(m.to_register()), if_false.to_register(), if_true.to_register()); },
61 [=]<zmm_register<std::uint32_t> _Vec>(_Vec if_true, _Vec if_false,
auto m)
62 {
return _mm512_mask_blend_epi32(__mmask16(m.to_register()), if_false.to_register(), if_true.to_register()); },
64 [=]<xmm_register<std::uint64_t> _Vec>(_Vec if_true, _Vec if_false,
auto m)
65 {
return _mm_mask_blend_epi64(__mmask8(m.to_register()), if_false.to_register(), if_true.to_register()); },
66 [=]<ymm_register<std::uint64_t> _Vec>(_Vec if_true, _Vec if_false,
auto m)
67 {
return _mm256_mask_blend_epi64(__mmask8(m.to_register()), if_false.to_register(), if_true.to_register()); },
68 [=]<zmm_register<std::uint64_t> _Vec>(_Vec if_true, _Vec if_false,
auto m)
69 {
return _mm512_mask_blend_epi64(__mmask8(m.to_register()), if_false.to_register(), if_true.to_register()); },
71 [=]<xmm_register<unsigned __int128> _Vec>(_Vec if_true, _Vec if_false,
auto m)
72 {
return (m.to_register() & 1) == 0 ? if_false.to_register() : if_true.to_register(); },
73 [=]<ymm_register<unsigned __int128> _Vec>(_Vec if_true, _Vec if_false,
auto m)
74 {
return _mm256_mask_blend_epi64(__mmask8(dupBits(m.to_register())), if_false.to_register(), if_true.to_register()); },
75 [=]<zmm_register<unsigned __int128> _Vec>(_Vec if_true, _Vec if_false,
auto m)
76 {
return _mm512_mask_blend_epi64(__mmask8(dupBits(m.to_register())), if_false.to_register(), if_true.to_register()); },
78 [=](
auto unhandled,
auto,
auto) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics");
return unhandled; }
81 auto wrapper = [=]<
typename _Vec>(_Vec ift, _Vec iff,
auto ifm) {
82 auto r = _Vec(impl(ift, iff, ifm));
83 return simd_bit_cast<_Tp>(r);
86 constexpr int _Np = basic_vec<_Tp, _Abi>::size;
87 using _C = container_for_type<_Tp>;
92 if (std::is_constant_evaluated())
98 auto wideM = vec<_C, _Np>([=](
auto i) -> _C {
if (if_mask[i])
return ~_C();
else return _C(); }).to_builtin();
99 basic_vec<_C, _Abi> r = (wideM & trueAsC.to_builtin()) | ((~wideM) & falseAsC.to_builtin());
100 return simd_bit_cast<_Tp>(r);
103 return chunked_invoke(wrapper, trueAsC, falseAsC, mask<_C, _Np>(if_mask));
107inline target_overloads msb_to_bitmask_impl {
109 [](xmm_register<std::uint8_t>
auto m) -> std::uint16_t {
return _mm_movemask_epi8(m.to_register()); },
110 [](xmm_register<std::uint16_t>
auto m) -> std::uint8_t {
114 const auto B = simd_bit_cast<std::uint8_t>(m);
115 const auto reqBits = permute<16>(B, [](
auto idx) {
return idx <
decltype(m)::size() ? idx * 2 : zero_element; });
116 return _mm_movemask_epi8(reqBits.to_register());
118 [](xmm_register<std::uint32_t>
auto m) -> std::uint8_t {
return _mm_movemask_ps(__m128(m.to_register()));},
119 [](xmm_register<std::uint64_t>
auto m) -> std::uint8_t {
return _mm_movemask_pd(__m128d(m.to_register()));},
120 [](xmm_register<unsigned __int128>
auto m) -> std::uint8_t {
return _mm_movemask_pd(__m128d(m.to_register())) >> 1;},
124 [](ymm_register<std::uint8_t>
auto m) -> std::uint32_t {
return _mm256_movemask_epi8(m.to_register()); },
125 [](ymm_register<std::uint16_t>
auto m) -> std::uint16_t {
return _pext_u64(_mm256_movemask_epi8(m.to_register()), 0xAAAAAAAA); },
126#elif defined(__AVX__)
128 [](ymm_register<std::uint8_t>
auto m) -> std::uint32_t {
130 auto lower = _mm_movemask_epi8(_mm256_castsi256_si128(m.to_register()));
131 auto upper = _mm_movemask_epi8(_mm256_extractf128_si256(m.to_register(), 1));
132 return std::uint32_t(lower) | (std::uint32_t(upper) << 16);
134 [](ymm_register<std::uint16_t>
auto m) -> std::uint16_t {
136 auto to8 = rebind_cast<std::uint8_t>(m >> 8);
137 return _mm_movemask_epi8(to8.to_register());
142 [](ymm_register<std::uint32_t>
auto m) -> std::uint8_t {
return _mm256_movemask_ps(__m256(m.to_register()));},
143 [](ymm_register<std::uint64_t>
auto m) -> std::uint8_t {
return _mm256_movemask_pd(__m256d(m.to_register()));},
144 [](ymm_register<unsigned __int128>
auto m) -> std::uint8_t {
145 auto fourMsbs = _mm256_movemask_pd(__m256d(m.to_register()));
147 #if defined(__BMI2__)
148 return _pext_u64(fourMsbs, 0b1010);
150 auto masked4 = (fourMsbs & 0b1010) >> 1;
151 return ((masked4 >> 1) | masked4) & 0b11;
156 [](
auto unhandled) ->
unsigned long {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics");
return 0;}
159template<mask_type _Mp>
160constexpr std::bitset<_Mp::size>
161mask_to_bitset(x86_sse_tag,
const _Mp& mask)
noexcept
163 constexpr int _Np = _Mp::size;
164 using _R = std::bitset<_Np>;
168 auto processBlock = [&](
auto v, simd_size_type idx) {
169 auto msbs = msb_to_bitmask_impl(
vec_as_container(v)) & ((1ULL << v.size()) - 1);
170 result |= _R(msbs) << idx;
176template<mask_type _Mp>
177constexpr std::bitset<_Mp::size>
178mask_to_bitset(x86_avx512_tag,
const _Mp& mask)
noexcept
180 constexpr int _Np = _Mp::size;
181 std::bitset<_Np> result;
185 for (simd_size_type i = 0; i<((_Np + 63) / 64); ++i)
187 const auto b = std::bitset<_Np>(std::uint64_t(
mask.
to_builtin() >> (64 * i)));
188 result |= (b << (64 * i));
195template<std::
floating_po
int _Tp,
typename _Abi>
196constexpr basic_vec<_Tp, _Abi>
fma(x86_avx2_tag,
const basic_vec<_Tp, _Abi>& x,
const basic_vec<_Tp, _Abi>& y,
const basic_vec<_Tp, _Abi>& acc)
198 target_overloads impl {
200 [=](xmm_register<float>
auto x,
auto y,
auto acc) {
return _mm_fmadd_ps(x.to_register(), y.to_register(), acc.to_register()); },
201 [=](xmm_register<double>
auto x,
auto y,
auto acc) {
return _mm_fmadd_pd(x.to_register(), y.to_register(), acc.to_register()); },
204 [=](ymm_register<float>
auto x,
auto y,
auto acc) {
return _mm256_fmadd_ps(x.to_register(), y.to_register(), acc.to_register()); },
205 [=](ymm_register<double>
auto x,
auto y,
auto acc) {
return _mm256_fmadd_pd(x.to_register(), y.to_register(), acc.to_register()); },
208#if defined(__AVX512F__)
209 [=](zmm_register<float>
auto x,
auto y,
auto acc) {
return _mm512_fmadd_ps(x.to_register(), y.to_register(), acc.to_register()); },
210 [=](zmm_register<double>
auto x,
auto y,
auto acc) {
return _mm512_fmadd_pd(x.to_register(), y.to_register(), acc.to_register()); },
213 [=](
auto unhandled,
auto,
auto) ->
decltype(unhandled) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics"); }
216 auto wrapper = [=](
auto a,
auto b,
auto c) {
return decltype(a)(impl(a, b, c)); };
220template<std::
floating_po
int _Tp,
typename _Abi>
223fmaddsub(x86_avx2_tag,
const basic_vec<_Tp, _Abi>& x,
const basic_vec<_Tp, _Abi>& y,
const basic_vec<_Tp, _Abi>& acc)
225 target_overloads impl {
227 [=](xmm_register<float>
auto x,
auto y,
auto acc) {
return _mm_fmaddsub_ps(x.to_register(), y.to_register(), acc.to_register()); },
228 [=](xmm_register<double>
auto x,
auto y,
auto acc) {
return _mm_fmaddsub_pd(x.to_register(), y.to_register(), acc.to_register()); },
231 [=](ymm_register<float>
auto x,
auto y,
auto acc) {
return _mm256_fmaddsub_ps(x.to_register(), y.to_register(), acc.to_register()); },
232 [=](ymm_register<double>
auto x,
auto y,
auto acc) {
return _mm256_fmaddsub_pd(x.to_register(), y.to_register(), acc.to_register()); },
235#if defined(__AVX512F__)
236 [=](zmm_register<float>
auto x,
auto y,
auto acc) {
return _mm512_fmaddsub_ps(x.to_register(), y.to_register(), acc.to_register()); },
237 [=](zmm_register<double>
auto x,
auto y,
auto acc) {
return _mm512_fmaddsub_pd(x.to_register(), y.to_register(), acc.to_register()); },
240 [=](
auto unhandled,
auto,
auto) ->
decltype(unhandled) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics"); }
243 auto wrapper = [=](
auto a,
auto b,
auto c) {
return decltype(a)(impl(a, b, c)); };
247template<std::
floating_po
int _Tp,
typename _Abi>
250fmsubadd(x86_avx2_tag,
const basic_vec<_Tp, _Abi>& x,
const basic_vec<_Tp, _Abi>& y,
const basic_vec<_Tp, _Abi>& acc)
252 target_overloads impl {
254 [=](xmm_register<float>
auto x,
auto y,
auto acc) {
return _mm_fmsubadd_ps(x.to_register(), y.to_register(), acc.to_register()); },
255 [=](xmm_register<double>
auto x,
auto y,
auto acc) {
return _mm_fmsubadd_pd(x.to_register(), y.to_register(), acc.to_register()); },
258 [=](ymm_register<float>
auto x,
auto y,
auto acc) {
return _mm256_fmsubadd_ps(x.to_register(), y.to_register(), acc.to_register()); },
259 [=](ymm_register<double>
auto x,
auto y,
auto acc) {
return _mm256_fmsubadd_pd(x.to_register(), y.to_register(), acc.to_register()); },
262#if defined(__AVX512F__)
263 [=](zmm_register<float>
auto x,
auto y,
auto acc) {
return _mm512_fmsubadd_ps(x.to_register(), y.to_register(), acc.to_register()); },
264 [=](zmm_register<double>
auto x,
auto y,
auto acc) {
return _mm512_fmsubadd_pd(x.to_register(), y.to_register(), acc.to_register()); },
267 [=](
auto unhandled,
auto,
auto) ->
decltype(unhandled) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics"); }
270 auto wrapper = [=](
auto a,
auto b,
auto c) {
return decltype(a)(impl(a, b, c)); };
274#if defined(__AVX512FP16__)
276template<vec_of<_Float16> _Vp>
277constexpr _Vp
fma(x86_avxspr_tag,
const _Vp& x,
const _Vp& y,
const _Vp& acc) {
278 target_overloads impl {
279 [=](xmm_register<_Float16>
auto x,
auto y,
auto acc) {
return _mm_fmadd_ph(x.to_register(), y.to_register(), acc.to_register()); },
280 [=](ymm_register<_Float16>
auto x,
auto y,
auto acc) {
return _mm256_fmadd_ph(x.to_register(), y.to_register(), acc.to_register()); },
281 [=](zmm_register<_Float16>
auto x,
auto y,
auto acc) {
return _mm512_fmadd_ph(x.to_register(), y.to_register(), acc.to_register()); },
282 [=](
auto unhandled,
auto,
auto) ->
decltype(unhandled) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics"); }
285 auto wrapper = [=](
auto a,
auto b,
auto c) {
return decltype(a)(impl(a, b, c)); };
289template<vec_of<std::complex<_Float16>> _Vp>
290constexpr _Vp
fma(x86_avxspr_tag,
const _Vp& x,
const _Vp& y,
const _Vp& acc) {
291 target_overloads impl {
292 [=](xmm_register<_Float16>
auto x,
auto y,
auto acc) {
return _mm_fmadd_pch(x.to_register(), y.to_register(), acc.to_register()); },
293 [=](ymm_register<_Float16>
auto x,
auto y,
auto acc) {
return _mm256_fmadd_pch(x.to_register(), y.to_register(), acc.to_register()); },
294 [=](zmm_register<_Float16>
auto x,
auto y,
auto acc) {
return _mm512_fmadd_pch(x.to_register(), y.to_register(), acc.to_register()); },
295 [=](
auto unhandled,
auto,
auto) ->
decltype(unhandled) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics"); }
298 auto wrapper = [=](
auto a,
auto b,
auto c) {
return decltype(a)(impl(a, b, c)); };
299 auto t =
chunked_invoke(wrapper, simd_bit_cast<_Float16>(x), simd_bit_cast<_Float16>(y), simd_bit_cast<_Float16>(acc));
300 return simd_bit_cast<std::complex<_Float16>>(t);
303template<vec_of<_Float16> _Vp>
304constexpr _Vp
fmaddsub(x86_avxspr_tag,
const _Vp& x,
const _Vp& y,
const _Vp& acc) {
305 target_overloads impl {
306 [=](xmm_register<_Float16>
auto x,
auto y,
auto acc) {
return _mm_fmaddsub_ph(x.to_register(), y.to_register(), acc.to_register()); },
307 [=](ymm_register<_Float16>
auto x,
auto y,
auto acc) {
return _mm256_fmaddsub_ph(x.to_register(), y.to_register(), acc.to_register()); },
308 [=](zmm_register<_Float16>
auto x,
auto y,
auto acc) {
return _mm512_fmaddsub_ph(x.to_register(), y.to_register(), acc.to_register()); },
309 [=](
auto unhandled,
auto,
auto) ->
decltype(unhandled) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics"); }
312 auto wrapper = [=](
auto a,
auto b,
auto c) {
return decltype(a)(impl(a, b, c)); };
316template<vec_of<_Float16> _Vp>
318fmsubadd(x86_avxspr_tag,
const _Vp& x,
const _Vp& y,
const _Vp& acc)
320 target_overloads impl {
321 [=](xmm_register<_Float16>
auto x,
auto y,
auto acc) {
return _mm_fmsubadd_ph(x.to_register(), y.to_register(), acc.to_register()); },
322 [=](ymm_register<_Float16>
auto x,
auto y,
auto acc) {
return _mm256_fmsubadd_ph(x.to_register(), y.to_register(), acc.to_register()); },
323 [=](zmm_register<_Float16>
auto x,
auto y,
auto acc) {
return _mm512_fmsubadd_ph(x.to_register(), y.to_register(), acc.to_register()); },
324 [=](
auto unhandled,
auto,
auto) ->
decltype(unhandled) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics"); }
327 auto wrapper = [=](
auto a,
auto b,
auto c) {
return decltype(a)(impl(a, b, c)); };
331template<vec_of<std::complex<_Float16>> _Vp>
332constexpr _Vp
builtin_operator(x86_avxspr_tag,
const _Vp& x,
const _Vp& y, std::multiplies<>)
334 target_overloads impl {
335 [=](xmm_register<_Float16>
auto x,
auto y) {
return _mm_fmul_pch(x.to_register(), y.to_register()); },
336 [=](ymm_register<_Float16>
auto x,
auto y) {
return _mm256_fmul_pch(x.to_register(), y.to_register()); },
337 [=](zmm_register<_Float16>
auto x,
auto y) {
return _mm512_fmul_pch(x.to_register(), y.to_register()); },
338 [=](
auto unhandled,
auto) ->
decltype(unhandled) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics"); }
341 auto wrapper = [=](
auto a,
auto b) {
return decltype(a)(impl(a, b)); };
342 auto t =
chunked_invoke(wrapper, simd_bit_cast<_Float16>(x), simd_bit_cast<_Float16>(y));
343 return simd_bit_cast<std::complex<_Float16>>(t);
351template<RoundOp _Op,
typename _Tp,
typename _Abi>
352requires (detail::is_floating_point_v<_Tp>)
355 [[maybe_unused]]
constexpr auto sseOp = std::array<int, 5>{
356 _MM_FROUND_TO_POS_INF, _MM_FROUND_TO_NEG_INF, _MM_FROUND_TO_ZERO,
357 _MM_FROUND_TO_NEAREST_INT, _MM_FROUND_CUR_DIRECTION
359 [[maybe_unused]]
constexpr auto avx512Op = std::array<int, 5>{
360 _MM_FROUND_CEIL, _MM_FROUND_FLOOR, _MM_FROUND_TO_ZERO,
361 _MM_FROUND_TO_NEAREST_INT, _MM_FROUND_CUR_DIRECTION
366 [=](xmm_register<float>
auto x) {
return _mm_round_ps(x.to_register(), sseOp); },
367 [=](xmm_register<double>
auto x) {
return _mm_round_pd(x.to_register(), sseOp); },
370 [=](ymm_register<float>
auto x) {
return _mm256_round_ps(x.to_register(), sseOp); },
371 [=](ymm_register<double>
auto x) {
return _mm256_round_pd(x.to_register(), sseOp); },
374#if defined(__AVX512F__)
375 [=](zmm_register<float>
auto x) {
return _mm512_roundscale_ps(x.to_register(), sseOp); },
376 [=](zmm_register<double>
auto x) {
return _mm512_roundscale_pd(x.to_register(), sseOp); },
379#if defined(__AVX512FP16__)
380 [=](xmm_register<_Float16>
auto x) {
return _mm_roundscale_ph(x.to_register(), avx512Op); },
381 [=](ymm_register<_Float16>
auto x) {
return _mm256_roundscale_ph(x.to_register(), avx512Op); },
382 [=](zmm_register<_Float16>
auto x) {
return _mm512_roundscale_ph(x.to_register(), avx512Op); },
385 [=](
auto unhandled) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics");
return unhandled; }
388 if constexpr (_Op == detail::RoundOp::ROUND)
392 constexpr _Tp almost_half = [] {
393 if constexpr (
sizeof(_Tp) == 2)
return static_cast<_Tp
>(0x1.ffcp-2);
394 else if constexpr (
sizeof(_Tp) == 4)
return 0x1.fffffep-2f;
395 else return 0x1.fffffffffffffp-2;
409 auto wrapper = [=](
auto x) {
return decltype(x)(impl(x)); };
426template<vec_type _Vp,
typename _Up, std::
size_t _Extent,
typename _Flags>
427constexpr void store_masked(x86_tag,
const _Vp& value_original, std::span<_Up, _Extent> to,
428 const typename _Vp::mask_type& mask_original, _Flags
flags)
430 mandates_for_store(value_original, to,
flags);
432 using _ToVec = rebind_t<_Up, _Vp>;
433 auto from = _ToVec(value_original);
434 auto from_mask =
typename _ToVec::mask_type(mask_original);
439 constexpr auto numSseElements = 16 /
sizeof(_Up);
440 auto impl = [=]<
typename _Vec>(_Vec x,
auto m,
auto idx) {
442 _mm_maskmoveu_si128(x.to_register(),
grow<numSseElements>(m).to_register(), (
char*)(to.data() + idx));
446 auto boundedMask = from_mask;
448 checkStaticMemoryBounds<
decltype(from)>(
"store", to, from_mask);
450 boundedMask &= from_mask.__mask_from_count(to.size());
457template<vec_type _Vp,
typename _Up, std::
size_t _Extent,
typename _Flags>
458requires (
sizeof(_Up) == 4 ||
sizeof(_Up) == 8)
459constexpr void store_masked(x86_avx2_tag,
const _Vp& value_original, std::span<_Up, _Extent> to,
460 const typename _Vp::mask_type& mask_original, _Flags
flags)
462 mandates_for_store(value_original, to,
flags);
464 using _ToVec = rebind_t<_Up, _Vp>;
465 auto from = _ToVec(value_original);
466 auto from_mask =
typename _ToVec::mask_type(mask_original);
468 auto impl = [=]<
typename _Vec>(_Vec s,
auto m,
auto idx) {
472 constexpr auto numAvxElements = 32 /
sizeof(_Up);
479 if constexpr (
sizeof(_Up) == 4)
480 _mm256_maskstore_epi32(
reinterpret_cast<int*
>(to.data() + idx), rm.to_register(), rc.to_register());
481 else if constexpr (
sizeof(_Up) == 8)
482 _mm256_maskstore_epi64(
reinterpret_cast<long long*
>(to.data() + idx), rm.to_register(), rc.to_register());
484 static_assert(dependent_false<_Vec>,
"Unimplemented masked store");
488 auto boundedMask = from_mask;
489 if constexpr (contains_flag<flag_unchecked>(flags))
490 checkStaticMemoryBounds<_ToVec>(
"store", to, from_mask);
492 boundedMask &= from_mask.__mask_from_count(to.size());
494 chunked_invoke(impl, vec_as_container(from), boundedMask);
498#if defined(__AVX512F__)
500template<vec_type _Vp,
typename _Up, std::
size_t _Extent,
typename _Flags>
501constexpr void store_masked(x86_avx512_tag,
const _Vp& value_original, std::span<_Up, _Extent> to,
502 const typename _Vp::mask_type& mask_original, _Flags flags)
504 mandates_for_store(value_original, to, flags);
506 target_overloads call_insn {
507 [=](xmm_register<std::uint8_t>
auto v,
auto m,
auto p) { _mm_mask_storeu_epi8(p, m, v.to_register()); },
508 [=](xmm_register<std::uint16_t>
auto v,
auto m,
auto p) { _mm_mask_storeu_epi16(p, m, v.to_register()); },
509 [=](xmm_register<std::uint32_t>
auto v,
auto m,
auto p) { _mm_mask_storeu_epi32(p, m, v.to_register()); },
510 [=](xmm_register<std::uint64_t>
auto v,
auto m,
auto p) { _mm_mask_storeu_epi64(p, m, v.to_register()); },
512 [=](ymm_register<std::uint8_t>
auto v,
auto m,
auto p) { _mm256_mask_storeu_epi8(p, m, v.to_register()); },
513 [=](ymm_register<std::uint16_t>
auto v,
auto m,
auto p) { _mm256_mask_storeu_epi16(p, m, v.to_register()); },
514 [=](ymm_register<std::uint32_t>
auto v,
auto m,
auto p) { _mm256_mask_storeu_epi32(p, m, v.to_register()); },
515 [=](ymm_register<std::uint64_t>
auto v,
auto m,
auto p) { _mm256_mask_storeu_epi64(p, m, v.to_register()); },
517 [=](zmm_register<std::uint8_t>
auto v,
auto m,
auto p) { _mm512_mask_storeu_epi8(p, m, v.to_register()); },
518 [=](zmm_register<std::uint16_t>
auto v,
auto m,
auto p) { _mm512_mask_storeu_epi16(p, m, v.to_register()); },
519 [=](zmm_register<std::uint32_t>
auto v,
auto m,
auto p) { _mm512_mask_storeu_epi32(p, m, v.to_register()); },
520 [=](zmm_register<std::uint64_t>
auto v,
auto m,
auto p) { _mm512_mask_storeu_epi64(p, m, v.to_register()); },
522 [=](xmm_register<unsigned __int128>
auto v,
auto m,
auto p) { _mm_mask_storeu_epi64(p, dupBits(m), v.to_register()); },
523 [=](ymm_register<unsigned __int128>
auto v,
auto m,
auto p) { _mm256_mask_storeu_epi64(p, dupBits(m), v.to_register()); },
524 [=](zmm_register<unsigned __int128>
auto v,
auto m,
auto p) { _mm512_mask_storeu_epi64(p, dupBits(m), v.to_register()); },
526 [=](
auto unhandled,
auto,
auto) {
static_assert(
dependent_false<
decltype(unhandled)>,
"Unimplemented partial masked store"); }
529 using _ToVec = rebind_t<_Up, _Vp>;
530 auto from = _ToVec(value_original);
531 auto from_mask =
typename _ToVec::mask_type(mask_original);
533 auto impl = [=]<
typename _Vec>(_Vec x,
auto m,
auto idx) {
535 constexpr auto partialMask =
typename _Vec::mask_type(
true);
536 const auto tm = m.to_builtin() & partialMask.to_builtin();
537 call_insn(x, tm, to.data() + idx);
541 auto boundedMask = from_mask;
542 if constexpr (contains_flag<flag_unchecked>(flags))
543 checkStaticMemoryBounds<_ToVec>(
"store", to, from_mask);
545 boundedMask &= from_mask.__mask_from_count(to.size());
552template<vec_type _Vp,
typename _Up, std::
size_t _Extent,
typename _Flags>
553constexpr void store(x86_tag,
const _Vp& value_original, std::span<_Up, _Extent> to, _Flags
flags)
555 mandates_for_store(value_original, to,
flags);
558 auto ptr = to.data();
568 checkStaticMemoryBounds<decltype(from)>(
"store", to);
572 auto impl = [=]<
typename _Vec>(_Vec s,
auto idx)
574 constexpr auto numBytes = _Vec::size *
sizeof(_Up);
577 if constexpr (numBytes == 16)
578 _mm_storeu_si128(
reinterpret_cast<__m128i *
>(ptr + idx), __m128i(s.to_register()));
580 else if constexpr (numBytes == 32)
581 _mm256_storeu_si256(
reinterpret_cast<__m256i *
>(ptr + idx), __m256i(s.to_register()));
583#if defined(__AVX512F__)
584 else if constexpr (numBytes == 64)
585 _mm512_storeu_si512(ptr + idx, __m512i(s.to_register()));
591 constexpr auto mask =
typename _Vec::mask_type(
true);
600#if defined(__AVX512F__)
602template<vec_type _V,
typename _Up, std::
size_t _Extent,
typename _Flags>
603constexpr _V load_masked(x86_avx512_tag, std::span<_Up, _Extent> from,
const typename _V::mask_type&
mask, _Flags
flags)
605 mandates_for_load(from, _V{}, flags);
611 target_overloads call_insn {
612 [=](xmm_register<std::uint8_t>
auto,
auto m,
auto p) {
return _mm_maskz_loadu_epi8(m, p); },
613 [=](xmm_register<std::uint16_t>
auto,
auto m,
auto p) {
return _mm_maskz_loadu_epi16(m, p); },
614 [=](xmm_register<std::uint32_t>
auto,
auto m,
auto p) {
return _mm_maskz_loadu_epi32(m, p); },
615 [=](xmm_register<std::uint64_t>
auto,
auto m,
auto p) {
return _mm_maskz_loadu_epi64(m, p); },
617 [=](ymm_register<std::uint8_t>
auto,
auto m,
auto p) {
return _mm256_maskz_loadu_epi8(m, p); },
618 [=](ymm_register<std::uint16_t>
auto,
auto m,
auto p) {
return _mm256_maskz_loadu_epi16(m, p); },
619 [=](ymm_register<std::uint32_t>
auto,
auto m,
auto p) {
return _mm256_maskz_loadu_epi32(m, p); },
620 [=](ymm_register<std::uint64_t>
auto,
auto m,
auto p) {
return _mm256_maskz_loadu_epi64(m, p); },
622 [=](zmm_register<std::uint8_t>
auto,
auto m,
auto p) {
return _mm512_maskz_loadu_epi8(m, p); },
623 [=](zmm_register<std::uint16_t>
auto,
auto m,
auto p) {
return _mm512_maskz_loadu_epi16(m, p); },
624 [=](zmm_register<std::uint32_t>
auto,
auto m,
auto p) {
return _mm512_maskz_loadu_epi32(m, p); },
625 [=](zmm_register<std::uint64_t>
auto,
auto m,
auto p) {
return _mm512_maskz_loadu_epi64(m, p); },
628 [=](xmm_register<unsigned __int128>
auto,
auto m,
auto p) {
return _mm_maskz_loadu_epi64(dupBits(m), p); },
629 [=](ymm_register<unsigned __int128>
auto,
auto m,
auto p) {
return _mm256_maskz_loadu_epi64(dupBits(m), p); },
630 [=](zmm_register<unsigned __int128>
auto,
auto m,
auto p) {
return _mm512_maskz_loadu_epi64(dupBits(m), p); },
632 [=](
auto unhandled,
auto,
auto) {
static_assert(
dependent_false<
decltype(unhandled)>,
"Unimplemented partial masked load");
return unhandled; }
635 using SourceType = rebind_t<_Up, _V>;
636 using _C = container_for_type<_Up>;
638 auto impl = [=]<
typename _CS>(_CS x,
auto m,
auto idx) {
641 constexpr auto partialMask =
typename _CS::mask_type(
true);
642 const auto tm = m.to_builtin() & partialMask.to_builtin();
643 return _CS(call_insn(x, tm, from.data() + idx));
647 auto boundedMask =
mask;
648 if constexpr (contains_flag<flag_unchecked>(flags))
649 checkStaticMemoryBounds<SourceType>(
"load_masked", from, mask);
655 auto t =
chunked_invoke(impl, rebind_t<_C, _V>(),
typename rebind_t<_C, _V>::mask_type(boundedMask));
656 return _V(simd_bit_cast<std::remove_cvref_t<_Up>>(t));
659template<vec_type _V,
typename _Up, std::
size_t _Extent,
typename _Flags>
660constexpr auto load(x86_avx512_tag, std::span<_Up, _Extent> from, _Flags flags)
662 if (contains_flag<flag_unchecked>(flags))
663 return load<_V>(generic_tag{}, from, flags);
665 return load_masked<_V>(target, from,
typename _V::mask_type(
true), flags);
676template<
typename _Range, std::integral _Idx,
typename _IdxAbi,
typename... _Flags>
682 std::ranges::contiguous_range<_Range> &&
683 std::ranges::sized_range<_Range> &&
684 (
sizeof(std::ranges::range_value_t<_Range>) <= 8)
689 using _Tp = std::ranges::range_value_t<_Range>;
690 auto data_ptr = std::ranges::data(r);
694#if defined(__AVX512F__)
696 [=](xmm_register<std::int32_t>
auto i,
auto m) {
return _mm_mmask_i32gather_epi32(__m128i(), m.to_register(), i.to_register(), data_ptr,
sizeof(_Tp)); },
697 [=](ymm_register<std::int32_t>
auto i,
auto m) {
return _mm256_mmask_i32gather_epi32(__m256i(), m.to_register(), i.to_register(), data_ptr,
sizeof(_Tp)); },
699 [=](xmm_register<std::int64_t>
auto i,
auto m) {
return _mm_mmask_i64gather_epi64(__m128i(), m.to_register(), i.to_register(), data_ptr,
sizeof(_Tp)); },
700 [=](ymm_register<std::int64_t>
auto i,
auto m) {
return _mm256_mmask_i64gather_epi64(__m256i(), m.to_register(), i.to_register(), data_ptr,
sizeof(_Tp)); },
702 [=](zmm_register<std::int32_t>
auto i,
auto m) {
return _mm512_mask_i32gather_epi32(__m512i(), m.to_register(), i.to_register(), data_ptr,
sizeof(_Tp)); },
703 [=](zmm_register<std::int64_t>
auto i,
auto m) {
return _mm512_mask_i64gather_epi64(__m512i(), m.to_register(), i.to_register(), data_ptr,
sizeof(_Tp)); },
705#elif defined(__AVX2__)
710 [=](xmm_register<std::int32_t>
auto i,
auto m)
711 {
return _mm_mask_i32gather_epi32(__m128i(),
reinterpret_cast<const int*
>(data_ptr), i.to_register(),
grow<4>(m).to_register(),
sizeof(_Tp)); },
712 [=](ymm_register<std::int32_t>
auto i,
auto m)
713 {
return _mm256_mask_i32gather_epi32(__m256i(),
reinterpret_cast<const int*
>(data_ptr), i.to_register(),
grow<8>(m).to_register(),
sizeof(_Tp)); },
716 [=](xmm_register<std::int64_t>
auto i,
auto m)
717 {
return _mm_mask_i64gather_epi64(__m128i(),
reinterpret_cast<const long long*
>(data_ptr), i.to_register(),
grow<2>(m).to_register(),
sizeof(_Tp)); },
718 [=](ymm_register<std::int64_t>
auto i,
auto m)
719 {
return _mm256_mask_i64gather_epi64(__m256i(),
reinterpret_cast<const long long*
>(data_ptr), i.to_register(),
grow<4>(m).to_register(),
sizeof(_Tp)); },
723 [=](
auto unhandled,
auto) {
static_assert(
dependent_false<
decltype(unhandled)>); }
728 constexpr auto idxNumBytes = std::max({
sizeof(std::int32_t),
sizeof(_Tp),
sizeof(_Idx)});
729 using _IW = std::make_signed_t<container_for_num_bytes<idxNumBytes>>;
730 const auto rmax = std::min<_IW>(std::ranges::size(r), std::numeric_limits<_Idx>::max());
732 auto wrapper = [=]<
typename _VecIdx>(_VecIdx idx,
auto m) {
733 using RetypedIndex = rebind_t<_IW, _VecIdx>;
734 auto ri = RetypedIndex(idx);
740 auto rm =
typename RetypedIndex::mask_type(m);
744 rm = rm && (ri >= _IW(0) && ri < rmax);
747 auto t = RetypedIndex(impl(ri, rm));
755 checkStaticMemoryBounds(
"gather", indexes, rmax,
mask);
766 constexpr auto m =
typename _Idx::mask_type(
true);
770#if defined(__AVX512F__)
772inline auto scatterImpl =
777 const std::uint64_t nm = justMaskBits & m;
779 constexpr bool isTp32 = (
sizeof(_Tp) == 4);
781 if constexpr (xmm_register<basic_vec<_Tp, _VT>>)
783 if constexpr (isTp32) _mm_mask_i32scatter_epi32(ptr, nm, idx.to_register(), v.to_register(), 4);
784 else _mm_mask_i32scatter_epi64(ptr, nm, idx.to_register(), v.to_register(), 8);
786 else if constexpr (ymm_register<basic_vec<_Tp, _VT>>)
788 if constexpr (isTp32) _mm256_mask_i32scatter_epi32(ptr, nm, idx.to_register(), v.to_register(), 4);
789 else _mm256_mask_i32scatter_epi64(ptr, nm, idx.to_register(), v.to_register(), 8);
793 if constexpr (isTp32) _mm512_mask_i32scatter_epi32(ptr, nm, idx.to_register(), v.to_register(), 4);
794 else _mm512_mask_i32scatter_epi64(ptr, nm, idx.to_register(), v.to_register(), 8);
799template<
typename _Range,
typename _Tp,
typename _TpAbi,
typename _Idx,
typename _IdxAbi,
typename... _Flags>
800requires (
sizeof(_Tp) == 4 ||
sizeof(_Tp) == 8)
803scatter_to(x86_avx512_tag,
const basic_vec<_Tp, _TpAbi>& values, _Range&& r,
804 const basic_vec<_Idx, _IdxAbi>& indexes, flags<_Flags...> flags)
806 auto ptr = std::ranges::data(r);
807 const auto rmax = std::int32_t(std::min(std::ranges::size(r), std::size_t(std::numeric_limits<_Idx>::max())));
811 auto impl = [=](
auto v,
auto i) {
812 if constexpr (contains_flag<detail::flag_unchecked>(flags))
813 scatterImpl(v, i, ptr, ~0ULL);
815 scatterImpl(v, i, ptr, (i >= 0 && i < rmax).to_register());
818 if constexpr (contains_flag<flag_unchecked>(flags))
819 checkStaticMemoryBounds(
"scatter", indexes, rmax);
823 auto indexAs32 = vec<std::int32_t, basic_vec<_Idx, _IdxAbi>::size>(indexes);
827template<
typename _Range,
typename _Tp,
typename _TpAbi,
typename _Idx,
typename _IdxAbi,
typename... _Flags>
828requires (
sizeof(_Tp) == 4 ||
sizeof(_Tp) == 8)
831scatter_to(x86_avx512_tag,
const basic_vec<_Tp, _TpAbi>& values, _Range&& r,
832 const typename basic_vec<_Idx, _IdxAbi>::mask_type& mask,
833 const basic_vec<_Idx, _IdxAbi>& indexes, flags<_Flags...> flags)
835 auto ptr = std::ranges::data(r);
838 const auto rmax = _Idx(std::min(std::ranges::size(r), std::size_t(std::numeric_limits<_Idx>::max())));
839 auto rangeCheckedMask =
840 contains_flag<detail::flag_unchecked>(flags) ?
mask : (indexes >= _Idx(0) && indexes < rmax) && mask;
842 if constexpr (contains_flag<flag_unchecked>(flags))
843 checkStaticMemoryBounds(
"scatter", indexes, rmax, mask);
846 auto impl = [=](
auto v,
auto i,
auto m) { scatterImpl(v, i, ptr, m.to_ullong()); };
850 auto indexAs32 = vec<std::int32_t, basic_vec<_Idx, _IdxAbi>::size>(indexes);
856template<
typename _Tp,
typename _Abi, mask_operator _Op>
859 using binop_type = std::remove_cvref_t<_Op>;
860 using value_type = std::remove_cvref_t<_Tp>;
863 constexpr bool is_int = std::is_integral_v<value_type> || std::is_enum_v<value_type>;
864 constexpr auto EQ_OP = is_int ? _MM_CMPINT_EQ : _CMP_EQ_OQ;
865 constexpr auto NEQ_OP = is_int ? _MM_CMPINT_NE : _CMP_NEQ_OQ;
866 constexpr auto LT_OP = is_int ? _MM_CMPINT_LT : _CMP_LT_OQ;
867 constexpr auto LE_OP = is_int ? _MM_CMPINT_LE : _CMP_LE_OQ;
868 constexpr auto GT_OP = is_int ? _MM_CMPINT_NLE : _CMP_GT_OQ;
869 constexpr auto GE_OP = is_int ? _MM_CMPINT_NLT : _CMP_GE_OQ;
871 constexpr auto op = (std::is_same_v<binop_type, std::equal_to<>>) ? EQ_OP :
872 (std::is_same_v<binop_type, std::not_equal_to<>>) ? NEQ_OP :
873 (std::is_same_v<binop_type, std::less<>>) ? LT_OP :
874 (std::is_same_v<binop_type, std::less_equal<>>) ? LE_OP :
875 (std::is_same_v<binop_type, std::greater<>>) ? GT_OP :
878 static_assert(op != GE_OP || std::is_same_v<binop_type, std::greater_equal<>>,
"Unknown binary operation");
880 auto impl = [=](
auto lhs,
auto rhs) {
882 constexpr bool isSigned = std::is_signed_v<value_type>;
884 auto lhsr = lhs.to_register();
885 auto rhsr = rhs.to_register();
887 if constexpr (is_int)
889 if constexpr (
sizeof(value_type) == 1) {
890 if constexpr (
sizeof(lhs) <= 16)
return isSigned ? _mm_cmp_epi8_mask(lhsr, rhsr, op) : _mm_cmp_epu8_mask(lhsr, rhsr, op);
891 else if constexpr (
sizeof(lhs) <= 32)
return isSigned ? _mm256_cmp_epi8_mask(lhsr, rhsr, op) : _mm256_cmp_epu8_mask(lhsr, rhsr, op);
892 else if constexpr (
sizeof(lhs) <= 64)
return isSigned ? _mm512_cmp_epi8_mask(lhsr, rhsr, op) : _mm512_cmp_epu8_mask(lhsr, rhsr, op);
894 else if constexpr (
sizeof(value_type) == 2) {
895 if constexpr (
sizeof(lhs) <= 16)
return isSigned ? _mm_cmp_epi16_mask(lhsr, rhsr, op) : _mm_cmp_epu16_mask(lhsr, rhsr, op);
896 else if constexpr (
sizeof(lhs) <= 32)
return isSigned ? _mm256_cmp_epi16_mask(lhsr, rhsr, op) : _mm256_cmp_epu16_mask(lhsr, rhsr, op);
897 else if constexpr (
sizeof(lhs) <= 64)
return isSigned ? _mm512_cmp_epi16_mask(lhsr, rhsr, op) : _mm512_cmp_epu16_mask(lhsr, rhsr, op);
899 else if constexpr (
sizeof(value_type) == 4) {
900 if constexpr (
sizeof(lhs) <= 16)
return isSigned ? _mm_cmp_epi32_mask(lhsr, rhsr, op) : _mm_cmp_epu32_mask(lhsr, rhsr, op);
901 else if constexpr (
sizeof(lhs) <= 32)
return isSigned ? _mm256_cmp_epi32_mask(lhsr, rhsr, op) : _mm256_cmp_epu32_mask(lhsr, rhsr, op);
902 else if constexpr (
sizeof(lhs) <= 64)
return isSigned ? _mm512_cmp_epi32_mask(lhsr, rhsr, op) : _mm512_cmp_epu32_mask(lhsr, rhsr, op);
904 if constexpr (
sizeof(value_type) == 8) {
905 if constexpr (
sizeof(lhs) <= 16)
return isSigned ? _mm_cmp_epi64_mask(lhsr, rhsr, op) : _mm_cmp_epu64_mask(lhsr, rhsr, op);
906 else if constexpr (
sizeof(lhs) <= 32)
return isSigned ? _mm256_cmp_epi64_mask(lhsr, rhsr, op) : _mm256_cmp_epu64_mask(lhsr, rhsr, op);
907 else if constexpr (
sizeof(lhs) <= 64)
return isSigned ? _mm512_cmp_epi64_mask(lhsr, rhsr, op) : _mm512_cmp_epu64_mask(lhsr, rhsr, op);
912 #if defined (__AVX512FP16__)
913 if constexpr (
sizeof(lhs) <= 16)
return _mm_cmp_ph_mask(lhsr, rhsr, op);
914 else if constexpr (
sizeof(lhs) <= 32)
return _mm256_cmp_ph_mask(lhsr, rhsr, op);
915 else if constexpr (
sizeof(lhs) <= 64)
return _mm512_cmp_ph_mask(lhsr, rhsr, op);
917 static_assert(
dependent_false<
decltype(lhs)>,
"Emulated support for _Float16 handled below");
920 else if constexpr (std::is_same_v<value_type, float>)
922 if constexpr (
sizeof(lhs) <= 16)
return _mm_cmp_ps_mask(lhsr, rhsr, op);
923 else if constexpr (
sizeof(lhs) <= 32)
return _mm256_cmp_ps_mask(lhsr, rhsr, op);
924 else if constexpr (
sizeof(lhs) <= 64)
return _mm512_cmp_ps_mask(lhsr, rhsr, op);
926 else if constexpr (std::is_same_v<value_type, double>)
929 if constexpr (
sizeof(lhs) <= 16)
return _mm_cmp_pd_mask(lhsr, rhsr, op);
930 else if constexpr (
sizeof(lhs) <= 32)
return _mm256_cmp_pd_mask(lhsr, rhsr, op);
931 else if constexpr (
sizeof(lhs) <= 64)
return _mm512_cmp_pd_mask(lhsr, rhsr, op);
952 auto wrapper = [=](
auto l,
auto r) {
return mask<_Tp,
decltype(l.size)::value>(impl(l, r)); };
972template<
typename _Tp,
typename _Abi, mask_operator _Op>
973requires (detail::is_floating_point_v<_Tp>)
976 using binop_type = std::remove_cvref_t<_Op>;
977 constexpr bool isEqualOp = std::is_same_v<binop_type, std::equal_to<>>;
978 constexpr bool isNotEqualOp = std::is_same_v<binop_type, std::not_equal_to<>>;
979 static_assert(isEqualOp || isNotEqualOp,
"Complex values only allow equality comparisons");
982 const auto reduceBitPair = [](
auto bits) {
983 if constexpr (isEqualOp)
984 return _mm256_cmp_epi16_mask(_mm256_movm_epi8(bits), _mm256_set1_epi16(-1), _MM_CMPINT_EQ);
986 return _mm256_cmp_epi16_mask(_mm256_movm_epi8(bits), __m256i(), _MM_CMPINT_NE);
989 auto impl = [=]<
typename _Vec>(_Vec x, _Vec y)
992 auto rb = reduceBitPair(bits.to_builtin());
993 return typename _Vec::mask_type(rb);
1022template<
int _Op,
typename _Tp,
typename _Abi>
1026#if defined (__AVX512FP16__)
1030 [=]<xmm_register<_Float16> _Vec>(_Vec v) -> __mmask8 {
1031 return __builtin_ia32_fpclassph128_mask(v.to_register(), _Op, (__mmask64(1) << _Vec::size) - 1);
1033 [=]<ymm_register<_Float16> _Vec>(_Vec v) -> __mmask16 {
1034 return __builtin_ia32_fpclassph256_mask(v.to_register(), _Op, (__mmask64(1) << _Vec::size) - 1);
1036 [=]<zmm_register<_Float16> _Vec>(_Vec v) -> __mmask32 {
1037 return __builtin_ia32_fpclassph512_mask(v.to_register(), _Op, (__mmask64(1) << _Vec::size) - 1);
1040 [=](xmm_register<float>
auto v) {
return _mm_fpclass_ps_mask(v.to_register(), _Op); },
1041 [=](ymm_register<float>
auto v) {
return _mm256_fpclass_ps_mask(v.to_register(), _Op); },
1043 [=](xmm_register<double>
auto v) {
return _mm_fpclass_pd_mask(v.to_register(), _Op); },
1044 [=](ymm_register<double>
auto v) {
return _mm256_fpclass_pd_mask(v.to_register(), _Op); },
1046 [=](zmm_register<float>
auto v) {
return _mm512_fpclass_ps_mask(v.to_register(), _Op); },
1047 [=](zmm_register<double>
auto v) {
return _mm512_fpclass_pd_mask(v.to_register(), _Op); },
1049 [=](
auto unhandled) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics");
return 0; }
1053 auto wrapper = [=](
auto _vec) {
1054 auto dwt = impl(_vec);
1055 return mask<_Tp,
decltype(_vec.size)::value>(dwt);
1064 detail::is_floating_point_v<_Tp> && (hasAvxFp16 || !
is_fp16_v<_Tp>);
1067template<
typename _Tp,
typename _Abi>
1075template<
typename _Tp,
typename _Abi>
1076requires (has_native_support_for_classify<_Tp>)
1083template<
typename _Tp,
typename _Abi>
1084requires (has_native_support_for_classify<_Tp>)
1088 constexpr int invalidClasses = X86_FP_CLASSIFY::QUIET_NAN | X86_FP_CLASSIFY::SIGNAL_NAN | X86_FP_CLASSIFY::POS_INF | X86_FP_CLASSIFY::NEG_INF;
1092template<
typename _Tp,
typename _Abi>
1093requires (has_native_support_for_classify<_Tp>)
1097 constexpr int invalidClasses =
1098 X86_FP_CLASSIFY::QUIET_NAN | X86_FP_CLASSIFY::SIGNAL_NAN | X86_FP_CLASSIFY::POS_INF | X86_FP_CLASSIFY::NEG_INF |
1099 X86_FP_CLASSIFY::POS_ZERO | X86_FP_CLASSIFY::NEG_ZERO | X86_FP_CLASSIFY::DENORM;
1103#if defined(__AVX512F__)
1104template<std::
integral _Up, std::
integral _Tp,
typename _Abi>
1105requires (std::signed_integral<_Up> == std::signed_integral<_Tp> &&
1106 sizeof(_Up) <
sizeof(_Tp))
1110 target_overloads impl {
1112 [=](xmm_register<std::int16_t>
auto in, std::int8_t) {
return _mm_cvtsepi16_epi8(in.to_register()); },
1113 [=](ymm_register<std::int16_t>
auto in, std::int8_t) {
return _mm256_cvtsepi16_epi8(in.to_register()); },
1114 [=](zmm_register<std::int16_t>
auto in, std::int8_t) {
return _mm512_cvtsepi16_epi8(in.to_register()); },
1117 [=](xmm_register<std::uint16_t>
auto in, std::uint8_t) {
return _mm_cvtusepi16_epi8(in.to_register()); },
1118 [=](ymm_register<std::uint16_t>
auto in, std::uint8_t) {
return _mm256_cvtusepi16_epi8(in.to_register()); },
1119 [=](zmm_register<std::uint16_t>
auto in, std::uint8_t) {
return _mm512_cvtusepi16_epi8(in.to_register()); },
1122 [=](xmm_register<std::int32_t>
auto in, std::int8_t) {
return _mm_cvtsepi32_epi8(in.to_register()); },
1123 [=](ymm_register<std::int32_t>
auto in, std::int8_t) {
return _mm256_cvtsepi32_epi8(in.to_register()); },
1124 [=](zmm_register<std::int32_t>
auto in, std::int8_t) {
return _mm512_cvtsepi32_epi8(in.to_register()); },
1127 [=](xmm_register<std::uint32_t>
auto in, std::uint8_t) {
return _mm_cvtusepi32_epi8(in.to_register()); },
1128 [=](ymm_register<std::uint32_t>
auto in, std::uint8_t) {
return _mm256_cvtusepi32_epi8(in.to_register()); },
1129 [=](zmm_register<std::uint32_t>
auto in, std::uint8_t) {
return _mm512_cvtusepi32_epi8(in.to_register()); },
1132 [=](xmm_register<std::int32_t>
auto in, std::int16_t) {
return _mm_cvtsepi32_epi16(in.to_register()); },
1133 [=](ymm_register<std::int32_t>
auto in, std::int16_t) {
return _mm256_cvtsepi32_epi16(in.to_register()); },
1134 [=](zmm_register<std::int32_t>
auto in, std::int16_t) {
return _mm512_cvtsepi32_epi16(in.to_register()); },
1137 [=](xmm_register<std::uint32_t>
auto in, std::uint16_t) {
return _mm_cvtusepi32_epi16(in.to_register()); },
1138 [=](ymm_register<std::uint32_t>
auto in, std::uint16_t) {
return _mm256_cvtusepi32_epi16(in.to_register()); },
1139 [=](zmm_register<std::uint32_t>
auto in, std::uint16_t) {
return _mm512_cvtusepi32_epi16(in.to_register()); },
1142 [=](xmm_register<std::int64_t>
auto in, std::int8_t) {
return _mm_cvtsepi64_epi8(in.to_register()); },
1143 [=](ymm_register<std::int64_t>
auto in, std::int8_t) {
return _mm256_cvtsepi64_epi8(in.to_register()); },
1144 [=](zmm_register<std::int64_t>
auto in, std::int8_t) {
return _mm512_cvtsepi64_epi8(in.to_register()); },
1147 [=](xmm_register<std::uint64_t>
auto in, std::uint8_t) {
return _mm_cvtusepi64_epi8(in.to_register()); },
1148 [=](ymm_register<std::uint64_t>
auto in, std::uint8_t) {
return _mm256_cvtusepi64_epi8(in.to_register()); },
1149 [=](zmm_register<std::uint64_t>
auto in, std::uint8_t) {
return _mm512_cvtusepi64_epi8(in.to_register()); },
1152 [=](xmm_register<std::int64_t>
auto in, std::int16_t) {
return _mm_cvtsepi64_epi16(in.to_register()); },
1153 [=](ymm_register<std::int64_t>
auto in, std::int16_t) {
return _mm256_cvtsepi64_epi16(in.to_register()); },
1154 [=](zmm_register<std::int64_t>
auto in, std::int16_t) {
return _mm512_cvtsepi64_epi16(in.to_register()); },
1157 [=](xmm_register<std::uint64_t>
auto in, std::uint16_t) {
return _mm_cvtusepi64_epi16(in.to_register()); },
1158 [=](ymm_register<std::uint64_t>
auto in, std::uint16_t) {
return _mm256_cvtusepi64_epi16(in.to_register()); },
1159 [=](zmm_register<std::uint64_t>
auto in, std::uint16_t) {
return _mm512_cvtusepi64_epi16(in.to_register()); },
1162 [=](xmm_register<std::int64_t>
auto in, std::int32_t) {
return _mm_cvtsepi64_epi32(in.to_register()); },
1163 [=](ymm_register<std::int64_t>
auto in, std::int32_t) {
return _mm256_cvtsepi64_epi32(in.to_register()); },
1164 [=](zmm_register<std::int64_t>
auto in, std::int32_t) {
return _mm512_cvtsepi64_epi32(in.to_register()); },
1167 [=](xmm_register<std::uint64_t>
auto in, std::uint32_t) {
return _mm_cvtusepi64_epi32(in.to_register()); },
1168 [=](ymm_register<std::uint64_t>
auto in, std::uint32_t) {
return _mm256_cvtusepi64_epi32(in.to_register()); },
1169 [=](zmm_register<std::uint64_t>
auto in, std::uint32_t) {
return _mm512_cvtusepi64_epi32(in.to_register()); },
1171 [=](
auto unhandled,
auto out)
1172 {
static_assert(
dependent_false<std::pair<
decltype(unhandled),
decltype(out)>>,
"No target overload");
return out;}
1175 auto wrapper = [=]<
typename _VT,
typename _VAbi>(basic_vec<_VT, _VAbi> v) {
1176 auto r = impl(v, _Up());
1177 return vec<_Up, basic_vec<_VT, _VAbi>::size>(r);
1185inline __m128i
shuffle_epi8(__m128i values, __m128i indexes) {
return _mm_shuffle_epi8(values, indexes); }
1188#if defined (__AVX2__)
1189 return _mm256_shuffle_epi8(values, indexes);
1194 const auto lowerShuf = _mm_shuffle_epi8(_mm256_castsi256_si128(values), _mm256_castsi256_si128(indexes));
1195 const auto upperShuf = _mm_shuffle_epi8(_mm256_extractf128_si256(values, 1), _mm256_extractf128_si256(indexes, 1));
1196 return _mm256_insertf128_si256(_mm256_castsi128_si256(lowerShuf), upperShuf, 1);
1200#if defined(__AVX512F__)
1201inline __m512i shuffle_epi8(__m512i values, __m512i indexes) {
return _mm512_shuffle_epi8(values, indexes); }
1212template<
typename _Gen,
typename _Abi>
1213constexpr basic_vec<std::uint8_t, _Abi>
1231template<
typename _Tp,
typename _Abi>
1234 const auto bitsInByte = []<
typename _Vec>(_Vec piece) -> _Vec {
1239 constexpr auto gen = [](
auto idx) -> std::uint8_t {
return std::popcount(std::uint8_t(idx & 0xF)); };
1250 if constexpr (
sizeof(_Tp) == 1)
return pc;
1253 if constexpr (
sizeof(_Tp) == 8) pc += (pc >> 32);
1254 if constexpr (
sizeof(_Tp) >= 4) pc += (pc >> 16);
1255 if constexpr (
sizeof(_Tp) >= 2) pc += (pc >> 8);
1258 return pc & cw<0xFF>;
1261#if defined(__AVX512VPOPCNTDQ__)
1263template<
typename _Tp,
typename _Abi>
1265 target_overloads impl {
1266 [=](xmm_register<std::uint8_t>
auto v) {
return _mm_popcnt_epi8(v.
to_register()); },
1267 [=](xmm_register<std::uint16_t>
auto v) {
return _mm_popcnt_epi16(v.
to_register()); },
1268 [=](xmm_register<std::uint32_t>
auto v) {
return _mm_popcnt_epi32(v.
to_register()); },
1269 [=](xmm_register<std::uint64_t>
auto v) {
return _mm_popcnt_epi64(v.
to_register()); },
1271 [=](ymm_register<std::uint8_t>
auto v) {
return _mm256_popcnt_epi8(v.
to_register()); },
1272 [=](ymm_register<std::uint16_t>
auto v) {
return _mm256_popcnt_epi16(v.
to_register()); },
1273 [=](ymm_register<std::uint32_t>
auto v) {
return _mm256_popcnt_epi32(v.
to_register()); },
1274 [=](ymm_register<std::uint64_t>
auto v) {
return _mm256_popcnt_epi64(v.
to_register()); },
1276 [=](zmm_register<std::uint8_t>
auto v) {
return _mm512_popcnt_epi8(v.
to_register()); },
1277 [=](zmm_register<std::uint16_t>
auto v) {
return _mm512_popcnt_epi16(v.
to_register()); },
1278 [=](zmm_register<std::uint32_t>
auto v) {
return _mm512_popcnt_epi32(v.
to_register()); },
1279 [=](zmm_register<std::uint64_t>
auto v) {
return _mm512_popcnt_epi64(v.
to_register()); },
1281 [=](
auto unhandled) ->
decltype(unhandled) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics"); }
1285 return chunked_invoke([=](
auto v) {
return decltype(v)(impl(v)); }, x);
1293template<
typename _Tp,
typename _Abi>
1302template<std::
unsigned_
integral _Tp,
typename _Abi>
1306 const auto clzByte = []<
typename _Vec>(_Vec piece) -> _Vec
1319 shuffle_epi8_lut([](
auto idx) -> std::uint8_t {
return std::countl_zero(std::uint8_t(idx & 0xF)); }, bytes & std::uint8_t(0xF));
1320 const auto highClz =
1321 shuffle_epi8_lut([](
auto idx) -> std::uint8_t {
return std::countl_zero(std::uint8_t(idx & 0xF)) - 4; }, bytes >> std::uint8_t(4));
1327 constexpr auto numElementBytes =
sizeof(_Tp);
1329 if constexpr (numElementBytes == 1)
1331 else if constexpr (numElementBytes == 2)
1337 return select(x > cw<0xFF>, clz8 >> 8, (clz8 & cw<0xFF>) + cw<8>);
1339 else if constexpr (numElementBytes == 4)
1358 else if constexpr (numElementBytes == 8)
1369 return select(x > 0xFFFFFFFF, tas64 >> 32, (tas64 & 0xFFFFFFFF) + cw<32>);
1384#if defined(__AVX512F__)
1385template<
typename _Tp,
typename _Abi>
1386requires (
sizeof(_Tp) == 4 ||
sizeof(_Tp) == 8)
1388 target_overloads impl {
1389 [=](xmm_register<std::uint32_t>
auto v) {
return _mm_lzcnt_epi32(v.
to_register()); },
1390 [=](xmm_register<std::uint64_t>
auto v) {
return _mm_lzcnt_epi64(v.
to_register()); },
1392 [=](ymm_register<std::uint32_t>
auto v) {
return _mm256_lzcnt_epi32(v.
to_register()); },
1393 [=](ymm_register<std::uint64_t>
auto v) {
return _mm256_lzcnt_epi64(v.
to_register()); },
1395 [=](zmm_register<std::uint32_t>
auto v) {
return _mm512_lzcnt_epi32(v.
to_register()); },
1396 [=](zmm_register<std::uint64_t>
auto v) {
return _mm512_lzcnt_epi64(v.
to_register()); },
1398 [=](
auto unhandled) ->
decltype(unhandled) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics"); }
1401 return chunked_invoke([=](
auto v) {
return decltype(v)(impl(v)); }, x);
1411template<
typename _Tp,
typename _Abi>
1412requires (
sizeof(_Tp) == 1)
1416 const auto byteBitReverser = []<
typename _Vec>(_Vec piece) -> _Vec {
1422 constexpr std::uint8_t _revbytes[16] = {0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe, 0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf};
1423 const auto lowRev =
shuffle_epi8_lut([](
auto idx) -> std::uint8_t {
return _revbytes[idx % 16] << 4; }, bytes & cw<0xF>);
1424 const auto highRev =
shuffle_epi8_lut([](
auto idx) -> std::uint8_t {
return _revbytes[idx % 16]; }, bytes >> 4);
1426 return _Vec(lowRev | highRev);
1436template<vec_
integral _Vp>
1437requires (
sizeof(
typename _Vp::value_type) == 1)
1439 {
return x86::gf2p8affine(x, std::uint64_t(0x8040201008040201)); }
1441template<
typename _Tp,
typename _Abi>
1442requires (detail::is_floating_point_v<_Tp>)
1446 [=](xmm_register<float>
auto x) {
return _mm_rcp_ps(x.to_register());},
1449 [=](ymm_register<float>
auto x) {
return _mm256_rcp_ps(x.to_register());},
1453 [=](
auto x) {
return _Tp(1) / x; }
1456 return chunked_invoke([=](
auto v) {
return decltype(v)(impl(v)); }, v);
1459#if defined(__AVX512F__)
1460template<std::
floating_po
int _Tp,
typename _Abi>
1463 target_overloads impl {
1464 [=](xmm_register<float>
auto x) {
return _mm_rcp14_ps(x.to_register());},
1465 [=](ymm_register<float>
auto x) {
return _mm256_rcp14_ps(x.to_register());},
1467 [=](xmm_register<double>
auto x) {
return _mm_rcp14_pd(x.to_register());},
1468 [=](ymm_register<double>
auto x) {
return _mm256_rcp14_pd(x.to_register());},
1470 [=](zmm_register<float>
auto x) {
return _mm512_rcp14_ps(x.to_register());},
1471 [=](zmm_register<double>
auto x) {
return _mm512_rcp14_pd(x.to_register());},
1474 return chunked_invoke([=](
auto v) {
return decltype(v)(impl(v)); }, v);
1478#if defined(__AVX512FP16__)
1479template<
typename _Abi>
1480constexpr basic_vec<_Float16, _Abi>
rcp(x86_avxspr_tag,
const basic_vec<_Float16, _Abi>& v)
1482 target_overloads impl {
1483 [=](xmm_register<_Float16>
auto x) {
return _mm_rcp_ph(x.to_register());},
1484 [=](ymm_register<_Float16>
auto x) {
return _mm256_rcp_ph(x.to_register());},
1485 [=](zmm_register<_Float16>
auto x) {
return _mm512_rcp_ph(x.to_register());},
1487 return chunked_invoke([=](
auto v) {
return decltype(v)(impl(v)); }, v);
1491template<
typename _Tp,
typename _Abi>
1492requires (detail::is_floating_point_v<_Tp>)
1496 [=](xmm_register<float>
auto x) {
return _mm_rsqrt_ps(x.to_register());},
1497 [=](xmm_register<double>
auto x) {
return 1.0 / _mm_sqrt_pd(x.to_register());},
1500 [=](ymm_register<float>
auto x) {
return _mm256_rsqrt_ps(x.to_register());},
1501 [=](ymm_register<double>
auto x) {
return 1.0 / _mm256_sqrt_pd(x.to_register());},
1505 [=](
auto x) {
return _Tp(1) /
sqrt(x); }
1508 return chunked_invoke([=](
auto v) {
return decltype(v)(impl(v)); }, v);
1511#if defined(__AVX512F__)
1512template<std::
floating_po
int _Tp,
typename _Abi>
1515 target_overloads impl {
1516 [=](xmm_register<float>
auto x) {
return _mm_rsqrt14_ps(x.to_register());},
1517 [=](ymm_register<float>
auto x) {
return _mm256_rsqrt14_ps(x.to_register());},
1519 [=](xmm_register<double>
auto x) {
return _mm_rsqrt14_pd(x.to_register());},
1520 [=](ymm_register<double>
auto x) {
return _mm256_rsqrt14_pd(x.to_register());},
1522 [=](zmm_register<float>
auto x) {
return _mm512_rsqrt14_ps(x.to_register());},
1523 [=](zmm_register<double>
auto x) {
return _mm512_rsqrt14_pd(x.to_register());},
1525 return chunked_invoke([=](
auto v) {
return decltype(v)(impl(v)); }, v);
1529#if defined(__AVX512FP16__)
1530template<
typename _Abi>
1531constexpr basic_vec<_Float16, _Abi>
rsqrt(x86_avxspr_tag,
const basic_vec<_Float16, _Abi>& v)
1533 target_overloads impl {
1534 [=](xmm_register<_Float16>
auto x) {
return _mm_rsqrt_ph(x.to_register());},
1535 [=](ymm_register<_Float16>
auto x) {
return _mm256_rsqrt_ph(x.to_register());},
1536 [=](zmm_register<_Float16>
auto x) {
return _mm512_rsqrt_ph(x.to_register());},
1538 return chunked_invoke([=](
auto v) {
return decltype(v)(impl(v)); }, v);
1542template<
typename _Tp,
typename _Abi>
1543requires (detail::is_floating_point_v<_Tp>)
1547 [=](xmm_register<float>
auto x) {
return _mm_sqrt_ps(x.to_register());},
1548 [=](xmm_register<double>
auto x) {
return _mm_sqrt_pd(x.to_register());},
1551 [=](ymm_register<float>
auto x) {
return _mm256_sqrt_ps(x.to_register());},
1552 [=](ymm_register<double>
auto x) {
return _mm256_sqrt_pd(x.to_register());},
1555#if defined(__AVX512F__)
1556 [=](zmm_register<float>
auto x) {
return _mm512_sqrt_ps(x.to_register());},
1557 [=](zmm_register<double>
auto x) {
return _mm512_sqrt_pd(x.to_register());},
1560#if defined(__AVX512FP16__)
1561 [=](xmm_register<_Float16>
auto x) {
return _mm_sqrt_ph(x.to_register());},
1562 [=](ymm_register<_Float16>
auto x) {
return _mm256_sqrt_ph(x.to_register());},
1563 [=](zmm_register<_Float16>
auto x) {
return _mm512_sqrt_ph(x.to_register());},
1566 [=](
auto unhandled) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics");
return unhandled; }
1576 return chunked_invoke([=](
auto v) {
return decltype(v)(impl(v)); }, v);
1586template<std::
integral _Tp,
typename _Abi>
1587requires (
sizeof(_Tp) <= 2)
1593 [=](xmm_register<std::uint8_t>
auto x,
auto y) {
return _mm_adds_epu8(x.to_register(), y.to_register()); },
1594 [=](xmm_register<std::int8_t>
auto x,
auto y) {
return _mm_adds_epi8(x.to_register(), y.to_register()); },
1595 [=](xmm_register<std::uint16_t>
auto x,
auto y) {
return _mm_adds_epu16(x.to_register(), y.to_register()); },
1596 [=](xmm_register<std::int16_t>
auto x,
auto y) {
return _mm_adds_epi16(x.to_register(), y.to_register()); },
1599#if defined(__AVX2__)
1600 [=](ymm_register<std::uint8_t>
auto x,
auto y) {
return _mm256_adds_epu8(x.to_register(), y.to_register()); },
1601 [=](ymm_register<std::int8_t>
auto x,
auto y) {
return _mm256_adds_epi8(x.to_register(), y.to_register()); },
1602 [=](ymm_register<std::uint16_t>
auto x,
auto y) {
return _mm256_adds_epu16(x.to_register(), y.to_register()); },
1603 [=](ymm_register<std::int16_t>
auto x,
auto y) {
return _mm256_adds_epi16(x.to_register(), y.to_register()); },
1607#if defined(__AVX512F__)
1608 [=](zmm_register<std::uint8_t>
auto x,
auto y) {
return _mm512_adds_epu8(x.to_register(), y.to_register()); },
1609 [=](zmm_register<std::int8_t>
auto x,
auto y) {
return _mm512_adds_epi8(x.to_register(), y.to_register()); },
1610 [=](zmm_register<std::uint16_t>
auto x,
auto y) {
return _mm512_adds_epu16(x.to_register(), y.to_register()); },
1611 [=](zmm_register<std::int16_t>
auto x,
auto y) {
return _mm512_adds_epi16(x.to_register(), y.to_register()); },
1614 [=](
auto unhandled,
auto) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics");
return unhandled;}
1621 auto wrapper = [=](
auto a,
auto b) {
return decltype(a)(impl(a, b)); };
1632template<std::
integral _Tp,
typename _Abi>
1633requires (
sizeof(_Tp) <= 2)
1639 [=](xmm_register<std::uint8_t>
auto x,
auto y) {
return _mm_subs_epu8(x.to_register(), y.to_register()); },
1640 [=](xmm_register<std::int8_t>
auto x,
auto y) {
return _mm_subs_epi8(x.to_register(), y.to_register()); },
1641 [=](xmm_register<std::uint16_t>
auto x,
auto y) {
return _mm_subs_epu16(x.to_register(), y.to_register()); },
1642 [=](xmm_register<std::int16_t>
auto x,
auto y) {
return _mm_subs_epi16(x.to_register(), y.to_register()); },
1645#if defined(__AVX2__)
1646 [=](ymm_register<std::uint8_t>
auto x,
auto y) {
return _mm256_subs_epu8(x.to_register(), y.to_register()); },
1647 [=](ymm_register<std::int8_t>
auto x,
auto y) {
return _mm256_subs_epi8(x.to_register(), y.to_register()); },
1648 [=](ymm_register<std::uint16_t>
auto x,
auto y) {
return _mm256_subs_epu16(x.to_register(), y.to_register()); },
1649 [=](ymm_register<std::int16_t>
auto x,
auto y) {
return _mm256_subs_epi16(x.to_register(), y.to_register()); },
1653#if defined(__AVX512F__)
1654 [=](zmm_register<std::uint8_t>
auto x,
auto y) {
return _mm512_subs_epu8(x.to_register(), y.to_register()); },
1655 [=](zmm_register<std::int8_t>
auto x,
auto y) {
return _mm512_subs_epi8(x.to_register(), y.to_register()); },
1656 [=](zmm_register<std::uint16_t>
auto x,
auto y) {
return _mm512_subs_epu16(x.to_register(), y.to_register()); },
1657 [=](zmm_register<std::int16_t>
auto x,
auto y) {
return _mm512_subs_epi16(x.to_register(), y.to_register()); },
1660 [=](
auto unhandled,
auto) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics");
return unhandled;}
1667 auto wrapper = [=](
auto a,
auto b) {
return decltype(a)(impl(a, b)); };
1671#if defined(__AVX512VBMI2__)
1672template<vec_
unsigned_
integral _Vp, vec_
integral _Sp>
1673 requires (
sizeof(_Vp::value_type) >= 2)
1674constexpr _Vp fsr(x86_avxsnc_tag,
const _Vp& high,
const _Vp& low,
const _Sp& shift)
1678 target_overloads impl {
1680 [=]<xmm_register<std::uint16_t> _Vec>(_Vec x, _Vec y, _Vec s)
1681 {
return _mm_shrdv_epi16(x.to_register(), y.to_register(), s.to_register()); },
1682 [=]<ymm_register<std::uint16_t> _Vec>(_Vec x, _Vec y, _Vec s)
1683 {
return _mm256_shrdv_epi16(x.to_register(), y.to_register(), s.to_register()); },
1684 [=]<zmm_register<std::uint16_t> _Vec>(_Vec x, _Vec y, _Vec s)
1685 {
return _mm512_shrdv_epi16(x.to_register(), y.to_register(), s.to_register()); },
1687 [=]<xmm_register<std::uint32_t> _Vec>(_Vec x, _Vec y, _Vec s)
1688 {
return _mm_shrdv_epi32(x.to_register(), y.to_register(), s.to_register()); },
1689 [=]<ymm_register<std::uint32_t> _Vec>(_Vec x, _Vec y, _Vec s)
1690 {
return _mm256_shrdv_epi32(x.to_register(), y.to_register(), s.to_register()); },
1691 [=]<zmm_register<std::uint32_t> _Vec>(_Vec x, _Vec y, _Vec s)
1692 {
return _mm512_shrdv_epi32(x.to_register(), y.to_register(), s.to_register()); },
1694 [=]<xmm_register<std::uint64_t> _Vec>(_Vec x, _Vec y, _Vec s)
1695 {
return _mm_shrdv_epi64(x.to_register(), y.to_register(), s.to_register()); },
1696 [=]<ymm_register<std::uint64_t> _Vec>(_Vec x, _Vec y, _Vec s)
1697 {
return _mm256_shrdv_epi64(x.to_register(), y.to_register(), s.to_register()); },
1698 [=]<zmm_register<std::uint64_t> _Vec>(_Vec x, _Vec y, _Vec s)
1699 {
return _mm512_shrdv_epi64(x.to_register(), y.to_register(), s.to_register()); },
1701 [=](
auto unhandled,
auto,
auto) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics");
return unhandled; }
1704 auto wrapper = [=]<
typename _Vec>(_Vec x, _Vec y, _Vec s) {
return _Vec(impl(x, y, s)); };
1705 return std::bit_cast<_Vp>(
chunked_invoke(wrapper, low, high, shift));
1708template<vec_
unsigned_
integral _Vp, vec_
integral _Sp>
1709 requires (
sizeof(_Vp::value_type) >= 2)
1710constexpr _Vp fsl(x86_avxsnc_tag,
const _Vp& high,
const _Vp& low,
const _Sp& shift)
1712 using namespace detail;
1714 target_overloads impl {
1715 [=]<xmm_register<std::uint16_t> _Vec>(_Vec x, _Vec y, _Vec s)
1716 {
return _mm_shldv_epi16(x.to_register(), y.to_register(), s.to_register()); },
1717 [=]<ymm_register<std::uint16_t> _Vec>(_Vec x, _Vec y, _Vec s)
1718 {
return _mm256_shldv_epi16(x.to_register(), y.to_register(), s.to_register()); },
1719 [=]<zmm_register<std::uint16_t> _Vec>(_Vec x, _Vec y, _Vec s)
1720 {
return _mm512_shldv_epi16(x.to_register(), y.to_register(), s.to_register()); },
1722 [=]<xmm_register<std::uint32_t> _Vec>(_Vec x, _Vec y, _Vec s)
1723 {
return _mm_shldv_epi32(x.to_register(), y.to_register(), s.to_register()); },
1724 [=]<ymm_register<std::uint32_t> _Vec>(_Vec x, _Vec y, _Vec s)
1725 {
return _mm256_shldv_epi32(x.to_register(), y.to_register(), s.to_register()); },
1726 [=]<zmm_register<std::uint32_t> _Vec>(_Vec x, _Vec y, _Vec s)
1727 {
return _mm512_shldv_epi32(x.to_register(), y.to_register(), s.to_register()); },
1729 [=]<xmm_register<std::uint64_t> _Vec>(_Vec x, _Vec y, _Vec s)
1730 {
return _mm_shldv_epi64(x.to_register(), y.to_register(), s.to_register()); },
1731 [=]<ymm_register<std::uint64_t> _Vec>(_Vec x, _Vec y, _Vec s)
1732 {
return _mm256_shldv_epi64(x.to_register(), y.to_register(), s.to_register()); },
1733 [=]<zmm_register<std::uint64_t> _Vec>(_Vec x, _Vec y, _Vec s)
1734 {
return _mm512_shldv_epi64(x.to_register(), y.to_register(), s.to_register()); },
1736 [=](
auto unhandled,
auto,
auto) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics");
return unhandled; }
1739 auto wrapper = [=]<
typename _Vec>(_Vec x, _Vec y, _Vec s) {
return _Vec(impl(x, y, s)); };
1740 return std::bit_cast<_Vp>(
chunked_invoke(wrapper, low, high, shift));
A mask containing boolean elements.
Definition mask.hpp:121
static constexpr basic_mask __mask_from_count(simd_size_type _n)
Definition mask.hpp:223
Vector of elements sharing an element type and ABI.
Definition simd:161
static constexpr auto size
Definition simd:187
basic_mask< sizeof(_Tp), _Abi > mask_type
Definition simd:184
Check for contiguous range with a finite range.
Definition core.hpp:170
constexpr _Vp min(const _Vp &lhs, const _Vp &rhs) noexcept
Compute the smaller of each respective vec element from the two input arguments.
Definition std_algorithm.hpp:34
constexpr auto select(const _M &m, const _T &if_true, const _U &if_false) noexcept -> decltype(simd_select_impl(m, if_true, if_false))
Select between corresponding elements according to a mask.
Definition std_algorithm.hpp:88
constexpr auto countl_zero(const _Vp &v) noexcept
Count consecutive zero bits from the most significant bit of each element.
Definition std_bit.hpp:90
constexpr auto popcount(const _Vp &x) noexcept
Count the one bits in each element.
Definition std_bit.hpp:133
constexpr auto vec_as_container(const _Vp &v)
Generate a variable containing the iota sequence (i.e., 0, 1, 2, ... size-1) for the given simd::vec ...
Definition generic_impl.hpp:116
constexpr void unchecked_store(const basic_vec< _Tp, _Abi > &value, _Range &&r, flags< _Flags... > f={})
Unchecked store into a contiguous sized range.
Definition loadstore.hpp:332
constexpr void partial_store(const basic_vec< _Tp, _Abi > &value, _Range &&r, flags< _Flags... > f={})
Store the elements from a basic_vec value into a contiguous range, performing a suitable conversion i...
Definition loadstore.hpp:231
basic_mask< sizeof(_Tp), simd_fixed_size_abi< _Np > > mask
Definition core.hpp:383
constexpr detail::deduced_vec_t< _Vp > fma(const _Vp &lhs, const _Vp &rhs, const _Vp &acc)
Compute (lhs * rhs) + acc element-wise with a single rounding.
Definition std_math.hpp:111
constexpr detail::deduced_vec_t< _Vp > copysign(const _Vp &magnitude, const _Vp &sign)
Combine corresponding magnitudes and signs.
Definition std_math.hpp:64
basic_vec< _Tp, simd_fixed_size_abi< _Np > > vec
Definition core.hpp:379
constexpr auto permute(const _Vp &v, detail::index_generator_function auto &&fn)
Permute the elements in a vec value using a generator function to compute the indexing.
Definition permute.hpp:94
constexpr auto partial_gather_from(_Range &&r, const _Indexes &indexes, flags< _Flags... > f={})
Gather the contents of a contiguous range using the integral indexes supplied in another vec value.
Definition permute.hpp:258
constexpr auto chunked_invoke(_Fn fn, const _Arg0 &arg0, const _Args &... args)
Invoke a callable on corresponding vector or mask chunks.
Definition invoke.hpp:61
constexpr rebind_t< _Up, _Vp > saturating_cast(const _Vp &v) noexcept
Compute a basic_vec where every element is converted to the new type, or if the value can't be repres...
Definition std_numeric.hpp:77
constexpr auto grow(const vec_or_mask_type auto &in)
Grow the vec or mask input to be N elements.
Definition named_permute.hpp:243
constexpr detail::RebindCast< _Up > rebind_cast
Rebinding CPO.
Definition std_rebind.hpp:55
constexpr detail::deduced_vec_t< _Vp > fmaddsub(const _Vp &lhs, const _Vp &rhs, const _Vp &acc)
Compute lhs * rhs - acc at even indexes and lhs * rhs + acc at odd indexes. Each expression is comput...
Definition std_math.hpp:136
constexpr detail::deduced_vec_t< _Vp > rcp(const _Vp &v)
Compute the approximate reciprocal of each element.
Definition std_math.hpp:85
constexpr detail::deduced_vec_t< _Vp > fmsubadd(const _Vp &lhs, const _Vp &rhs, const _Vp &acc)
Compute lhs * rhs + acc at even indexes and lhs * rhs - acc at odd indexes. Each expression is comput...
Definition std_math.hpp:147
_Vc trunc(const _Vc &value)
Truncate the real and imaginary components of each element.
Definition std_complex.hpp:80
constexpr detail::deduced_vec_t< _Vp > rsqrt(const _Vp &v)
Compute the approximate reciprocal square root of each element.
Definition std_math.hpp:91
constexpr auto to_builtin() const
Return the builtin mask value directly.
Definition mask.hpp:275
constexpr traits::register_type to_register() const
Definition simd:344
constexpr auto simd_bit_cast(const _From &v) noexcept
A simd-generic version of simd_bit_cast which directly invokes std::bit_cast.
Definition core.hpp:585
Detect a scoped enum. This will be provided in C++23 onwards.
Definition compat.hpp:27
constexpr basic_vec< std::uint8_t, _Abi > shuffle_epi8_lut(_Gen, basic_vec< std::uint8_t, _Abi > indexes)
Definition x86_impl.hpp:1214
X86_FP_CLASSIFY
Constants which specify to the compiler intrinsic which FP classification operation to perform.
Definition x86_impl.hpp:1002
constexpr basic_vec< _Tp, _Abi > popcount(x86_sse_tag, const basic_vec< _Tp, _Abi > &x) noexcept
}@
Definition x86_impl.hpp:1232
constexpr detail::deduced_mask_t< _Vp > isnormal(generic_tag, const _Vp &v)
Definition std_math.hpp:263
constexpr basic_vec< _Tp, _Abi > clz(x86_sse_tag, const basic_vec< _Tp, _Abi > &x) noexcept
Count the number of leading zero bits in each element.
Definition x86_impl.hpp:1303
constexpr bool is_fp16_v
Definition utilities.hpp:90
constexpr basic_vec< _Tp, _Abi > rsqrt(x86_sse_tag, const basic_vec< _Tp, _Abi > &v)
Constants which specify to the compiler intrinsic which FP classification operation to perform.
Definition x86_impl.hpp:1493
constexpr T saturating_sub(T lhs, T rhs) noexcept
Subtract two integers, clamping an unrepresentable mathematical result.
Definition compat.hpp:90
__m128i shuffle_epi8(__m128i values, __m128i indexes)
Definition x86_impl.hpp:1185
constexpr void store(x86_tag, const _Vp &value_original, std::span< _Up, _Extent > to, _Flags flags)
Special case code for handling 4 and 8 byte masked stores on Intel AVX2. This is faster than using th...
Definition x86_impl.hpp:553
constexpr auto builtin_operator(generic_tag, _Vp value, _Op op)
General purpose unary operator. Passes through to the compiler's own builtin.
Definition operators.hpp:28
constexpr basic_vec< _Tp, _Abi > sqrt(x86_sse_tag, const basic_vec< _Tp, _Abi > &v)
Constants which specify to the compiler intrinsic which FP classification operation to perform.
Definition x86_impl.hpp:1544
constexpr detail::deduced_mask_t< _Vp > isinf(generic_tag tag, const _Vp &v)
Definition std_math.hpp:251
constexpr bool contains_flag(flags< _Flags... >)
Query whether the given flags contains the specified type.
Definition core.hpp:415
constexpr auto has_single_bit(x86_avxsnc_tag, const basic_vec< _Tp, _Abi > &x) noexcept
Query whether each element has exactly one bit set.
Definition x86_impl.hpp:1294
constexpr basic_vec< _Tp, _Abi > rcp(x86_sse_tag, const basic_vec< _Tp, _Abi > &v)
Constants which specify to the compiler intrinsic which FP classification operation to perform.
Definition x86_impl.hpp:1443
constexpr void store_masked(x86_tag, const _Vp &value_original, std::span< _Up, _Extent > to, const typename _Vp::mask_type &mask_original, _Flags flags)
Implement masked stores for Intel SSE and Intel AVX targets.
Definition x86_impl.hpp:427
auto x86_fp_classify_mask(const basic_vec< _Tp, _Abi > &value)
Classify the floating-point vec values using the fpclassify intrinsic.
Definition x86_impl.hpp:1023
constexpr deduced_mask_t< _Vp > isnan(generic_tag, const _Vp &v)
Definition std_math.hpp:228
constexpr bool dependent_false
Definition utilities.hpp:103
constexpr auto gather_from(x86_avx2_tag, const _Range &r, const typename basic_vec< _Idx, _IdxAbi >::mask_type &mask, const basic_vec< _Idx, _IdxAbi > &indexes, flags< _Flags... > flags)
Masked gather operation.
Definition x86_impl.hpp:685
constexpr T saturating_add(T lhs, T rhs) noexcept
Add two integers, clamping an unrepresentable mathematical result.
Definition compat.hpp:68
constexpr basic_vec< _Tp, _Abi > round_op(x86_tag, const basic_vec< _Tp, _Abi > &v)
Rounding operations (ceil, floor, etc.).
Definition x86_impl.hpp:353
constexpr basic_vec< _Tp, _Abi > bitreverse(x86_sse_tag, const basic_vec< _Tp, _Abi > &x) noexcept
Count the number of leading zero bits in each element.
Definition x86_impl.hpp:1413
constexpr bool has_native_support_for_classify
Detect whether native classification support is available for the given type. Emulated FP16 has no su...
Definition x86_impl.hpp:1063
constexpr detail::deduced_mask_t< _Vp > isfinite(generic_tag tag, const _Vp &v)
Definition std_math.hpp:257
Allow overloaded lambdas to be used for target-specific code. Described at https://www....
Definition utilities.hpp:48