11#include <xvec/detail/config.hpp>
16namespace _XVEC_NAMESPACE::simd {
27inline std::uint64_t dupBits (std::uint32_t bits) {
28 auto spreadBits = _pdep_u64(std::uint64_t(bits), 0x55555555);
29 return spreadBits | (spreadBits << 1);
37template<
typename _Tp,
typename _AbiT,
typename _Ip,
typename _AbiI>
40 static_assert(
sizeof(_Tp) == 16,
"Assume 128-bit is the maximum to deal with for now");
56 constexpr auto oddOnes =
58 return (dup * cw<2>) + oddOnes;
72template<
typename _Tp,
typename _AbiT,
typename _Ip,
typename _AbiI>
75 static_assert(
sizeof(_Tp) == 1 &&
sizeof(_Ip) == 1);
83template<std::
unsigned_
integral _Tp,
typename _AbiT, std::
integral _Ip,
typename _AbiI>
92 auto impl = [_rv]<
typename _ImplT>(_ImplT i) {
96 if constexpr (
sizeof(_Tp) == 4)
97 return _mm256_permutevar8x32_epi32(__m256i(_rv.to_register()), block.to_register());
98 else if constexpr (
sizeof(_Tp) == 8)
102 const auto dup32 = _mm256_castps_si256(_mm256_moveldup_ps(_mm256_castsi256_ps(block.to_register())));
103 const auto mulBy2 = _mm256_slli_epi32(dup32, 1);
104 const auto idx32 = _mm256_add_epi64(mulBy2, __v4di() + 0x100000000);
105 return _mm256_permutevar8x32_epi32(_rv.to_register(), idx32);
111 return fit_to_size<_ImplT::size>(
vec<_Tp>(r));
117#if defined(__AVX512BW__) && defined(__AVX512F__)
120_mm512_permutex2var_epi8_emulated(__m512i v_data_0,
132 auto evenPerm = _mm512_permutex2var_epi16(v_data_0, v_shuf_idxs >> 1, v_data_1);
133 auto oddPerm = _mm512_permutex2var_epi16(v_data_0, v_shuf_idxs >> 9, v_data_1);
141 constexpr __v64qi shuffle_identity = {
142 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
143 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
144 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
145 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
147 auto lsbSelector = _mm512_ternarylogic_epi32(__m512i(__v64qi() + 0x1), __m512i(shuffle_identity), v_shuf_idxs, 0xAC);
152 auto evenResult = _mm512_shuffle_epi8(evenPerm, lsbSelector);
153 auto combinedResult = _mm512_mask_shuffle_epi8(evenResult, 0xAAAAAAAAAAAAAAAA, oddPerm, lsbSelector);
155 return combinedResult;
159#if defined(__AVX512F__)
160inline target_overloads permutexvar {
161 #if defined(__AVX512VBMI__)
162 []<ymm_register<std::uint8_t> _Vec>(_Vec v, _Vec i) {
return _mm256_permutexvar_epi8(i.to_register(), v.to_register()); },
163 []<zmm_register<std::uint8_t> _Vec>(_Vec v, _Vec i) {
return _mm512_permutexvar_epi8(i.to_register(), v.to_register()); },
165 []<ymm_register<std::uint8_t> _Vec>(_Vec v, _Vec i) {
return __m256i(_mm512_permutex2var_epi8_emulated(v.to_register(), i.to_register(), __m512i())); },
166 []<zmm_register<std::uint8_t> _Vec>(_Vec v, _Vec i) {
return _mm512_permutex2var_epi8_emulated(v.to_register(), i.to_register(), __m512i()); },
169 []<ymm_register<std::uint16_t> _Vec>(_Vec v, _Vec i) {
return _mm256_permutexvar_epi16(i.to_register(), v.to_register()); },
170 []<ymm_register<std::uint32_t> _Vec>(_Vec v, _Vec i) {
return _mm256_permutexvar_epi32(i.to_register(), v.to_register()); },
171 []<ymm_register<std::uint64_t> _Vec>(_Vec v, _Vec i) {
return _mm256_permutexvar_epi64(i.to_register(), v.to_register()); },
173 []<zmm_register<std::uint16_t> _Vec>(_Vec v, _Vec i) {
return _mm512_permutexvar_epi16(i.to_register(), v.to_register()); },
174 []<zmm_register<std::uint32_t> _Vec>(_Vec v, _Vec i) {
return _mm512_permutexvar_epi32(i.to_register(), v.to_register()); },
175 []<zmm_register<std::uint64_t> _Vec>(_Vec v, _Vec i) {
return _mm512_permutexvar_epi64(i.to_register(), v.to_register()); },
178inline target_overloads permutex2var {
179 #if defined(__AVX512VBMI__)
180 []<ymm_register<std::uint8_t> _Vec>(_Vec v0, _Vec i, _Vec v1) {
return _mm256_permutex2var_epi8(v0.to_register(), i.to_register(), v1.to_register()); },
181 []<zmm_register<std::uint8_t> _Vec>(_Vec v0, _Vec i, _Vec v1) {
return _mm512_permutex2var_epi8(v0.to_register(), i.to_register(), v1.to_register()); },
183 []<ymm_register<std::uint8_t> _Vec>(_Vec v0, _Vec i, _Vec v1) {
return __m256i(_mm512_permutex2var_epi8_emulated(v0.to_register(), i.to_register(), v1.to_register())); },
184 []<zmm_register<std::uint8_t> _Vec>(_Vec v0, _Vec i, _Vec v1) {
return _mm512_permutex2var_epi8_emulated(v0.to_register(), i.to_register(), v1.to_register()); },
187 []<ymm_register<std::uint16_t> _Vec>(_Vec v0, _Vec i, _Vec v1) {
return _mm256_permutex2var_epi16(v0.to_register(), i.to_register(), v1.to_register()); },
188 []<ymm_register<std::uint32_t> _Vec>(_Vec v0, _Vec i, _Vec v1) {
return _mm256_permutex2var_epi32(v0.to_register(), i.to_register(), v1.to_register()); },
189 []<ymm_register<std::uint64_t> _Vec>(_Vec v0, _Vec i, _Vec v1) {
return _mm256_permutex2var_epi64(v0.to_register(), i.to_register(), v1.to_register()); },
191 []<zmm_register<std::uint16_t> _Vec>(_Vec v0, _Vec i, _Vec v1) {
return _mm512_permutex2var_epi16(v0.to_register(), i.to_register(), v1.to_register()); },
192 []<zmm_register<std::uint32_t> _Vec>(_Vec v0, _Vec i, _Vec v1) {
return _mm512_permutex2var_epi32(v0.to_register(), i.to_register(), v1.to_register()); },
193 []<zmm_register<std::uint64_t> _Vec>(_Vec v0, _Vec i, _Vec v1) {
return _mm512_permutex2var_epi64(v0.to_register(), i.to_register(), v1.to_register()); },
199template<std::
unsigned_
integral _Tp,
typename _AbiT, std::
integral _Ip,
typename _AbiI>
200requires (basic_vec<_Tp, _AbiT>::size <= vec<_Tp>::size)
206 auto impl = [=]<
typename _Vec>(_Vec i) {
210 return fit_to_size<_Vec::size()>(t);
217template<std::
unsigned_
integral _Tp,
typename _AbiT, std::
integral _Ip,
typename _AbiI>
220 sizeof(_Tp) ==
sizeof(_Ip))
236 auto impl = [=]<
typename _Vec>(_Vec idx) {
239 return fit_to_size<_Vec::size>(
vec<_Tp>(t));
248template<std::
unsigned_
integral _Tp,
typename _AbiT, vec_
integral _Ip>
252 constexpr auto halfSize = nextPowerOf2 / 2;
262 const auto indexedLower =
permute(lowerValues, indexes);
263 const auto indexedUpper =
permute(upperValues, indexes);
266 return select((indexes & cw<halfSize>) == cw<0>, indexedLower, indexedUpper);
274template<
typename _Tp,
typename _AbiTp, std::
integral _Ip,
typename _AbiIp, std::derived_from<x86_avx2_tag> VENDOR>
275constexpr basic_vec<_Tp, _AbiIp>
278 using _C = container_for_type<_Tp>;
282 if constexpr (
sizeof(_Tp) == 1 &&
sizeof(_Ip) == 1 &&
287 else if constexpr (
sizeof(_Tp) > 8)
290 else if constexpr (std::same_as<VENDOR, x86_avx2_tag> &&
sizeof(_Tp) < 4)
298 else if constexpr (
sizeof(_Tp) !=
sizeof(_Ip))
319template<std::
size_t _BytesTp,
typename _AbiTp,
typename _Ip,
typename _AbiIp>
331 constexpr auto nonZeroMaskElement =
vec<_Ip, _Np>(_Ip(~_Ip()));
334 const auto r =
permute(maskAsSimd, indexes) != cw<0>;
339template<simd_
size_type _OutSize, std::
size_t _Bytes,
typename _Abi,
typename _Gp>
343#if defined(__AVX512BMI__)
344 using _I = std::uint8_t;
350 using _I = std::conditional_t<basic_mask<_Bytes, _Abi>::size <= 16, std::uint8_t, std::uint16_t>;
358 constexpr auto indexes = [=]{
360 return generate<_R>([](
auto idx) -> _I {
return _I(_Gp{}(idx, _OutSize)); });
362 return generate<_R>([](
auto idx) -> _I {
return _I(_Gp{}(idx)); });
369 using OutputMask = resize_t<_OutSize, basic_mask<_Bytes, _Abi>>;
370 constexpr auto notZeroMask = [=]{
372 return OutputMask([=](
auto i) {
return _Gp{}(i, _OutSize) != zero_element; });
374 return OutputMask([=](
auto i) {
return _Gp{}(i) != zero_element; });
379 return select(notZeroMask, p, OutputMask());
384#if defined(__AVX512F__)
385inline target_overloads compress_one_register {
386#if defined(__AVX512VBMI2__)
390 []<xmm_register<std::uint8_t> _Vec>(_Vec v,
auto m,
auto f)
391 {
return _Vec(_mm_mask_compress_epi8(f.to_register(), m.to_register(), v.to_register())); },
392 []<ymm_register<std::uint8_t> _Vec>(_Vec v,
auto m,
auto f)
393 {
return _Vec(_mm256_mask_compress_epi8(f.to_register(), m.to_register(), v.to_register())); },
394 []<zmm_register<std::uint8_t> _Vec>(_Vec v,
auto m,
auto f)
395 {
return _Vec(_mm512_mask_compress_epi8(f.to_register(), m.to_register(), v.to_register())); },
398 []<xmm_register<std::uint16_t> _Vec>(_Vec v,
auto m,
auto f)
399 {
return _Vec(_mm_mask_compress_epi16(f.to_register(), m.to_register(), v.to_register())); },
400 []<ymm_register<std::uint16_t> _Vec>(_Vec v,
auto m,
auto f)
401 {
return _Vec(_mm256_mask_compress_epi16(f.to_register(), m.to_register(), v.to_register())); },
402 []<zmm_register<std::uint16_t> _Vec>(_Vec v,
auto m,
auto f)
403 {
return _Vec(_mm512_mask_compress_epi16(f.to_register(), m.to_register(), v.to_register())); },
407 []<xmm_register<std::uint8_t> _Vec>(_Vec v,
auto m,
auto f) {
408 auto compressed = _mm512_mask_compress_epi32(_mm512_cvtepi8_epi32(f.to_register()),
410 _mm512_cvtepi8_epi32(v.to_register()));
411 return _Vec(_mm512_cvtepi32_epi8(compressed));
413 []<xmm_register<std::uint16_t> _Vec>(_Vec v,
auto m,
auto f) {
414 auto compressed = _mm256_mask_compress_epi32(_mm256_cvtepi16_epi32(f.to_register()),
416 _mm256_cvtepi16_epi32(v.to_register()));
417 return _Vec(_mm256_cvtepi32_epi16(compressed));
419 []<ymm_register<std::uint16_t> _Vec>(_Vec v,
auto m,
auto f) {
420 auto compressed = _mm512_mask_compress_epi32(_mm512_cvtepi16_epi32(f.to_register()),
422 _mm512_cvtepi16_epi32(v.to_register()));
423 return _Vec(_mm512_cvtepi32_epi16(compressed));
429 []<xmm_register<std::uint32_t> _Vec>(_Vec v,
auto m,
auto f)
430 {
return _Vec(_mm_mask_compress_epi32(f.to_register(), m.to_register(), v.to_register())); },
431 []<ymm_register<std::uint32_t> _Vec>(_Vec v,
auto m,
auto f)
432 {
return _Vec(_mm256_mask_compress_epi32(f.to_register(), m.to_register(), v.to_register())); },
433 []<zmm_register<std::uint32_t> _Vec>(_Vec v,
auto m,
auto f)
434 {
return _Vec(_mm512_mask_compress_epi32(f.to_register(), m.to_register(), v.to_register())); },
437 []<xmm_register<std::uint64_t> _Vec>(_Vec v,
auto m,
auto f)
438 {
return _Vec(_mm_mask_compress_epi64(f.to_register(), m.to_register(), v.to_register())); },
439 []<ymm_register<std::uint64_t> _Vec>(_Vec v,
auto m,
auto f)
440 {
return _Vec(_mm256_mask_compress_epi64(f.to_register(), m.to_register(), v.to_register())); },
441 []<zmm_register<std::uint64_t> _Vec>(_Vec v,
auto m,
auto f)
442 {
return _Vec(_mm512_mask_compress_epi64(f.to_register(), m.to_register(), v.to_register())); },
445 []<xmm_register<unsigned __int128> _Vec>(_Vec v,
auto m,
auto f)
446 {
return _Vec(_mm_mask_compress_epi64(f.to_register(), __mmask8(dupBits(m.to_register())), v.to_register())); },
447 []<ymm_register<unsigned __int128> _Vec>(_Vec v,
auto m,
auto f)
448 {
return _Vec(_mm256_mask_compress_epi64(f.to_register(), __mmask8(dupBits(m.to_register())), v.to_register())); },
449 []<zmm_register<unsigned __int128> _Vec>(_Vec v,
auto m,
auto f)
450 {
return _Vec(_mm512_mask_compress_epi64(f.to_register(), __mmask8(dupBits(m.to_register())), v.to_register())); },
452 [](
auto unhandled,
auto,
auto) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics");
return unhandled;}
456 template <
typename _Tp,
typename _Abi,
typename VENDOR>
458 compress_by_mask_to_memory(VENDOR, _Tp* ptr,
const typename basic_vec<_Tp, _Abi>::mask_type& selector,
459 const basic_vec<_Tp, _Abi>& values, _Tp fill_value)
463 auto compressBlock = [&]<
typename _Vec>(_Vec x,
typename _Vec::mask_type m)
465 const auto fillV = _Vec(fill_value);
466 auto compressed = compress_one_register(x, m, fillV);
473 constexpr int numBlocks = (
sizeof(_Tp) >= 4 || std::derived_from<VENDOR, x86_avxsnc_tag>) ? vec<_Tp>::size : 16;
474 chunked_invoke<numBlocks>(compressBlock, values, selector);
479 template<vec_type _V>
481compress_by_mask_snc(
const _V& values,
const typename _V::mask_type& selector,
typename _V::value_type fill_value)
483 constexpr int _Np = _V::size;
484 static_assert(_Np <= detail::maxBytesInVec,
"Cannot compress more than one native register at once");
489 constexpr auto indexes = iota<vec<std::uint8_t, _Np>>;
490 const auto compressedIndexes =
chunked_invoke(compress_one_register, indexes, mask<std::uint8_t, _Np>(selector), vec<std::uint8_t, _Np>());
491 auto compressed =
permute(values, compressedIndexes);
493 auto finalMask = _V::mask_type::__mask_from_count(numSetBits);
494 return select(finalMask, compressed, _V(fill_value));
500template<vec_type _V, std::derived_from<x86_avx512_tag> VENDOR>
502compress_by_mask(VENDOR tag,
const _V& values,
const typename _V::mask_type& selector,
503 typename _V::value_type fill_value = {})
505 constexpr int _Np = _V::size;
506 using _Tp =
typename _V::value_type;
507 static_assert(vec_container_element<_Tp>,
"Caller must convert to container");
510 constexpr bool isSnc = std::derived_from<VENDOR, x86_avxsnc_tag>;
511 constexpr int maxElementsPerExpand = isSnc ? vec<_Tp>::size : std::min<int>(16, vec<_Tp>::size);
513 if constexpr (_Np <= maxElementsPerExpand)
517 return compress_one_register(values, selector, _V(fill_value));
519 else if constexpr (isSnc && _Np <= detail::maxBytesInVec)
521 return compress_by_mask_snc(values, selector, fill_value);
525 _V result(fill_value);
526 compress_by_mask_to_memory(tag,
reinterpret_cast<_Tp*
>(&result), selector, values, fill_value);
533template<mask_type _M>
535compress_mask_by_mask(x86_avx2_tag,
const _M& values,
const _M& selector)
537 constexpr int _Np = _M::size;
541 static_assert(_Np <= 128);
543 using _R = std::conditional_t<(_Np > 64), __uint128_t, std::uint64_t>;
546 constexpr int lowerN = std::min<int>(_Np, 64);
547 auto maskAsUll = detail::fit_to_size<lowerN>(selector).to_ullong();
548 _R resultInt = _pext_u64(detail::fit_to_size<lowerN>(values).to_ullong(), maskAsUll);
551 if constexpr (_Np > 64)
553 _R upperCompressed = _pext_u64(drop<64>(values).to_ullong(), drop<64>(selector).to_ullong());
554 resultInt |= (upperCompressed << std::popcount((std::uint64_t)maskAsUll));
558 _M resultMask((std::uint64_t)resultInt);
559 if constexpr (_Np > 64) {
560 auto upper = resize_t<_Np - 64, _M>(std::uint64_t(resultInt >> 64));
561 resultMask =
cat(take<64>(resultMask), upper);
567template<mask_type _M>
569compress_mask_by_mask(x86_avx512_tag,
const _M& v,
const _M& selector)
572 constexpr int _Np = _M::size;
574 auto vd = v.to_builtin();
575 auto md = selector.to_builtin();
579 static_assert(_Np <= 128);
582 using _B =
typename _M::builtin_type;
583 _B result = _pext_u64((std::uint64_t)vd, (std::uint64_t)md);
586 if constexpr (_Np > 64)
588 _B upperCompressed = _pext_u64(vd >> 64, md >> 64);
589 result |= (upperCompressed << std::popcount((std::uint64_t)md));
592 return _M::from_builtin(result);
596#if defined(__AVX512F__)
597inline target_overloads expand_one_register {
598#if defined(__AVX512VBMI2__)
600 []<xmm_register<std::uint8_t> _Vec>(_Vec v,
auto m,
auto alt)
601 {
return _Vec(_mm_mask_expand_epi8(alt.to_register(), m.to_register(), v.to_register())); },
602 []<ymm_register<std::uint8_t> _Vec>(_Vec v,
auto m,
auto alt)
603 {
return _Vec(_mm256_mask_expand_epi8(alt.to_register(), m.to_register(), v.to_register())); },
604 []<zmm_register<std::uint8_t> _Vec>(_Vec v,
auto m,
auto alt)
605 {
return _Vec(_mm512_mask_expand_epi8(alt.to_register(), m.to_register(), v.to_register())); },
607 []<xmm_register<std::uint16_t> _Vec>(_Vec v,
auto m,
auto alt)
608 {
return _Vec(_mm_mask_expand_epi16(alt.to_register(), m.to_register(), v.to_register())); },
609 []<ymm_register<std::uint16_t> _Vec>(_Vec v,
auto m,
auto alt)
610 {
return _Vec(_mm256_mask_expand_epi16(alt.to_register(), m.to_register(), v.to_register())); },
611 []<zmm_register<std::uint16_t> _Vec>(_Vec v,
auto m,
auto alt)
612 {
return _Vec(_mm512_mask_expand_epi16(alt.to_register(), m.to_register(), v.to_register())); },
619 []<xmm_register<std::uint8_t> _Vec>(_Vec v,
auto m,
auto alt) {
620 auto compressed = _mm512_maskz_expand_epi32(m.to_register(),
621 _mm512_cvtepi8_epi32(v.to_register()));
622 return _Vec(_mm512_mask_cvtepi32_epi8(alt.to_register(), m.to_register(), compressed));
624 []<xmm_register<std::uint16_t> _Vec>(_Vec v,
auto m,
auto alt) {
625 auto compressed = _mm256_maskz_expand_epi32(m.to_register(),
626 _mm256_cvtepi16_epi32(v.to_register()));
627 return _Vec(_mm256_mask_cvtepi32_epi16(alt.to_register(), m.to_register(), compressed));
629 []<ymm_register<std::uint16_t> _Vec>(_Vec v,
auto m,
auto alt) {
630 auto compressed = _mm512_maskz_expand_epi32(m.to_register(),
631 _mm512_cvtepi16_epi32(v.to_register()));
632 return _Vec(_mm512_mask_cvtepi32_epi16(alt.to_register(), m.to_register(), compressed));
637 []<xmm_register<std::uint32_t> _Vec>(_Vec v,
auto m,
auto alt)
638 {
return _Vec(_mm_mask_expand_epi32(alt.to_register(), m.to_register(), v.to_register())); },
639 []<ymm_register<std::uint32_t> _Vec>(_Vec v,
auto m,
auto alt)
640 {
return _Vec(_mm256_mask_expand_epi32(alt.to_register(), m.to_register(), v.to_register())); },
641 []<zmm_register<std::uint32_t> _Vec>(_Vec v,
auto m,
auto alt)
642 {
return _Vec(_mm512_mask_expand_epi32(alt.to_register(), m.to_register(), v.to_register())); },
645 []<xmm_register<std::uint64_t> _Vec>(_Vec v,
auto m,
auto alt)
646 {
return _Vec(_mm_mask_expand_epi64(alt.to_register(), m.to_register(), v.to_register())); },
647 []<ymm_register<std::uint64_t> _Vec>(_Vec v,
auto m,
auto alt)
648 {
return _Vec(_mm256_mask_expand_epi64(alt.to_register(), m.to_register(), v.to_register())); },
649 []<zmm_register<std::uint64_t> _Vec>(_Vec v,
auto m,
auto alt)
650 {
return _Vec(_mm512_mask_expand_epi64(alt.to_register(), m.to_register(), v.to_register())); },
653 []<xmm_register<unsigned __int128> _Vec>(_Vec v,
auto m,
auto alt)
654 {
return _Vec(_mm_mask_expand_epi64(alt.to_register(), __mmask8(dupBits(m.to_register())), v.to_register())); },
655 []<ymm_register<unsigned __int128> _Vec>(_Vec v,
auto m,
auto alt)
656 {
return _Vec(_mm256_mask_expand_epi64(alt.to_register(), __mmask8(dupBits(m.to_register())), v.to_register())); },
657 []<zmm_register<unsigned __int128> _Vec>(_Vec v,
auto m,
auto alt)
658 {
return _Vec(_mm512_mask_expand_epi64(alt.to_register(), __mmask8(dupBits(m.to_register())), v.to_register())); },
660 [](
auto unhandled,
auto,
auto) {
static_assert(
dependent_false<
decltype(unhandled)>,
"No target intrinsics");
return unhandled;}
663template<vec_type _V,
typename VENDOR>
665expand_by_mask_from_memory(VENDOR,
const typename _V::value_type* ptr,
666 const typename _V::mask_type& selector,
const _V& original)
668 using _Tp =
typename _V::value_type;
672 auto expandBlock = [&]<
typename _Vec>(_Vec x,
typename _Vec::mask_type m) {
673 auto t = detail::load<_Vec>(target, std::span<const _Tp, _Vec::size>(nextPos, _Vec::size), detail::add_unchecked_flag(flags<>{}));
675 return expand_one_register(t, m, x);
680 constexpr int numBlocks = (
sizeof(_Tp) >= 4 || std::derived_from<VENDOR, x86_avxsnc_tag>) ? vec<_Tp>::size : 16;
681 return chunked_invoke<numBlocks>(expandBlock, original, selector);
686expand_by_mask_snc(
const _V& value,
const typename _V::mask_type& selector,
const _V& original)
688 constexpr int _Np = _V::size;
690 static_assert(_Np <= detail::maxBytesInVec);
695 constexpr auto indexes = iota<vec<std::uint8_t, _Np>>;
696 const auto expandedIndexes =
chunked_invoke(expand_one_register, indexes, mask<std::uint8_t, _Np>(selector), vec<std::uint8_t, _Np>());
697 auto expanded =
permute(value, expandedIndexes);
698 return select(selector, expanded, original);
701template<vec_type _V, std::derived_from<x86_avx512_tag> VENDOR>
703expand_by_mask(VENDOR tag,
const _V& values,
const typename _V::mask_type& selector,
const _V& original)
705 using _Tp =
typename _V::value_type;
706 static_assert(vec_container_element<_Tp>,
"Caller must convert to container");
707 constexpr auto _Np = _V::size;
710 constexpr bool isSnc = std::derived_from<VENDOR, x86_avxsnc_tag>;
711 const int maxElementsPerExpand = isSnc ? vec<_Tp>::size() : std::
min<int>(16,
vec<_Tp>::size());
713 if constexpr (_Np <= maxElementsPerExpand)
716 return expand_one_register(values, selector, original);
717 else if constexpr (isSnc && _Np <= detail::maxBytesInVec)
719 return expand_by_mask_snc(values, selector, original);
723 return expand_by_mask_from_memory(tag,
reinterpret_cast<const _Tp*
>(&values), selector, original);
729template<mask_type _M>
731expand_mask_by_mask(x86_avx_tag,
const _M& values,
const _M& selector,
const _M& original)
733 constexpr int _Np = _M::size;
737 static_assert(_Np <= 128);
739 using _R = std::conditional_t<(_Np > 64), __uint128_t, std::uint64_t>;
742 constexpr int lowerN = std::min<int>(_Np, 64);
743 auto lowerSelector = detail::fit_to_size<lowerN>(selector).to_ullong();
744 auto lowerValue = detail::fit_to_size<lowerN>(values).to_ullong();
745 _R resultInt = _pdep_u64(lowerValue, lowerSelector);
748 if constexpr (_Np > 64)
750 _R upper64Value = drop<64>(values).to_ullong();
751 std::uint64_t upper64Selector = drop<64>(selector).to_ullong();
752 auto wholeValue = (upper64Value << 64) | _R(lowerValue);
753 auto bitsToExpand = std::uint64_t(wholeValue >> std::popcount(lowerSelector));
754 _R expandedBits = _pdep_u64(bitsToExpand, upper64Selector);
755 resultInt |= _R(expandedBits) << 64;
758 _M resultMask((std::uint64_t)resultInt);
759 if constexpr (_Np > 64)
760 resultMask =
cat(take<64>(resultMask), resize_t<_Np - 64, _M>(std::uint64_t(resultInt >> 64)));
763 auto final = resultMask.to_builtin() | (original.to_builtin() & ~(selector.to_builtin()));
764 return _M::from_builtin(
final);
767template<mask_type _M>
769expand_mask_by_mask(x86_avx512_tag,
const _M& values,
const _M& selector,
const _M& original)
771 constexpr int _Np = _M::size;
773 auto vd = values.to_builtin();
774 auto sd = selector.to_builtin();
778 static_assert(_Np <= 128);
781 typename _M::builtin_type result = _pdep_u64((std::uint64_t)vd, (std::uint64_t)sd);
784 if constexpr (_Np > 64)
786 std::uint64_t useBits = (std::uint64_t)(vd >> std::popcount((std::uint64_t)sd));
787 typename _M::builtin_type expandedBits = _pdep_u64(useBits, (std::uint64_t)(sd >> 64));
788 result |= expandedBits << 64;
792 return _M::from_builtin(result | (original.to_builtin() & ~sd));
801template <
int _Np, mask_type _Mp>
804 static_assert(_Mp::size >= _Np,
"Too short to drop N elements");
805 return resize_t<_Mp::size - _Np, _Mp>::from_builtin(in.to_register() >> _Np);
809template <
int _Np, mask_type _Mp>
812 static_assert(_Np <= _Mp::size,
"Too short to drop N elements");
813 return resize_t<_Np, _Mp>::from_builtin(in.to_register());
817template<
int _Np, mask_type _Mp>
819 static_assert(_Np >= _Mp::size,
"Must grow to a size bigger than the input");
820 return resize_t<_Np, _Mp>::from_builtin(in.to_register());
824template<
int _Np, mask_type _Mp>
825constexpr auto permute_grow(x86_avx512_tag,
const _Mp& in,
bool fill)
noexcept {
826 static_assert(_Np >= _Mp::size,
"Must grow to a size bigger than the input");
827 using _NewMp = resize_t<_Np, _Mp>;
828 using _CM =
typename _NewMp::builtin_type;
830 auto f = fill ? ~(_CM()) : _CM();
831 const auto lowerMask = ~(_CM()) << _Mp::size();
833 return _NewMp::from_builtin(in.to_register() | (f & lowerMask));
840template<mask_type _Mp,
int _Str
ide>
842 using _Out = resize_t<_Mp::size * _Stride, _Mp>;
843 using _I =
typename _Out::builtin_type;
844 return [=]<std::size_t... _Iota>(std::index_sequence<_Iota...>) {
845 return ((_I(1) << (_Iota * _Stride)) | ...);
846 }(std::make_index_sequence<_Mp::size>{});
849#if defined(__AVX512F__)
851template<
int _Np, mask_type _Mp>
852constexpr auto permute_stride(x86_avx512_tag,
const _Mp& in)
noexcept {
853 constexpr auto _OldSize = _Mp::size;
856 static_assert(_OldSize <= 128,
"Hard-coded to a pair of 64-bit pext instructions");
859 if constexpr (_Np == 1)
return in;
861 constexpr int _NewSize = (_OldSize + (_Np - 1)) / _Np;
862 using _NewMp = resize_t<_NewSize, _Mp>;
864 auto iv = in.to_register();
870 constexpr auto bits = every_nth_bit<_NewMp, _Np>();
871 std::uint64_t t = _pext_u64((std::uint64_t)iv, (std::uint64_t)bits);
874 if constexpr (_OldSize > 64) {
875 auto upperBits = std::uint64_t(bits >> 64);
876 std::uint64_t b = _pext_u64((std::uint64_t)(iv >> 64), upperBits);
877 t |= b << std::popcount(std::uint64_t(bits & 0xFFFFFFFFFFFFFFFF));
880 return _NewMp::from_builtin(t);
A mask containing boolean elements.
Definition mask.hpp:121
Vector of elements sharing an element type and ABI.
Definition simd:161
static constexpr auto size
Return the number of elements contained in the vec.
Definition simd:187
constexpr _Vp min(const _Vp &lhs, const _Vp &rhs) noexcept
Compute the smaller of each respective vec element from the two input arguments.
Definition std_algorithm.hpp:34
constexpr auto select(const _M &m, const _T &if_true, const _U &if_false) noexcept -> decltype(simd_select_impl(m, if_true, if_false))
Select between corresponding elements according to a mask.
Definition std_algorithm.hpp:88
constexpr auto chunk(const _S &container) noexcept
Definition split_concat.hpp:107
constexpr auto cat(const basic_mask< _Bytes, _Abis > &... masks) noexcept
Concatenate several smaller mask values with the same element size into a single large mask value.
Definition split_concat.hpp:144
constexpr void unchecked_store(const basic_vec< _Tp, _Abi > &value, _Range &&r, flags< _Flags... > f={})
Unchecked store into a contiguous sized range.
Definition loadstore.hpp:332
basic_mask< sizeof(_Tp), simd_fixed_size_abi< _Np > > mask
Definition core.hpp:383
static constexpr auto size
Return the number of elements contained in the mask.
Definition mask.hpp:148
constexpr simd_size_type reduce_count(const mask_type auto &mask) noexcept
Count the true mask elements.
Definition mask.hpp:507
basic_vec< _Tp, simd_fixed_size_abi< _Np > > vec
Definition core.hpp:379
constexpr auto permute(const _Vp &v, detail::index_generator_function auto &&fn)
Permute the elements in a vec value using a generator function to compute the indexing.
Definition permute.hpp:94
constexpr auto chunked_invoke(_Fn fn, const _Arg0 &arg0, const _Args &... args)
Invoke a callable on corresponding vector or mask chunks.
Definition invoke.hpp:61
constexpr auto take(const vec_or_mask_type auto &in)
Definition named_permute.hpp:237
constexpr auto drop(const vec_or_mask_type auto &in)
Discard the first _Np elements of the input, returning the remainder.
Definition named_permute.hpp:240
constexpr detail::RebindCast< _Up > rebind_cast
Rebinding CPO.
Definition std_rebind.hpp:55
constexpr traits::register_type to_register() const
Definition simd:344
constexpr auto simd_bit_cast(const _From &v) noexcept
A simd-generic version of simd_bit_cast which directly invokes std::bit_cast.
Definition core.hpp:585
constexpr auto permute_grow(generic_tag, const _Sp &in) noexcept
A named permute which increases the number of elements in the vec or mask, placing a copy of the give...
Definition named_permute.hpp:63
constexpr auto every_nth_bit()
Definition permute.hpp:841
constexpr auto permute_large_elements(const basic_vec< _Tp, _AbiT > &values, const basic_vec< _Ip, _AbiI > &indexes)
Definition permute.hpp:38
constexpr basic_vec< _Tp, _AbiIp > permute(VENDOR tag, const basic_vec< _Tp, _AbiTp > &v, const basic_vec< _Ip, _AbiIp > &indexes)
Definition permute.hpp:276
constexpr auto permute_drop(generic_tag, const _Sp &in) noexcept
A named permute which discards the first _Np values of a vec or mask, returning the remainder.
Definition named_permute.hpp:30
auto permute_x86_register(x86_avx2_tag, const basic_vec< _Tp, _AbiT > &v, const basic_vec< _Ip, _AbiI > &indexes)
Permute one register of Intel AVX2 values.
Definition permute.hpp:85
constexpr auto generated_permute(x86_avx512_tag, const basic_mask< _Bytes, _Abi > &m, _Gp)
Use a generator to permute a compact bit mask.
Definition permute.hpp:341
constexpr bool dependent_false
Definition utilities.hpp:103
constexpr auto permute_take(generic_tag, const _Sp &in)
A named permute which keeps the first _Np values of a simd and discards the remainder.
Definition named_permute.hpp:22
constexpr auto permute_byte_shuffle(const basic_vec< _Tp, _AbiT > &v, const basic_vec< _Ip, _AbiI > &indexes)
Definition permute.hpp:73