21#include <xvec/detail/config.hpp>
22#include <xvec/detail/utilities.hpp>
24namespace _XVEC_NAMESPACE::simd
27inline constexpr simd_size_type zero_element = 9999123;
28inline constexpr simd_size_type uninit_element = 9999124;
41 requires (vec_type<_Tp> || std::is_arithmetic_v<_Tp>)
42inline constexpr _Tp iota =
43 []{
if constexpr (vec_type<_Tp>)
return _Tp([](simd_size_type i) {
return typename _Tp::value_type(i); });
44 else return _Tp{}; }();
50const auto perm_uninitResize = [](
auto idx,
auto isize) {
return idx < isize ? idx : uninit_element; };
53template<vec_type _Vp,
typename _Gp>
54constexpr auto generate(_Gp generator) {
55 if constexpr (complex_number<typename _Vp::value_type>)
57 using _Tp =
typename _Vp::traits::element_type;
58 return _Vp([=]<simd_size_type... _Idx>(std::integer_sequence<simd_size_type, _Idx...>) {
59 return typename _Vp::traits::builtin_type{_Tp((_Idx % 2) ?
60 generator(size_constant<_Idx / 2>()).imag() :
61 generator(size_constant<_Idx / 2>()).real())...};
62 }(std::make_integer_sequence<simd_size_type, _Vp::size() * 2>()));
66 return _Vp([=]<simd_size_type... _Idx>(std::integer_sequence<simd_size_type, _Idx...>) {
67 return typename _Vp::traits::builtin_type{std::bit_cast<typename _Vp::traits::element_type>(
68 generator(size_constant<_Idx>{}))...};
69 }(std::make_integer_sequence<simd_size_type, _Vp::size()>()));
73template<simd_
size_type _NewSize, vec_or_mask_type _Vp>
74constexpr resize_t<_NewSize, _Vp> fit_to_size(
const _Vp& v) {
75 if constexpr (_NewSize == _Vp::size())
return v;
76 else return permute<_NewSize>(v, [](simd_size_type i) {
return (i < _Vp::size()) ? i : zero_element; });
79template<vec_type _Vp, contiguous_sized_range _Range,
typename _Flags>
80constexpr void mandates_for_store(
const _Vp&,
const _Range&, _Flags f)
82 using _Tp =
typename _Vp::value_type;
83 using _Rp = std::ranges::range_value_t<_Range>;
85 static_assert(detail::contains_flag<detail::flag_convert>(f) ||
86 !detail::needs_explicit_conversion<_Tp, _Rp>,
87 "Store or scatter conversion requires flag_convert");
90template<vec_type _Vp, contiguous_sized_range _Range,
typename _Flags>
91constexpr void mandates_for_load(
const _Range&,
const _Vp&, _Flags f)
93 using _Tp =
typename _Vp::value_type;
94 using _Rp = std::ranges::range_value_t<_Range>;
96 static_assert(detail::vectorizable<_Rp>);
97 static_assert(detail::explicitly_convertible_to<_Rp, _Tp>);
98 static_assert(std::same_as<std::remove_cvref_t<_Vp>, _Vp>);
102 static_assert(detail::contains_flag<detail::flag_convert>(f) ||
103 !detail::needs_explicit_conversion<_Rp, _Tp>,
104 "Load or gather conversion requires flag_convert");
115template<vec_type _Vp>
125template<vec_or_mask_type _Vp>
126constexpr typename _Vp::value_type subscript(generic_tag,
const _Vp& v, simd_size_type pos) {
127 assert(pos < _Vp::size());
128 auto t = detail::vec_as_container(v).to_builtin()[pos];
130 if constexpr (vec_complex<_Vp>)
135 using _E = complex_element_type<typename _Vp::value_type>;
136 auto asCmplxArray = std::bit_cast<std::array<_E, 2>>(t);
137 return std::complex<_E>(asCmplxArray[0], asCmplxArray[1]);
139 else if constexpr (mask_type<_Vp>)
140 return typename _Vp::value_type(t);
142 return std::bit_cast<typename _Vp::value_type>(t);
145template<mask_type _Mp>
146constexpr bool subscript(compact_mask_tag,
const _Mp& v, simd_size_type pos)
147 {
return (v.to_register() >> pos) & 1; }
152template<mask_type _Mp,
typename _Gp>
153constexpr _Mp generate_mask(generic_tag, _Gp generator) {
154 using _Tp = container_for_num_bytes<simd_mask_element_size_v<_Mp>>;
155 using _Vp =
typename _Mp::builtin_type;
157 auto r = [=]<simd_size_type... _Idx>(std::integer_sequence<simd_size_type, _Idx...>) {
158 return _Vp{_Tp(generator(size_constant<_Idx>()) ? ~_Tp() : _Tp())...};
159 }(std::make_integer_sequence<simd_size_type, _Mp::size()>());
161 return _Mp::from_builtin(r);
164template<mask_type _Mp,
typename _Gp>
165constexpr _Mp generate_mask(compact_mask_tag, _Gp generator) {
166 return _Mp::from_builtin([=]<simd_size_type... _Idx>(std::integer_sequence<simd_size_type, _Idx...>) {
167 return ((typename _Mp::builtin_type(generator(size_constant<_Idx>{})) << _Idx) | ...);
168 } (std::make_integer_sequence<simd_size_type, _Mp::size()>()));
172template<mask_type _Mp>
173constexpr std::bitset<_Mp::size>
174mask_to_bitset(generic_tag,
const _Mp& mask)
noexcept
178 std::bitset<_Mp::size> result;
179 for (
int i=0; i<_Mp::size; i++)
180 result[i] = (mask[i] != 0);
184constexpr unsigned long long mask_to_ullong(generic_tag,
const mask_type
auto& mask)
187constexpr unsigned long long mask_to_ullong(compact_mask_tag,
const mask_type
auto& mask)
201template<vec_type _Vp, vec_
integral _Ip>
202constexpr resize_t<_Ip::size(), _Vp>
203permute(generic_tag,
const _Vp& v,
const _Ip& indexes)
204 {
return partial_gather_from(std::span(
reinterpret_cast<const typename _Vp::value_type*
>(&v), _Vp::size()), indexes); }
206template<mask_type _Mp, vec_
integral _Ip>
207constexpr resize_t<_Ip::size(), _Mp>
208permute(generic_tag,
const _Mp& m,
const _Ip& indexes)
211template<mask_type _Mp, vec_
integral _Ip>
212constexpr resize_t<_Ip::size(), _Mp>
213permute(compact_mask_tag,
const _Mp& m,
const _Ip& indexes)
214 {
return permute(-m, indexes) != cw<0>; }
224template<vec_type _Vp>
226select_if_else(generic_tag,
const typename _Vp::mask_type& mask,
const _Vp& if_true,
const _Vp& if_false)
228 using _C =
typename _Vp::traits::element_container_type;
231 auto ta = simd_bit_cast<_C>(if_true).to_builtin();
232 auto tb = simd_bit_cast<_C>(if_false).to_builtin();
235 if (std::is_constant_evaluated())
236 r = (tm & ta) | ((~tm) & tb);
240 return simd_bit_cast<typename _Vp::value_type>(r);
243template<vec_type _Vp>
245select_if_else(compact_mask_tag,
const typename _Vp::mask_type& mask,
const _Vp& if_true,
const _Vp& if_false)
246 {
return _Vp([=](
auto i) {
return mask[i] ? if_true[i] : if_false[i]; }); }
248template<mask_type _Mp>
250select_if_else(compact_mask_tag,
const _Mp& if_mask,
const _Mp& if_true,
const _Mp& if_false) {
251 auto r = (if_true.to_builtin() & if_mask.to_builtin()) | (if_false.to_builtin() & ~if_mask.to_builtin());
252 return _Mp::from_builtin(r);
261template<mask_type _Mp>
263select_if_else(generic_tag,
const _Mp& mask,
const _Mp& if_true,
const _Mp& if_false)
noexcept
264{
return _Mp::from_builtin(
mask.
to_builtin() ? if_true.to_builtin() : if_false.to_builtin()); }
276template<vec_type _Vp,
typename _Up, std::
size_t _Extent,
typename _Flags>
277constexpr void store_masked(generic_tag,
const _Vp& value_original, std::span<_Up, _Extent> to,
278 const typename _Vp::mask_type& mask_original, _Flags flags)
280 mandates_for_store(value_original, to, flags);
282 auto limit = _Vp::size();
283 if constexpr (contains_flag<flag_unchecked>(flags))
284 checkStaticMemoryBounds<rebind_t<_Up, _Vp>>(
"store", to, mask_original);
286 limit = std::min<std::size_t>(to.size(), _Vp::size());
288 for (
int i=0; i<limit; i++)
289 if (mask_original[i])
290 to[i] =
static_cast<_Up
>(value_original[i]);
293template<vec_type _Vp,
typename _Up, std::
size_t _Extent,
typename _Flags>
294constexpr void store(generic_tag,
const _Vp& value_original, std::span<_Up, _Extent> to, _Flags flags)
296 mandates_for_store(value_original, to, flags);
298 auto limit = _Vp::size();
299 if constexpr (contains_flag<flag_unchecked>(flags))
300 checkStaticMemoryBounds<rebind_t<_Up, _Vp>>(
"store", to);
302 limit = std::min<std::size_t>(to.size(), _Vp::size());
304 for (
int i=0; i<limit; i++)
305 to[i] =
static_cast<_Up
>(value_original[i]);
317template<vec_type _V,
typename _Up, std::
size_t _Extent,
typename _Flags>
318constexpr _V load(generic_tag, std::span<_Up, _Extent> from, _Flags flags)
320 mandates_for_load(from, _V{}, flags);
322 using SourceType = rebind_t<_Up, _V>;
325 std::remove_const_t<typename SourceType::builtin_type> result = {};
326 if constexpr (contains_flag<flag_unchecked>(flags))
328 checkStaticMemoryBounds<SourceType>(
"load", from);
329 if (std::is_constant_evaluated())
330 return _V([=](
auto i) ->
typename _V::value_type {
return from[i]; });
333 memcpy(&result, from.data(),
sizeof(_Up) * _V::size);
334 return _V(SourceType(result));
340 auto numElementsToLoad = std::min<std::size_t>(_V::size, from.size());
342 if (std::is_constant_evaluated())
343 return _V([=](
auto i) ->
typename _V::value_type {
return i < numElementsToLoad ? from[i] : _Up(); });
346 __builtin_memcpy(&result, from.data(),
sizeof(_Up) * numElementsToLoad);
347 return _V(SourceType(result));
362template<vec_type _V,
typename _Up, std::
size_t _Extent,
typename _Flags>
363constexpr _V load_masked(generic_tag, std::span<_Up, _Extent> from,
const typename _V::mask_type& m, _Flags flags)
365 mandates_for_load(from, _V{}, flags);
367 using SourceType = vec<_Up, _V::size>;
369 std::remove_const_t<typename SourceType::builtin_type> result = {};
370 if constexpr (contains_flag<flag_unchecked>(flags))
372 checkStaticMemoryBounds<SourceType>(
"load", from, m);
373 if (std::is_constant_evaluated())
374 return select(m, _V([=](
auto i) ->
typename _V::value_type {
return from[i]; }), _V());
379 const auto numElementsToLoad = std::min<std::size_t>(_V::size, from.size());
380 memcpy(&result, from.data(),
sizeof(_Up) * numElementsToLoad);
381 return select(m, _V(SourceType(result)), _V());
387 auto numElementsToLoad = std::min<std::size_t>(_V::size, from.size());
388 if (std::is_constant_evaluated())
389 return select(m, _V([=](
auto i) ->
typename _V::value_type {
return i < numElementsToLoad ? from[i] : _Up(); }), _V());
392 memcpy(&result, from.data(),
sizeof(_Up) * numElementsToLoad);
393 return select(m, _V(SourceType(result)), _V());
402template<simd_
size_type _OutSize, vec_or_mask_type _Vp,
typename _Gp>
403constexpr resize_t<_OutSize, _Vp> permute_pair(
const _Vp& v0,
const _Vp& v1, _Gp) {
407 using _C =
typename decltype(c0)::value_type;
408 constexpr int _Np = _Vp::size;
411 auto getIndex = [](
auto idx) {
417 if constexpr (idx >= _OutSize)
418 return size_constant<-1>();
421 constexpr int outIdx = simd_size_type(_Gp{}(idx));
426 constexpr int paddingAdjustedIdx = outIdx < _Np ? outIdx : int(outIdx - _Np) + int(vec_traits<target_tag, _C, _Np>::_numStorageElements);
428 return size_constant<paddingAdjustedIdx>();
432 using OutTraits = vec_traits<target_tag, _C, _OutSize>;
433 auto r = [=]<std::size_t... _Idx>(std::index_sequence<_Idx...>) ->
typename OutTraits::builtin_type {
434 return __builtin_shufflevector(c0.to_builtin(), c1.to_builtin(), (getIndex(std::integral_constant<int, _Idx>()).value)...);
435 }(std::make_index_sequence<OutTraits::_numStorageElements>());
437 if constexpr(vec_type<_Vp>)
438 return simd_bit_cast<typename _Vp::value_type>(vec<_C, _OutSize>(r));
441 return resize_t<_OutSize, _Vp>::from_builtin(r);
453template<simd_
size_type _OutSize, vec_or_mask_type _Vp,
typename _Gp>
454constexpr resize_t<_OutSize, _Vp>
455generated_permute(generic_tag,
const _Vp& v, _Gp) {
456 constexpr simd_size_type _Np = _Vp::size;
459 using _C =
typename decltype(c0)::value_type;
465 constexpr auto zeroIndex = vec_traits<target_tag, _C, _Np>::_numStorageElements;
467 auto getIndex = []<
auto idx>() {
469 if constexpr (idx >= _OutSize) {
470 return size_constant<-1>{};
473 constexpr simd_size_type gi = [] {
474 if constexpr (index_generator_function_with_size<_Gp>)
475 return _Gp{}(size_constant<idx>{}, _Np);
477 return _Gp{}(size_constant<idx>{});
480 if constexpr (gi == zero_element)
481 return size_constant<zeroIndex>{};
482 else if constexpr (gi == uninit_element)
484 return size_constant<-1>{};
487 static_assert(gi >= 0 && gi < _Np,
"Computed index isn't in valid input range");
488 return size_constant<gi>{};
493 using OutTraits = vec_traits<target_tag, _C, _OutSize>;
495 typename OutTraits::builtin_type r;
496 constexpr auto emptyBuiltin =
decltype(c0.to_builtin()){};
497 if (std::is_constant_evaluated())
507 r = [=]<std::size_t... _Idx>(std::index_sequence<_Idx...>) ->
typename OutTraits::builtin_type {
508 auto fixUninit = [](
auto idx) {
return idx < 0 ? zeroIndex : idx; };
509 return __builtin_shufflevector(c0.to_builtin(), emptyBuiltin, fixUninit(getIndex.template
operator()<_Idx>().value)...);
510 }(std::make_index_sequence<OutTraits::_numStorageElements>());
513 r = [=]<std::size_t... _Idx>(std::index_sequence<_Idx...>) ->
typename OutTraits::builtin_type {
514 return __builtin_shufflevector(c0.to_builtin(), emptyBuiltin, (getIndex.template
operator()<_Idx>().value)...);
515 }(std::make_index_sequence<OutTraits::_numStorageElements>());
518 if constexpr (vec_type<_Vp>)
519 return simd_bit_cast<typename _Vp::value_type>(vec<_C, _OutSize>(r));
522 return resize_t<_OutSize, _Vp>::from_builtin(r);
525template<simd_
size_type _OutSize, mask_type _Mp,
typename _Gp>
527generated_permute(compact_mask_tag,
const _Mp& m, _Gp)
528 {
return permute<_OutSize>(-m, _Gp{}) != cw<0>; }
540template<vec_type _Vp>
542expand_by_mask(generic_tag,
const _Vp& v,
const typename _Vp::mask_type& mask,
const _Vp& original)
544 typename _Vp::value_type result[_Vp::size];
547 auto inIter = (
const typename _Vp::value_type*)(&v);
549 for (
int i=0; i<_Vp::size; ++i)
565template<mask_type _Mp>
567expand_mask_by_mask(generic_tag,
const _Mp& value,
const _Mp& mask,
const _Mp& original = {})
570template<mask_type _Mp>
572expand_mask_by_mask(compact_mask_tag,
const _Mp& v,
const _Mp& selector,
const _Mp& original = {})
573 {
return expand(-v, selector, -original) != cw<0>; }
588template<vec_type _Vp,
typename _FillType = compress_fill_uninitialized_t>
590compress_by_mask(
generic_tag,
const _Vp& v,
const typename _Vp::mask_type&
mask, _FillType fill_value = {})
592 constexpr int _Np = _Vp::size;
593 typename _Vp::value_type result[_Np];
598 if constexpr (!std::same_as<compress_fill_uninitialized_t, _FillType>)
599 unchecked_store(_Vp(fill_value), result);
601 auto outIter = result;
603 auto inIter = (
const typename _Vp::value_type*)(&v);
604 auto maskIter = (
const typename _Vp::value_type*)(&mask);
606 for (
int i=0; i<_Np; ++i)
610 *outIter = inIter[i];
620compress_by_mask(compact_mask_tag,
const _V& values,
const typename _V::mask_type& selector,
621 typename _V::value_type fill_value = {})
623 std::array<typename _V::value_type, _V::size> result;
624 std::fill(result.begin(), result.end(), fill_value);
627 for (
int i=0; i<_V::size(); ++i)
631 result[outIdx] = values[i];
643template<mask_type _Mp>
645compress_mask_by_mask(generic_tag,
const _Mp& v,
const _Mp& mask)
648template<mask_type _Mp>
650compress_mask_by_mask(generic_tag,
const _Mp& v,
const _Mp& mask,
bool fill_value)
652 using FillType =
typename _Mp::traits::element_container_type;
653 FillType fill = fill_value ? ~FillType() : FillType();
657template<mask_type _M>
659compress_mask_by_mask(compact_mask_tag,
const _M& v,
const _M& selector) {
661 return compress(-v, selector) != cw<0>;
664template<mask_type _M>
666compress_mask_by_mask(compact_mask_tag,
const _M& v,
const _M& selector,
bool fill_value)
668 auto allFill = _M(fill_value);
688 return compress(v ^ allFill, selector) ^ allFill;
701template<detail::contiguous_sized_range _Range, vec_integral _Idx,
typename... _Flags>
702constexpr auto gather_from(generic_tag,
const _Range& r,
const _Idx& indexes, flags<_Flags...> flags)
704 using _Tp = std::ranges::range_value_t<_Range>;
705 const auto rmax = std::ranges::size(r);
707 if constexpr (contains_flag<flag_unchecked>(flags))
708 checkStaticMemoryBounds(
"gather", indexes, rmax);
710 constexpr int numIndexes = _Idx::size();
715 const auto indexPtr =
reinterpret_cast<const typename _Idx::value_type*
>(&indexes);
716 _Tp result[numIndexes];
718 for (
int i=0; i<numIndexes; ++i)
720 auto idx = indexPtr[i];
721 bool validIndex = (idx >= 0 && std::cmp_less(+idx, rmax));
722 result[i] = contains_flag<flag_unchecked>(flags) || validIndex ? r[idx] : _Tp();
725 return rebind_t<_Tp, _Idx>(result);
737template<detail::contiguous_sized_range _Range, vec_integral _Idx,
typename... _Flags>
738constexpr auto gather_from(generic_tag,
const _Range& r,
739 const typename _Idx::mask_type& mask,
740 const _Idx& indexes, flags<_Flags...> flags)
742 using _Tp = std::ranges::range_value_t<_Range>;
743 const auto rmax = std::ranges::size(r);
745 if constexpr (contains_flag<flag_unchecked>(flags))
746 checkStaticMemoryBounds(
"gather", indexes, rmax, mask);
748 constexpr int numIndexes = _Idx::size();
753 const auto indexPtr =
reinterpret_cast<const typename _Idx::value_type*
>(&indexes);
754 _Tp result[numIndexes];
759 for (
int i=0; i<numIndexes; ++i)
761 auto idx = indexPtr[i];
762 bool validIndex = contains_flag<flag_unchecked>(flags) || (idx >= 0 && std::cmp_less(+idx, rmax));
763 result[i] = validIndex && mb[i] ? r[idx] : _Tp();
766 return rebind_t<_Tp, _Idx>(result);
778template<
typename _Range, vec_type _Vp, vec_integral _Idx,
typename... _Flags>
780scatter_to(generic_tag,
const _Vp& values, _Range&& r,
const _Idx& indexes, flags<_Flags...> flags)
782 const auto rmax = std::ranges::size(r);
783 auto data_ptr = std::ranges::data(r);
785 if constexpr (contains_flag<flag_unchecked>(flags))
786 checkStaticMemoryBounds(
"scatter", indexes, rmax);
788 auto doWrite = [&](
auto i) {
789 const auto idx = indexes[i];
790 const bool isValidIndex = idx >= 0 && std::cmp_less(+idx, rmax);
791 if (contains_flag<flag_unchecked>(flags) || isValidIndex)
792 data_ptr[idx] = values[i];
795 [&]<std::size_t... _Iota>(std::index_sequence<_Iota...>) {
796 (doWrite(_Iota), ...);
797 }(std::make_index_sequence<_Idx::size>());
810template<
typename _Range, vec_type _Vp, vec_integral _Idx,
typename... _Flags>
812scatter_to(generic_tag,
const _Vp& values, _Range&& r,
813 const typename _Idx::mask_type& mask,
814 const _Idx& indexes, flags<_Flags...> flags)
816 const auto rmax = std::ranges::size(r);
817 auto data_ptr = std::ranges::data(r);
819 if constexpr (contains_flag<flag_unchecked>(flags))
820 checkStaticMemoryBounds(
"scatter", indexes, rmax, mask);
822 auto doWrite = [&](
auto i) {
826 const auto idx = indexes[i];
827 const bool isValidIndex = idx >= 0 && std::cmp_less(+idx, rmax);
828 if (contains_flag<flag_unchecked>(flags) || isValidIndex)
829 data_ptr[idx] = values[i];
832 [=]<std::size_t... _Iota>(std::index_sequence<_Iota...>) {
833 (doWrite(_Iota), ...);
834 }(std::make_index_sequence<_Idx::size>());
845template<
typename _Tp,
typename _Abi>
846requires (detail::is_floating_point_v<_Tp>)
847constexpr basic_vec<_Tp, _Abi>
848fma(generic_tag,
const basic_vec<_Tp, _Abi>& x,
const basic_vec<_Tp, _Abi>& y,
const basic_vec<_Tp, _Abi>& acc)
850 #if __has_builtin(__builtin_elementwise_fma)
851 return __builtin_elementwise_fma(x.to_builtin(), y.to_builtin(), acc.to_builtin());
853 return x.to_builtin() * y.to_builtin() + acc.to_builtin();
866template<
typename _Tp,
typename _Abi>
867constexpr basic_vec<_Tp, _Abi>
868fmaddsub(generic_tag,
const basic_vec<_Tp, _Abi>& x,
const basic_vec<_Tp, _Abi>& y,
const basic_vec<_Tp, _Abi>& acc)
872 constexpr auto plusMinus = basic_vec<_Tp, _Abi>([](
auto idx) -> _Tp {
return (idx % 2 == 0) ? -1 : +1; });
873 return fma(x, y, plusMinus * acc);
885template<
typename _Tp,
typename _Abi>
886constexpr basic_vec<_Tp, _Abi>
887fmsubadd(generic_tag,
const basic_vec<_Tp, _Abi>& x,
const basic_vec<_Tp, _Abi>& y,
const basic_vec<_Tp, _Abi>& acc)
891 constexpr auto minusPlus = basic_vec<_Tp, _Abi>([](
auto idx) -> _Tp {
return (idx % 2 == 0) ? +1 : -1; });
892 return fma(x, y, minusPlus * acc);
904template<
typename _Tp,
typename _Abi>
905inline auto fma(generic_tag,
906 const basic_vec<std::complex<_Tp>, _Abi>& x,
907 const basic_vec<std::complex<_Tp>, _Abi>& y,
908 const basic_vec<std::complex<_Tp>, _Abi>& acc)
911 const auto ex = simd_bit_cast<_Tp>(x);
912 const auto ey = simd_bit_cast<_Tp>(y);
915 const auto dupXReal =
permute(ex, [](simd_size_type i) -> simd_size_type {
return (i / 2) * 2; });
916 const auto dupXImag =
permute(ex, [](simd_size_type i) -> simd_size_type {
return i | 1; });
917 const auto swapY =
permute(ey, [](simd_size_type i) -> simd_size_type {
return i ^ 1; });
920 const auto tmp =
fmaddsub(dupXImag, swapY, simd_bit_cast<_Tp>(acc));
921 return simd_bit_cast<std::complex<_Tp>>(
fmaddsub(dupXReal, simd_bit_cast<_Tp>(y), tmp));
930template<
typename _Tp,
typename _Abi>
931constexpr auto conj(generic_tag,
const basic_vec<std::complex<_Tp>, _Abi>& v) {
934 static_assert(detail::is_floating_point_v<_Tp>,
935 "Uses floating-point trick for conjugation - need to specialise for integral later if needed");
938 using _C = container_for_type<_Tp>;
939 const auto msbImag = vec<_C, _Abi::num_elements * 2>([](
auto idx) {
return idx % 2 == 0 ? _C() : std::
rotr(_C(1), 1); });
940 const auto t = simd_bit_cast<_C>(v) ^ msbImag;
942 return simd_bit_cast<std::complex<_Tp>>(t);
955template<
typename _Tp, std::
integral _Sp,
typename _Abi>
956constexpr basic_vec<_Tp, _Abi> rotate_left(generic_tag,
const basic_vec<_Tp, _Abi>& value,
const basic_vec<_Sp, _Abi>& s) {
957 constexpr int numBits =
sizeof(_Tp) * 8;
958 const auto sm = basic_vec<_Tp, _Abi>(s).to_builtin() % numBits;
959 const auto r = (value.to_builtin() << sm) | (value.to_builtin() >> ((numBits - sm) % numBits));
960 return basic_vec<_Tp, _Abi>(sm != 0 ? r : value.to_builtin());
963template<
typename _Tp, std::
integral _Sp,
typename _Abi>
964constexpr basic_vec<_Tp, _Abi> rotate_right(generic_tag,
const basic_vec<_Tp, _Abi>& value,
const basic_vec<_Sp, _Abi>& s) {
965 constexpr int numBits =
sizeof(_Tp) * 8;
966 const auto sm = basic_vec<_Tp, _Abi>(s).to_builtin() % numBits;
967 const auto r = (value.to_builtin() >> sm) | (value.to_builtin() << ((numBits - sm) % numBits));
968 return basic_vec<_Tp, _Abi>(sm != 0 ? r : value.to_builtin());
984template<vec_
unsigned_
integer _Vp, vec_
integral _Sp>
985 requires (
sizeof(
typename _Vp::value_type) ==
sizeof(
typename _Sp::value_type))
986constexpr _Vp fsr(generic_tag,
const _Vp& high,
const _Vp& low,
const _Sp& s) {
987 constexpr std::uint8_t numBits =
sizeof(
typename _Vp::value_type) * 8;
988 const auto sm = _Vp(s) % numBits;
989 const _Vp r = (low >> sm) | (high << ((numBits - sm) % numBits));
990 return select(sm != _Vp(), r, low);
993template<vec_
unsigned_
integer _Vp, vec_
integral _Sp>
994 requires (
sizeof(
typename _Vp::value_type) ==
sizeof(
typename _Sp::value_type))
995constexpr _Vp fsl(generic_tag,
const _Vp& high,
const _Vp& low,
const _Sp& s) {
996 constexpr std::uint8_t numBits =
sizeof(
typename _Vp::value_type) * 8;
997 const auto sm = _Vp(s) % numBits;
998 const _Vp r = (high << sm) | (low >> ((numBits - sm) % numBits));
999 return select(sm != _Vp(), r, high);
1012template<
typename _Tp,
typename _Abi>
1013constexpr basic_vec<_Tp, _Abi>
abs(generic_tag,
const basic_vec<_Tp, _Abi>& v)
1015 if constexpr (detail::is_floating_point_v<_Tp>)
1019 using _C = detail::container_for_type<_Tp>;
1020 constexpr _C msb = _C(~_C()) >> 1;
1021 const auto c = simd_bit_cast<_C>(v);
1022 auto r = basic_vec<_C, _Abi>(c & msb);
1023 return simd_bit_cast<_Tp>(r);
1027 const auto vd = v.to_builtin();
1028 return basic_vec<_Tp, _Abi>(vd < _Tp() ? -vd : vd);
1044template<vec_type _Vp>
1045constexpr _Vp minimum(generic_tag,
const _Vp& lhs,
const _Vp& rhs) {
return select(rhs < lhs, rhs, lhs); }
1047template<vec_type _Vp>
1048constexpr _Vp maximum(generic_tag,
const _Vp& lhs,
const _Vp& rhs) {
return select(lhs < rhs, rhs, lhs); }
1059template<std::
integral _To, vec_
integral _From>
1063 using _FromT =
typename _From::value_type;
1071 if constexpr (std::in_range<_FromT>(std::numeric_limits<_To>::min()))
1072 value =
max(value, _From(
static_cast<_FromT
>(std::numeric_limits<_To>::min())));
1080 if constexpr (std::in_range<_FromT>(std::numeric_limits<_To>::max()))
1081 value =
min(value, _From(
static_cast<_FromT
>(std::numeric_limits<_To>::max())));
1085 return rebind_cast<_To>(value);
1088template<
typename _Vendor,
typename _Tp,
typename _Abi>
1089constexpr basic_vec<_Tp, _Abi>
byteswap(_Vendor,
const basic_vec<_Tp, _Abi>& x)
noexcept {
1096 auto impl = [](
auto piece)
1098 const auto asBytes = simd_bit_cast<std::uint8_t>(piece);
1099 const auto rev =
permute(asBytes, [](simd_size_type idx) -> simd_size_type {
return idx ^ (
sizeof(_Tp) - 1); });
1100 return simd_bit_cast<_Tp>(rev);
1110template<vec_
integral _Vp>
1111constexpr _Vp bitreverse(generic_tag,
const _Vp& x)
noexcept {
1113 auto doBrev = []<vec_type X>(X bits) {
1114 auto asBytes = simd_bit_cast<std::uint8_t>(bits);
1115 using BM =
decltype(asBytes);
1117 constexpr BM m1(cw<0x55u>);
1118 constexpr BM m2(cw<0x33u>);
1119 constexpr BM m3(cw<0x0Fu>);
1121 asBytes = ((asBytes & m1) << 1u) | ((asBytes >> 1u) & m1);
1122 asBytes = ((asBytes & m2) << 2u) | ((asBytes >> 2u) & m2);
1123 asBytes = ((asBytes & m3) << 4u) | ((asBytes >> 4u) & m3);
1125 return byteswap(simd_bit_cast<X>(asBytes));
1144template<
int _Value,
typename _Tp, std::
size_t _Bytes,
typename _Abi>
1145constexpr basic_vec<_Tp, _Abi> mask_to_vec(generic_tag,
const basic_mask<_Bytes, _Abi>& mask)
1147 static_assert(_Value == 1 || _Value == -1,
"Only some values are permitted for mask to vec output");
1148 static_assert(_Bytes ==
sizeof(_Tp),
"Invalid mask or destination - differing sizes");
1150 using _To = basic_vec<_Tp, _Abi>;
1151 using _B =
typename _To::builtin_type;
1153 if constexpr (std::integral<_Tp> && _Value == -1)
1156 else if constexpr (std::integral<_Tp> && _Value == 1)
1161 constexpr auto _vs = _To(_Tp(_Value));
1162 return select(mask, _vs, _To());
1166template<
int _Value,
typename _Tp, std::
size_t _Bytes,
typename _Abi>
1167constexpr basic_vec<_Tp, _Abi> mask_to_vec(compact_mask_tag,
const basic_mask<_Bytes, _Abi>& m)
1169 static_assert(_Value == 1 || _Value == -1,
"Only some values are permitted for mask to vec output");
1170 constexpr auto _vs = basic_vec<_Tp, _Abi>(_Tp(_Value));
1171 return select(m, _vs, basic_vec<_Tp, _Abi>());
1181template<
typename _Tp,
typename _Abi>
1182constexpr auto has_single_bit(generic_tag,
const basic_vec<_Tp, _Abi>& x)
noexcept {
1183 return (x != std::uint8_t(0)) && ((x & (x - std::uint8_t(1))) == std::uint8_t(0));
1191constexpr bool mask_all_of(generic_tag,
const mask_type
auto& mask)
noexcept {
1193 auto processBlock = [&]<mask_type _S>(_S m) {
1194 constexpr std::uint64_t allBits = (std::uint64_t(1) << _S::size) - 1;
1195 allSet &= (allBits == m.to_ullong());
1202template<mask_type _Mp>
1203constexpr bool mask_all_of(compact_mask_tag,
const _Mp& mask)
noexcept {
1207constexpr bool mask_any_of(generic_tag,
const mask_type
auto& mask)
noexcept {
1212 std::uint64_t reducedBits = 0;
1214 return reducedBits != 0;
1217template<std::
size_t _Bytes,
typename _Abi>
1218constexpr bool mask_any_of(compact_mask_tag,
const basic_mask<_Bytes, _Abi>& mask)
noexcept {
1222constexpr bool mask_none_of(generic_tag,
const mask_type
auto& mask)
noexcept {
1227 std::uint64_t reducedBits = 0;
1229 return reducedBits == 0;
1232template<std::
size_t _Bytes,
typename _Abi>
1233constexpr bool mask_none_of(compact_mask_tag,
const basic_mask<_Bytes, _Abi>& mask)
noexcept {
1242constexpr simd_size_type mask_reduce_count(generic_tag,
const mask_type
auto& mask)
noexcept {
1245 simd_size_type numBits = 0;
1250constexpr simd_size_type mask_reduce_count(compact_mask_tag,
const mask_type
auto& mask)
noexcept {
1260constexpr simd_size_type mask_reduce_min_index(generic_tag, mask_type
auto& mask)
noexcept {
1261 simd_size_type lowestSetBit =
mask.
size();
1265 auto processBlock = [&](mask_type
auto m,
auto idx)
1267 auto bits = m.to_ullong();
1269 lowestSetBit = std::min<int>(std::countr_zero(bits) + idx, lowestSetBit);
1273 return lowestSetBit;
1276template<mask_type _Mp>
1277constexpr simd_size_type mask_reduce_min_index(compact_mask_tag,
const _Mp& mask)
noexcept {
1280 static_assert(_Mp::size <= 128,
"This function only handles up to 128 bits");
1282 auto lowResult = std::countr_zero((std::uint64_t)
mask.
to_builtin());
1284 if constexpr (_Mp::size <= 64)
1288 auto highResult = std::countr_zero((std::uint64_t)(
mask.
to_builtin() >> 64));
1289 if (lowResult < 64)
return lowResult;
1290 else return highResult + 64;
1299constexpr simd_size_type mask_reduce_max_index(generic_tag,
const mask_type
auto& mask)
noexcept {
1300 simd_size_type highestSetBit = 0;
1304 auto processBlock = [&](mask_type
auto m,
auto idx) {
1305 auto bits = m.to_ullong();
1307 highestSetBit = std::max<int>((63 - std::countl_zero(bits)) + idx, highestSetBit);
1311 return highestSetBit;
1314template<mask_type _Mp>
1315constexpr simd_size_type mask_reduce_max_index(compact_mask_tag,
const _Mp& mask)
noexcept {
1318 static_assert(_Mp::size <= 128,
"This function only handles up to 128 bits");
1320 auto lowResult = 63 - std::countl_zero((std::uint64_t)
mask.
to_builtin());
1322 if constexpr (_Mp::size <= 64)
1328 return 127 - std::countl_zero(upperBits);
1343template<mask_type _Mp>
1344constexpr auto get_n_bit_mask(generic_tag,
const _Mp&, simd_size_type _n) {
1345 static_assert(_Mp::size <= 256,
"The code below won't work for more than 256 elements");
1346 if constexpr (simd_mask_element_size_v<_Mp> == 1 && _Mp::size >= 128)
1352 auto unsignedIota = iota<typename _Mp::traits::unsigned_vec_for_mask>;
1353 return unsignedIota < std::uint8_t(std::min<std::size_t>(_n, 255));
1355 using _X =
typename _Mp::traits::signed_vec_for_mask;
1356 return iota<_X> < (
typename _X::value_type)std::min(_n, _Mp::size());
1360template<mask_type _M>
1361constexpr auto get_n_bit_mask(compact_mask_tag,
const _M&, simd_size_type _n)
1363 static_assert(_M::size <= 255);
1365 if constexpr (_M::size <= 64)
1367 const auto clamped = std::min(_M::size(), _n);
1368 return _M::from_builtin(
typename _M::builtin_type(clamped == 64 ? ~0ULL : (1ULL << clamped) - 1));
1371 return _M::from_builtin(_M(
true).to_builtin() >> (_M::size - std::min(_n, _M::size())));
1379template<vec_
unsigned_
integral _Vp>
1380constexpr _Vp clz(generic_tag,
const _Vp& x)
noexcept
1381 {
return _Vp([=](
auto i) ->
typename _Vp::value_type {
return std::countl_zero(x[i]); }); }
1387template<vec_
unsigned_
integral _Vp>
1388constexpr _Vp
popcount(generic_tag,
const _Vp& x)
noexcept
1389 {
return _Vp([=](
auto i) ->
typename _Vp::value_type {
return std::popcount(x[i]); }); }
1398template<std::
integral _Tp,
typename _Abi>
1399constexpr basic_vec<_Tp, _Abi>
saturating_add(generic_tag,
const basic_vec<_Tp, _Abi>& lhs,
const basic_vec<_Tp, _Abi>& rhs)
noexcept
1401 #if __has_builtin(__builtin_elementwise_add_sat)
1402 return __builtin_elementwise_add_sat(lhs.to_builtin(), rhs.to_builtin());
1406 if constexpr (std::unsigned_integral<_Tp>)
1409 return select(t < lhs, _Tp(~_Tp()), t);
1414 using asUnsigned = std::make_unsigned_t<_Tp>;
1415 auto ux = simd_bit_cast<asUnsigned>(lhs);
1416 auto uy = simd_bit_cast<asUnsigned>(rhs);
1421 ux = (ux >> (
sizeof(_Tp) * 8 - 1)) + asUnsigned(std::numeric_limits<_Tp>::max());
1423 auto t = simd_bit_cast<_Tp>((ux ^ uy) | ~(ux ^ res));
1424 return simd_bit_cast<_Tp>(
select(t >= _Tp(), ux, res));
1436template <std::
integral _Tp,
typename _Abi>
1437constexpr basic_vec<_Tp, _Abi>
saturating_sub(generic_tag,
const basic_vec<_Tp, _Abi>& lhs,
const basic_vec<_Tp, _Abi>& rhs)
noexcept
1439 #if __has_builtin(__builtin_elementwise_sub_sat)
1440 return __builtin_elementwise_sub_sat(lhs.to_builtin(), rhs.to_builtin());
1444 if constexpr (std::unsigned_integral<_Tp>)
1446 auto diff = lhs - rhs;
1447 return select(diff > lhs, basic_vec<_Tp, _Abi>{}, diff);
1452 using asUnsigned = std::make_unsigned_t<_Tp>;
1453 auto ux = simd_bit_cast<asUnsigned>(lhs);
1454 auto uy = simd_bit_cast<asUnsigned>(rhs);
1459 ux = (ux >> (
sizeof(_Tp) * 8 - 1)) + asUnsigned(std::numeric_limits<_Tp>::max());
1461 auto t = simd_bit_cast<_Tp>((ux ^ uy) & (ux ^ res));
1462 return simd_bit_cast<_Tp>(
select(t < _Tp(), ux, res));
1473template<mask_type _To, mask_type _From>
1474constexpr _To convert_mask(generic_tag,
const _From& v)
1476 constexpr auto fromSize = simd_mask_element_size_v<_From>;
1477 constexpr auto toSize = simd_mask_element_size_v<_To>;
1481 using _ToType = container_for_num_bytes<toSize>;
1483 if constexpr (fromSize == toSize)
1485 else if constexpr (toSize < fromSize)
1487 return _To::from_builtin(rebind_cast<_ToType>(c).to_builtin());
1490 return rebind_cast<_ToType>(c) != _ToType();
1493template<mask_type _To, mask_type _From>
1494constexpr _To convert_mask(compact_mask_tag,
const _From& m) {
return _To::from_builtin(m.to_builtin()); }
1502template<
typename _Result>
1503constexpr auto generate_mask_from_unsigned(generic_tag, std::unsigned_integral
auto mi) {
1505 auto processBlock = [&]<mask_type _Mask>(_Mask, simd_size_type idx) -> _Mask {
1506 using _Mp = container_for_num_bytes<simd_mask_element_size_v<_Mask>>;
1507 using _Vp = vec<_Mp, _Mask::size>;
1509 auto bits = mi >> idx;
1513 if constexpr (
sizeof(_Mp) != 1 || _Result::size <= 8)
1521 return ((_Mp(1) << iota<_Vp>) & _Mp(bits)) != _Vp();
1531 const auto bytesAs64BitVec = dependent_type<_Result, vec<std::uint64_t, 1>>(bits);
1532 auto t = simd_bit_cast<std::uint8_t>(bytesAs64BitVec);
1533 auto expandedBytes = permute<_Mask::size>(t, [](
int idx) {
return idx / 8; });
1534 return ((_Mp(1) << (iota<_Vp> % _Mp(8))) & expandedBytes) != _Vp();
1540 constexpr int numBits = std::min<int>(
sizeof(mi) * 8, _Result::size());
1541 auto r =
chunked_invoke(processBlock, resize_t<numBits, _Result>());
1542 return fit_to_size<_Result::size()>(r);
1545template<mask_type _Result>
1546constexpr _Result generate_mask_from_unsigned(compact_mask_tag, std::unsigned_integral
auto mi) {
1549 return _Result::from_builtin(
typename _Result::builtin_type(mi));
1553template<vec_
floating_po
int _Vp>
1554constexpr _Vp
rcp(generic_tag,
const _Vp& x)
1555 {
return std::uint8_t(1) / x; }
1557template<vec_
floating_po
int _Vp>
1558constexpr _Vp
sqrt(generic_tag,
const _Vp& x) {
1559 using _Tp =
typename _Vp::value_type;
1560 if constexpr (is_fp16_v<_Tp>)
1561 return _Vp([=](
auto i) {
return typename _Vp::value_type(std::sqrt(
float(x[i]))); });
1563 return _Vp([=](
auto i) {
return typename _Vp::value_type(std::sqrt(x[i])); });
1566template<vec_
floating_po
int _Vp>
1567constexpr _Vp
rsqrt(generic_tag,
const _Vp& x)
1568 {
return std::uint8_t(1) /
sqrt(x); }
1570enum class RoundOp{CEIL, FLOOR, TRUNC, ROUND, RINT};
1572template<RoundOp _Op, vec_
floating_po
int _Vp>
1573constexpr _Vp round_op(generic_tag,
const _Vp& v)
1575 auto doOp = [](
auto f) {
1577 case RoundOp::CEIL:
return std::ceil(f);
1578 case RoundOp::FLOOR:
return std::floor(f);
1579 case RoundOp::TRUNC:
return std::trunc(f);
1580 case RoundOp::ROUND:
return std::round(f);
1581 case RoundOp::RINT:
return std::rint(f);
1585 if constexpr (is_fp16_v<typename _Vp::value_type>)
1586 return _Vp([=](
auto i) {
return typename _Vp::value_type(doOp(
float(v[i]))); });
1588 return _Vp([=](
auto i) {
return typename _Vp::value_type(doOp(v[i])); });
constexpr _Vp min(const _Vp &lhs, const _Vp &rhs) noexcept
Compute the smaller of each respective vec element from the two input arguments.
Definition std_algorithm.hpp:34
constexpr _Vp max(const _Vp &lhs, const _Vp &rhs) noexcept
Compute the larger of each respective vec element from the two input arguments.
Definition std_algorithm.hpp:48
constexpr auto select(const _M &m, const _T &if_true, const _U &if_false) noexcept -> decltype(simd_select_impl(m, if_true, if_false))
Select between corresponding elements according to a mask.
Definition std_algorithm.hpp:88
constexpr auto popcount(const _Vp &x) noexcept
Count the one bits in each element.
Definition std_bit.hpp:133
constexpr _Up rotr(const _Up &u, const _Sp &s)
Rotate each unsigned element right by its corresponding offset. Bits rotated off one end reappear at ...
Definition std_bit.hpp:68
constexpr _Vp::mask_type has_single_bit(const _Vp &x) noexcept
Identify elements that are integral powers of two.
Definition std_bit.hpp:146
constexpr _Vp byteswap(const _Vp &v) noexcept
Reverse the bytes within each integral element.
Definition std_bit.hpp:226
constexpr auto abs(const _Vc &value)
Compute the magnitude of each complex element.
Definition std_complex.hpp:55
constexpr _Vc conj(const _Vc &value)
Compute the conjugate of each complex element.
Definition std_complex.hpp:31
constexpr auto vec_as_container(const _Vp &v)
Generate a variable containing the iota sequence (i.e., 0, 1, 2, ... size-1) for the given simd::vec ...
Definition generic_impl.hpp:116
constexpr void unchecked_store(const basic_vec< _Tp, _Abi > &value, _Range &&r, flags< _Flags... > f={})
Unchecked store into a contiguous sized range.
Definition loadstore.hpp:332
constexpr std::bitset< size()> to_bitset() const noexcept
Return a bitset containing the value of each mask element.
Definition mask.hpp:292
basic_mask< sizeof(_Tp), simd_fixed_size_abi< _Np > > mask
Definition core.hpp:383
static constexpr auto size
Return the number of elements contained in the mask.
Definition mask.hpp:148
constexpr detail::deduced_vec_t< _Vp > sqrt(const _Vp &v)
Compute the square root of each element.
Definition std_math.hpp:102
constexpr detail::deduced_vec_t< _Vp > fma(const _Vp &lhs, const _Vp &rhs, const _Vp &acc)
Compute (lhs * rhs) + acc element-wise with a single rounding.
Definition std_math.hpp:111
constexpr _V compress(const _V &v, const typename _V::mask_type &selector)
Compress a basic_vec or basic_mask value by a mask. The elements corresponding to any set mask bits w...
Definition permute.hpp:145
constexpr _V expand(const _V &v, const typename _V::mask_type &selector, const _V &original={})
Expand consecutive input elements into positions selected by a mask.
Definition permute.hpp:214
constexpr auto permute(const _Vp &v, detail::index_generator_function auto &&fn)
Permute the elements in a vec value using a generator function to compute the indexing.
Definition permute.hpp:94
constexpr auto partial_gather_from(_Range &&r, const _Indexes &indexes, flags< _Flags... > f={})
Gather the contents of a contiguous range using the integral indexes supplied in another vec value.
Definition permute.hpp:258
constexpr auto chunked_invoke(_Fn fn, const _Arg0 &arg0, const _Args &... args)
Invoke a callable on corresponding vector or mask chunks.
Definition invoke.hpp:61
constexpr detail::deduced_vec_t< _Vp > fmaddsub(const _Vp &lhs, const _Vp &rhs, const _Vp &acc)
Compute lhs * rhs - acc at even indexes and lhs * rhs + acc at odd indexes. Each expression is comput...
Definition std_math.hpp:136
constexpr detail::deduced_vec_t< _Vp > rcp(const _Vp &v)
Compute the approximate reciprocal of each element.
Definition std_math.hpp:85
constexpr detail::deduced_vec_t< _Vp > fmsubadd(const _Vp &lhs, const _Vp &rhs, const _Vp &acc)
Compute lhs * rhs + acc at even indexes and lhs * rhs - acc at odd indexes. Each expression is comput...
Definition std_math.hpp:147
constexpr detail::deduced_vec_t< _Vp > rsqrt(const _Vp &v)
Compute the approximate reciprocal square root of each element.
Definition std_math.hpp:91
constexpr auto to_builtin() const
Return the builtin mask value directly.
Definition mask.hpp:275
constexpr auto simd_bit_cast(const _From &v) noexcept
A simd-generic version of simd_bit_cast which directly invokes std::bit_cast.
Definition core.hpp:585
constexpr T saturating_sub(T lhs, T rhs) noexcept
Subtract two integers, clamping an unrepresentable mathematical result.
Definition compat.hpp:90
constexpr T saturating_add(T lhs, T rhs) noexcept
Add two integers, clamping an unrepresentable mathematical result.
Definition compat.hpp:68
constexpr To saturating_cast(From value) noexcept
Definition compat.hpp:50
Special type used to indicate that the compress operation will not fill unused values.
Definition generic_impl.hpp:577
Definition target_config.hpp:23