Intel xvec
Experimental C++ simd class library
Loading...
Searching...
No Matches
generic_impl.hpp
1//===----------------------------------------------------------------------===//
2//
3// Copyright (C) 2021 Intel Corporation
4//
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#pragma once
10
11#include <bitset>
12#include <cmath> // std::ceil, std::floor, std::trunc, std::round, std::rint, std::sqrt
13#include <cstddef> // std::size_t
14#include <cstdint> // std::uint8_t, std::uint64_t
15#include <limits> // std::numeric_limits
16#include <type_traits>
17#include <algorithm> // std::min, std::max
18#include <utility> // std::index_sequence
19#include <cstring> // for std::memcpy
20
21#include <xvec/detail/config.hpp>
22#include <xvec/detail/utilities.hpp>
23
24namespace _XVEC_NAMESPACE::simd
25{
26
27inline constexpr simd_size_type zero_element = 9999123; // No specific value other than being big enough to fall outside valid vec indexes.
28inline constexpr simd_size_type uninit_element = 9999124; // No specific value other than being big enough to fall outside valid vec indexes.
29
40template<typename _Tp>
41 requires (vec_type<_Tp> || std::is_arithmetic_v<_Tp>)
42inline constexpr _Tp iota =
43 []{ if constexpr (vec_type<_Tp>) return _Tp([](simd_size_type i) { return typename _Tp::value_type(i); });
44 else return _Tp{}; }();
45
46namespace detail
47{
48
49// :TODO: Move into utilities.
50const auto perm_uninitResize = [](auto idx, auto isize) { return idx < isize ? idx : uninit_element; };
51
52// Provide a way to initialise the values in a vec_impl::vector from a generator.
53template<vec_type _Vp, typename _Gp>
54constexpr auto generate(_Gp generator) {
55 if constexpr (complex_number<typename _Vp::value_type>)
56 {
57 using _Tp = typename _Vp::traits::element_type;
58 return _Vp([=]<simd_size_type... _Idx>(std::integer_sequence<simd_size_type, _Idx...>) {
59 return typename _Vp::traits::builtin_type{_Tp((_Idx % 2) ?
60 generator(size_constant<_Idx / 2>()).imag() :
61 generator(size_constant<_Idx / 2>()).real())...};
62 }(std::make_integer_sequence<simd_size_type, _Vp::size() * 2>()));
63 }
64 else
65 {
66 return _Vp([=]<simd_size_type... _Idx>(std::integer_sequence<simd_size_type, _Idx...>) {
67 return typename _Vp::traits::builtin_type{std::bit_cast<typename _Vp::traits::element_type>(
68 generator(size_constant<_Idx>{}))...};
69 }(std::make_integer_sequence<simd_size_type, _Vp::size()>()));
70 }
71}
72
73template<simd_size_type _NewSize, vec_or_mask_type _Vp>
74constexpr resize_t<_NewSize, _Vp> fit_to_size(const _Vp& v) {
75 if constexpr (_NewSize == _Vp::size()) return v;
76 else return permute<_NewSize>(v, [](simd_size_type i) { return (i < _Vp::size()) ? i : zero_element; });
77}
78
79template<vec_type _Vp, contiguous_sized_range _Range, typename _Flags>
80constexpr void mandates_for_store(const _Vp&, const _Range&, _Flags f)
81{
82 using _Tp = typename _Vp::value_type;
83 using _Rp = std::ranges::range_value_t<_Range>;
84
85 static_assert(detail::contains_flag<detail::flag_convert>(f) ||
86 !detail::needs_explicit_conversion<_Tp, _Rp>,
87 "Store or scatter conversion requires flag_convert");
88}
89
90template<vec_type _Vp, contiguous_sized_range _Range, typename _Flags>
91constexpr void mandates_for_load(const _Range&, const _Vp&, _Flags f)
92{
93 using _Tp = typename _Vp::value_type;
94 using _Rp = std::ranges::range_value_t<_Range>;
95
96 static_assert(detail::vectorizable<_Rp>);
97 static_assert(detail::explicitly_convertible_to<_Rp, _Tp>);
98 static_assert(std::same_as<std::remove_cvref_t<_Vp>, _Vp>);
99
100 // flag_convert is required if and only if the equivalent simd converting
101 // constructor would be explicit.
102 static_assert(detail::contains_flag<detail::flag_convert>(f) ||
103 !detail::needs_explicit_conversion<_Rp, _Tp>,
104 "Load or gather conversion requires flag_convert");
105}
106
115template<vec_type _Vp>
117
119template<mask_type _Mp> constexpr auto vec_as_container(const _Mp& m) { return vec_as_container(-m); }
121
125template<vec_or_mask_type _Vp>
126constexpr typename _Vp::value_type subscript(generic_tag, const _Vp& v, simd_size_type pos) {
127 assert(pos < _Vp::size());
128 auto t = detail::vec_as_container(v).to_builtin()[pos];
129
130 if constexpr (vec_complex<_Vp>)
131 {
132 // std::bit_cast for std::complex in clang doesn't work until 2025.1.0.
133 // Before that version it is necessary to get the complex value from its
134 // individual components.
135 using _E = complex_element_type<typename _Vp::value_type>;
136 auto asCmplxArray = std::bit_cast<std::array<_E, 2>>(t);
137 return std::complex<_E>(asCmplxArray[0], asCmplxArray[1]);
138 }
139 else if constexpr (mask_type<_Vp>)
140 return typename _Vp::value_type(t);
141 else
142 return std::bit_cast<typename _Vp::value_type>(t);
143}
144
145template<mask_type _Mp>
146constexpr bool subscript(compact_mask_tag, const _Mp& v, simd_size_type pos)
147 { return (v.to_register() >> pos) & 1; }
149
152template<mask_type _Mp, typename _Gp>
153constexpr _Mp generate_mask(generic_tag, _Gp generator) {
154 using _Tp = container_for_num_bytes<simd_mask_element_size_v<_Mp>>;
155 using _Vp = typename _Mp::builtin_type;
156
157 auto r = [=]<simd_size_type... _Idx>(std::integer_sequence<simd_size_type, _Idx...>) {
158 return _Vp{_Tp(generator(size_constant<_Idx>()) ? ~_Tp() : _Tp())...};
159 }(std::make_integer_sequence<simd_size_type, _Mp::size()>());
160
161 return _Mp::from_builtin(r);
162}
163
164template<mask_type _Mp, typename _Gp>
165constexpr _Mp generate_mask(compact_mask_tag, _Gp generator) {
166 return _Mp::from_builtin([=]<simd_size_type... _Idx>(std::integer_sequence<simd_size_type, _Idx...>) {
167 return ((typename _Mp::builtin_type(generator(size_constant<_Idx>{})) << _Idx) | ...);
168 } (std::make_integer_sequence<simd_size_type, _Mp::size()>()));
169}
171
172template<mask_type _Mp>
173constexpr std::bitset<_Mp::size>
174mask_to_bitset(generic_tag, const _Mp& mask) noexcept
175{
176 // Very inefficient, but all targets should provide their own version. Note
177 // also that bitsets are not constexpr before C++23.
178 std::bitset<_Mp::size> result;
179 for (int i=0; i<_Mp::size; i++)
180 result[i] = (mask[i] != 0);
181 return result;
182}
183
184constexpr unsigned long long mask_to_ullong(generic_tag, const mask_type auto& mask)
185 { return mask.to_bitset().to_ullong(); }
186
187constexpr unsigned long long mask_to_ullong(compact_mask_tag, const mask_type auto& mask)
188 { return (unsigned long long)mask.to_builtin(); }
189
201template<vec_type _Vp, vec_integral _Ip>
202constexpr resize_t<_Ip::size(), _Vp>
203permute(generic_tag, const _Vp& v, const _Ip& indexes)
204 { return partial_gather_from(std::span(reinterpret_cast<const typename _Vp::value_type*>(&v), _Vp::size()), indexes); }
205
206template<mask_type _Mp, vec_integral _Ip>
207constexpr resize_t<_Ip::size(), _Mp>
208permute(generic_tag, const _Mp& m, const _Ip& indexes)
209 { return resize_t<_Ip::size(), _Mp>::from_builtin(permute(vec_as_container(m), indexes).to_builtin()); }
210
211template<mask_type _Mp, vec_integral _Ip>
212constexpr resize_t<_Ip::size(), _Mp>
213permute(compact_mask_tag, const _Mp& m, const _Ip& indexes)
214 { return permute(-m, indexes) != cw<0>; }
216
224template<vec_type _Vp>
225constexpr _Vp
226select_if_else(generic_tag, const typename _Vp::mask_type& mask, const _Vp& if_true, const _Vp& if_false)
227{
228 using _C = typename _Vp::traits::element_container_type;
229
230 auto tm = mask.to_builtin();
231 auto ta = simd_bit_cast<_C>(if_true).to_builtin();
232 auto tb = simd_bit_cast<_C>(if_false).to_builtin();
233
234 rebind_t<_C, _Vp> r;
235 if (std::is_constant_evaluated())
236 r = (tm & ta) | ((~tm) & tb); // operator? doesn't work while in constexpr.
237 else
238 r = tm ? ta : tb;
239
240 return simd_bit_cast<typename _Vp::value_type>(r);
241}
242
243template<vec_type _Vp>
244constexpr _Vp
245select_if_else(compact_mask_tag, const typename _Vp::mask_type& mask, const _Vp& if_true, const _Vp& if_false)
246 { return _Vp([=](auto i) { return mask[i] ? if_true[i] : if_false[i]; }); }
247
248template<mask_type _Mp>
249constexpr _Mp
250select_if_else(compact_mask_tag, const _Mp& if_mask, const _Mp& if_true, const _Mp& if_false) {
251 auto r = (if_true.to_builtin() & if_mask.to_builtin()) | (if_false.to_builtin() & ~if_mask.to_builtin());
252 return _Mp::from_builtin(r);
253}
254
261template<mask_type _Mp>
262constexpr _Mp
263select_if_else(generic_tag, const _Mp& mask, const _Mp& if_true, const _Mp& if_false) noexcept
264{ return _Mp::from_builtin(mask.to_builtin() ? if_true.to_builtin() : if_false.to_builtin()); }
265
276template<vec_type _Vp, typename _Up, std::size_t _Extent, typename _Flags>
277constexpr void store_masked(generic_tag, const _Vp& value_original, std::span<_Up, _Extent> to,
278 const typename _Vp::mask_type& mask_original, _Flags flags)
279{
280 mandates_for_store(value_original, to, flags);
281
282 auto limit = _Vp::size();
283 if constexpr (contains_flag<flag_unchecked>(flags))
284 checkStaticMemoryBounds<rebind_t<_Up, _Vp>>("store", to, mask_original);
285 else
286 limit = std::min<std::size_t>(to.size(), _Vp::size());
287
288 for (int i=0; i<limit; i++)
289 if (mask_original[i])
290 to[i] = static_cast<_Up>(value_original[i]);
291}
292
293template<vec_type _Vp, typename _Up, std::size_t _Extent, typename _Flags>
294constexpr void store(generic_tag, const _Vp& value_original, std::span<_Up, _Extent> to, _Flags flags)
295{
296 mandates_for_store(value_original, to, flags);
297
298 auto limit = _Vp::size();
299 if constexpr (contains_flag<flag_unchecked>(flags))
300 checkStaticMemoryBounds<rebind_t<_Up, _Vp>>("store", to);
301 else
302 limit = std::min<std::size_t>(to.size(), _Vp::size());
303
304 for (int i=0; i<limit; i++)
305 to[i] = static_cast<_Up>(value_original[i]);
306}
308
317template<vec_type _V, typename _Up, std::size_t _Extent, typename _Flags>
318constexpr _V load(generic_tag, std::span<_Up, _Extent> from, _Flags flags)
319{
320 mandates_for_load(from, _V{}, flags);
321
322 using SourceType = rebind_t<_Up, _V>;
323
324 // :TODO: Make use of the fixed size where permitted.
325 std::remove_const_t<typename SourceType::builtin_type> result = {};
326 if constexpr (contains_flag<flag_unchecked>(flags))
327 {
328 checkStaticMemoryBounds<SourceType>("load", from);
329 if (std::is_constant_evaluated())
330 return _V([=](auto i) -> typename _V::value_type { return from[i]; });
331 else
332 {
333 memcpy(&result, from.data(), sizeof(_Up) * _V::size);
334 return _V(SourceType(result));
335 }
336 }
337 else
338 {
339 // Use a runtime check to ensure no overrun for smaller sources.
340 auto numElementsToLoad = std::min<std::size_t>(_V::size, from.size());
341
342 if (std::is_constant_evaluated())
343 return _V([=](auto i) -> typename _V::value_type { return i < numElementsToLoad ? from[i] : _Up(); });
344 else
345 {
346 __builtin_memcpy(&result, from.data(), sizeof(_Up) * numElementsToLoad);
347 return _V(SourceType(result));
348 }
349 }
350
351}
352
362template<vec_type _V, typename _Up, std::size_t _Extent, typename _Flags>
363constexpr _V load_masked(generic_tag, std::span<_Up, _Extent> from, const typename _V::mask_type& m, _Flags flags)
364{
365 mandates_for_load(from, _V{}, flags);
366
367 using SourceType = vec<_Up, _V::size>;
368
369 std::remove_const_t<typename SourceType::builtin_type> result = {};
370 if constexpr (contains_flag<flag_unchecked>(flags))
371 {
372 checkStaticMemoryBounds<SourceType>("load", from, m);
373 if (std::is_constant_evaluated())
374 return select(m, _V([=](auto i) -> typename _V::value_type { return from[i]; }), _V());
375 else
376 {
377 // Only the elements within the span may be read; the bounds check above
378 // permits a span smaller than the vector as long as the mask excludes the rest.
379 const auto numElementsToLoad = std::min<std::size_t>(_V::size, from.size());
380 memcpy(&result, from.data(), sizeof(_Up) * numElementsToLoad);
381 return select(m, _V(SourceType(result)), _V());
382 }
383 }
384 else
385 {
386 // Use a runtime check to ensure no overrun for smaller sources.
387 auto numElementsToLoad = std::min<std::size_t>(_V::size, from.size());
388 if (std::is_constant_evaluated())
389 return select(m, _V([=](auto i) -> typename _V::value_type { return i < numElementsToLoad ? from[i] : _Up(); }), _V());
390 else
391 {
392 memcpy(&result, from.data(), sizeof(_Up) * numElementsToLoad);
393 return select(m, _V(SourceType(result)), _V());
394 }
395 }
396
397}
398
402template<simd_size_type _OutSize, vec_or_mask_type _Vp, typename _Gp>
403constexpr resize_t<_OutSize, _Vp> permute_pair(const _Vp& v0, const _Vp& v1, _Gp) {
404 auto c0 = vec_as_container(v0);
405 auto c1 = vec_as_container(v1);
406
407 using _C = typename decltype(c0)::value_type;
408 constexpr int _Np = _Vp::size;
409
410 // Tweak the numbers to work on gcc and clang uniformly.
411 auto getIndex = [](auto idx) {
412 // Compute the required index to use. Note that this will be called for
413 // every storage element, but any elements past the end of the actual
414 // output storage will be left as undefined. This is required since gcc always
415 // initialises the entire simd::vec, but the generator function shouldn't be
416 // called for values which don't map to actual elements.
417 if constexpr (idx >= _OutSize)
418 return size_constant<-1>();
419 else
420 {
421 constexpr int outIdx = simd_size_type(_Gp{}(idx));
422
423 // Adjust the index to reference the second vector where necessary. Clang
424 // assumes that two vectors of size N will have indexes [0..2N), but gcc may
425 // insert padding.
426 constexpr int paddingAdjustedIdx = outIdx < _Np ? outIdx : int(outIdx - _Np) + int(vec_traits<target_tag, _C, _Np>::_numStorageElements);
427
428 return size_constant<paddingAdjustedIdx>();
429 }
430 };
431
432 using OutTraits = vec_traits<target_tag, _C, _OutSize>;
433 auto r = [=]<std::size_t... _Idx>(std::index_sequence<_Idx...>) -> typename OutTraits::builtin_type {
434 return __builtin_shufflevector(c0.to_builtin(), c1.to_builtin(), (getIndex(std::integral_constant<int, _Idx>()).value)...);
435 }(std::make_index_sequence<OutTraits::_numStorageElements>());
436
437 if constexpr(vec_type<_Vp>)
438 return simd_bit_cast<typename _Vp::value_type>(vec<_C, _OutSize>(r));
439 else
440 // Mask type goes directly from container to mask without a need to bit cast.
441 return resize_t<_OutSize, _Vp>::from_builtin(r);
442}
443
452
453template<simd_size_type _OutSize, vec_or_mask_type _Vp, typename _Gp>
454constexpr resize_t<_OutSize, _Vp>
455generated_permute(generic_tag, const _Vp& v, _Gp) {
456 constexpr simd_size_type _Np = _Vp::size;
457
458 auto c0 = vec_as_container(v);
459 using _C = typename decltype(c0)::value_type;
460
461 // The second value in the shufflevector arguments is zero, so return the
462 // first index of that as the zero index. Note that GCC stores values
463 // differently to clang (power-of-2) so return the actual index of the
464 // storage, not the apparent type.
465 constexpr auto zeroIndex = vec_traits<target_tag, _C, _Np>::_numStorageElements;
466
467 auto getIndex = []<auto idx>() {
468 // Do not invoke the generator for padding elements in the output storage.
469 if constexpr (idx >= _OutSize) {
470 return size_constant<-1>{};
471 } else {
472 // Get the generated index, passing the input size when supported.
473 constexpr simd_size_type gi = [] {
474 if constexpr (index_generator_function_with_size<_Gp>)
475 return _Gp{}(size_constant<idx>{}, _Np);
476 else
477 return _Gp{}(size_constant<idx>{});
478 }();
479
480 if constexpr (gi == zero_element)
481 return size_constant<zeroIndex>{};
482 else if constexpr (gi == uninit_element)
483 // Mark as unknown
484 return size_constant<-1>{};
485 else
486 {
487 static_assert(gi >= 0 && gi < _Np, "Computed index isn't in valid input range");
488 return size_constant<gi>{};
489 }
490 }
491 };
492
493 using OutTraits = vec_traits<target_tag, _C, _OutSize>;
494
495 typename OutTraits::builtin_type r;
496 constexpr auto emptyBuiltin = decltype(c0.to_builtin()){};
497 if (std::is_constant_evaluated())
498 {
499 // Nasty code to work around differences in gcc and llvm. In a constexpr
500 // context gcc is happy with an index of -1 (meaning uninitialised), but
501 // clang is not. But putting a work-around inside the lambda to enable clang
502 // to detect and remove -1 in a constexpr context then breaks gcc's
503 // constexpr evaluation. The `fix` is to detect constexpr context outside
504 // the loop, so that the inner-most part of the shufflevector doesn't have
505 // to deal with is_constnt_evaulated directly, but the downside is two
506 // slightly different copies of the lambda.
507 r = [=]<std::size_t... _Idx>(std::index_sequence<_Idx...>) -> typename OutTraits::builtin_type {
508 auto fixUninit = [](auto idx) { return idx < 0 ? zeroIndex : idx; };
509 return __builtin_shufflevector(c0.to_builtin(), emptyBuiltin, fixUninit(getIndex.template operator()<_Idx>().value)...);
510 }(std::make_index_sequence<OutTraits::_numStorageElements>());
511 } else {
512 // As above, but without the -1 workaround.
513 r = [=]<std::size_t... _Idx>(std::index_sequence<_Idx...>) -> typename OutTraits::builtin_type {
514 return __builtin_shufflevector(c0.to_builtin(), emptyBuiltin, (getIndex.template operator()<_Idx>().value)...);
515 }(std::make_index_sequence<OutTraits::_numStorageElements>());
516 }
517
518 if constexpr (vec_type<_Vp>)
519 return simd_bit_cast<typename _Vp::value_type>(vec<_C, _OutSize>(r));
520 else
521 // Mask type goes directly from container to mask without a bit cast.
522 return resize_t<_OutSize, _Vp>::from_builtin(r);
523}
524
525template<simd_size_type _OutSize, mask_type _Mp, typename _Gp>
526constexpr auto
527generated_permute(compact_mask_tag, const _Mp& m, _Gp)
528 { return permute<_OutSize>(-m, _Gp{}) != cw<0>; }
529
540template<vec_type _Vp>
541constexpr _Vp
542expand_by_mask(generic_tag, const _Vp& v, const typename _Vp::mask_type& mask, const _Vp& original)
543{
544 typename _Vp::value_type result[_Vp::size];
545 unchecked_store(original, result);
546
547 auto inIter = (const typename _Vp::value_type*)(&v);
548
549 for (int i=0; i<_Vp::size; ++i)
550 {
551 if (mask[i])
552 {
553 result[i] = *inIter;
554 inIter++;
555 }
556 }
557
558 return _Vp(result);
559}
560
565template<mask_type _Mp>
566constexpr _Mp
567expand_mask_by_mask(generic_tag, const _Mp& value, const _Mp& mask, const _Mp& original = {})
568 { return _Mp::from_builtin(expand(vec_as_container(value), mask, vec_as_container(original)).to_builtin()); }
569
570template<mask_type _Mp>
571constexpr _Mp
572expand_mask_by_mask(compact_mask_tag, const _Mp& v, const _Mp& selector, const _Mp& original = {})
573 { return expand(-v, selector, -original) != cw<0>; }
575
578
588template<vec_type _Vp, typename _FillType = compress_fill_uninitialized_t>
589constexpr _Vp
590compress_by_mask(generic_tag, const _Vp& v, const typename _Vp::mask_type& mask, _FillType fill_value = {})
591{
592 constexpr int _Np = _Vp::size;
593 typename _Vp::value_type result[_Np]; // The output is deliberately left undefined to avoid users thinking that they can rely on
594 // contents of the result which are not explicitly written.
595
596 // If a fill value is supplied then use that. Typically, filling in unused
597 // values costs cycles so prefer to avoid this is possible.
598 if constexpr (!std::same_as<compress_fill_uninitialized_t, _FillType>)
599 unchecked_store(_Vp(fill_value), result);
600
601 auto outIter = result;
602
603 auto inIter = (const typename _Vp::value_type*)(&v);
604 auto maskIter = (const typename _Vp::value_type*)(&mask);
605
606 for (int i=0; i<_Np; ++i)
607 {
608 if (maskIter[i])
609 {
610 *outIter = inIter[i];
611 outIter++;
612 }
613 }
614
615 return _Vp(result);
616}
617
618template<vec_type _V>
619constexpr _V
620compress_by_mask(compact_mask_tag, const _V& values, const typename _V::mask_type& selector,
621 typename _V::value_type fill_value = {})
622{
623 std::array<typename _V::value_type, _V::size> result;
624 std::fill(result.begin(), result.end(), fill_value);
625
626 auto outIdx = 0;
627 for (int i=0; i<_V::size(); ++i)
628 {
629 if (selector[i])
630 {
631 result[outIdx] = values[i];
632 outIdx++;
633 }
634 }
635
636 return _V(result);
637}
638
643template<mask_type _Mp>
644constexpr _Mp
645compress_mask_by_mask(generic_tag, const _Mp& v, const _Mp& mask)
646 { return _Mp::from_builtin(compress(vec_as_container(v), mask).to_builtin()); }
647
648template<mask_type _Mp>
649constexpr _Mp
650compress_mask_by_mask(generic_tag, const _Mp& v, const _Mp& mask, bool fill_value)
651{
652 using FillType = typename _Mp::traits::element_container_type;
653 FillType fill = fill_value ? ~FillType() : FillType();
654 return _Mp::from_builtin(compress(vec_as_container(v), mask, fill).to_builtin());
655}
656
657template<mask_type _M>
658constexpr _M
659compress_mask_by_mask(compact_mask_tag, const _M& v, const _M& selector) {
660 // Not an efficient implementation, so targets with compact masks should provide their own version of this function.
661 return compress(-v, selector) != cw<0>;
662}
663
664template<mask_type _M>
665constexpr _M
666compress_mask_by_mask(compact_mask_tag, const _M& v, const _M& selector, bool fill_value)
667{
668 auto allFill = _M(fill_value);
669
670 // Handling the fill is done by exploiting the knowledge that the normal
671 // compress will zero out the excess bits. If the fill value must be
672 // true, then the excess bits need to be set which in turn means we want to
673 // zero fill and then flip the bits. Consequently the input bits must also be
674 // flipped, so that they can be flipped again to set them. An example
675 // illustrates this:
676 //
677 // mask: 101010
678 // input: 110010
679 // flipped Input: 001101
680 // compressed: 000011
681 // flipped output: 111100
682 //
683 // The input and final output only need to be flipped on a true fill value, so
684 // use an xor toggle.
685 //
686 // This code is very efficient because it only requires an extra two very
687 // cheap XOR operations over the non-fill version.
688 return compress(v ^ allFill, selector) ^ allFill;
689}
691
693// :TODO: :COMPILER: Expose the gather llvm primitive and use it directly?
701template<detail::contiguous_sized_range _Range, vec_integral _Idx, typename... _Flags>
702constexpr auto gather_from(generic_tag, const _Range& r, const _Idx& indexes, flags<_Flags...> flags)
703{
704 using _Tp = std::ranges::range_value_t<_Range>;
705 const auto rmax = std::ranges::size(r);
706
707 if constexpr (contains_flag<flag_unchecked>(flags))
708 checkStaticMemoryBounds("gather", indexes, rmax);
709
710 constexpr int numIndexes = _Idx::size();
711
712 // Put the indexes and the result into memory, as this generates better code
713 // than trying to pull out individual elements of a simd register to pass to a
714 // separate load instruction, and to put the results back into a register.
715 const auto indexPtr = reinterpret_cast<const typename _Idx::value_type*>(&indexes);
716 _Tp result[numIndexes];
717
718 for (int i=0; i<numIndexes; ++i)
719 {
720 auto idx = indexPtr[i];
721 bool validIndex = (idx >= 0 && std::cmp_less(+idx, rmax));
722 result[i] = contains_flag<flag_unchecked>(flags) || validIndex ? r[idx] : _Tp();
723 }
724
725 return rebind_t<_Tp, _Idx>(result);
726}
727
737template<detail::contiguous_sized_range _Range, vec_integral _Idx, typename... _Flags>
738constexpr auto gather_from(generic_tag, const _Range& r,
739 const typename _Idx::mask_type& mask,
740 const _Idx& indexes, flags<_Flags...> flags)
741{
742 using _Tp = std::ranges::range_value_t<_Range>;
743 const auto rmax = std::ranges::size(r);
744
745 if constexpr (contains_flag<flag_unchecked>(flags))
746 checkStaticMemoryBounds("gather", indexes, rmax, mask);
747
748 constexpr int numIndexes = _Idx::size();
749
750 // Put the indexes and the result into memory, as this generates better code
751 // than trying to pull out individual elements of a simd register to pass to a
752 // separate load instruction, and to put the results back into a register.
753 const auto indexPtr = reinterpret_cast<const typename _Idx::value_type*>(&indexes);
754 _Tp result[numIndexes];
755
756 // Need simple mask which can be queried by index.
757 const auto mb = mask.to_bitset();
758
759 for (int i=0; i<numIndexes; ++i)
760 {
761 auto idx = indexPtr[i];
762 bool validIndex = contains_flag<flag_unchecked>(flags) || (idx >= 0 && std::cmp_less(+idx, rmax));
763 result[i] = validIndex && mb[i] ? r[idx] : _Tp();
764 }
765
766 return rebind_t<_Tp, _Idx>(result);
767}
768
778template<typename _Range, vec_type _Vp, vec_integral _Idx, typename... _Flags>
779constexpr void
780scatter_to(generic_tag, const _Vp& values, _Range&& r, const _Idx& indexes, flags<_Flags...> flags)
781{
782 const auto rmax = std::ranges::size(r);
783 auto data_ptr = std::ranges::data(r);
784
785 if constexpr (contains_flag<flag_unchecked>(flags))
786 checkStaticMemoryBounds("scatter", indexes, rmax);
787
788 auto doWrite = [&](auto i) {
789 const auto idx = indexes[i];
790 const bool isValidIndex = idx >= 0 && std::cmp_less(+idx, rmax);
791 if (contains_flag<flag_unchecked>(flags) || isValidIndex)
792 data_ptr[idx] = values[i];
793 };
794
795 [&]<std::size_t... _Iota>(std::index_sequence<_Iota...>) {
796 (doWrite(_Iota), ...);
797 }(std::make_index_sequence<_Idx::size>());
798}
799
810template<typename _Range, vec_type _Vp, vec_integral _Idx, typename... _Flags>
811constexpr void
812scatter_to(generic_tag, const _Vp& values, _Range&& r,
813 const typename _Idx::mask_type& mask,
814 const _Idx& indexes, flags<_Flags...> flags)
815{
816 const auto rmax = std::ranges::size(r);
817 auto data_ptr = std::ranges::data(r);
818
819 if constexpr (contains_flag<flag_unchecked>(flags))
820 checkStaticMemoryBounds("scatter", indexes, rmax, mask);
821
822 auto doWrite = [&](auto i) {
823 if (!mask[i])
824 return;
825
826 const auto idx = indexes[i];
827 const bool isValidIndex = idx >= 0 && std::cmp_less(+idx, rmax);
828 if (contains_flag<flag_unchecked>(flags) || isValidIndex)
829 data_ptr[idx] = values[i];
830 };
831
832 [=]<std::size_t... _Iota>(std::index_sequence<_Iota...>) {
833 (doWrite(_Iota), ...);
834 }(std::make_index_sequence<_Idx::size>());
835}
836
845template<typename _Tp, typename _Abi>
846requires (detail::is_floating_point_v<_Tp>)
847constexpr basic_vec<_Tp, _Abi>
848fma(generic_tag, const basic_vec<_Tp, _Abi>& x, const basic_vec<_Tp, _Abi>& y, const basic_vec<_Tp, _Abi>& acc)
849{
850 #if __has_builtin(__builtin_elementwise_fma)
851 return __builtin_elementwise_fma(x.to_builtin(), y.to_builtin(), acc.to_builtin());
852 #else
853 return x.to_builtin() * y.to_builtin() + acc.to_builtin();
854 #endif
855}
856
866template<typename _Tp, typename _Abi>
867constexpr basic_vec<_Tp, _Abi>
868fmaddsub(generic_tag, const basic_vec<_Tp, _Abi>& x, const basic_vec<_Tp, _Abi>& y, const basic_vec<_Tp, _Abi>& acc)
869{
870 // Not the most efficient way to do this but it is meant to be overriden by
871 // target-specific improvements.
872 constexpr auto plusMinus = basic_vec<_Tp, _Abi>([](auto idx) -> _Tp { return (idx % 2 == 0) ? -1 : +1; });
873 return fma(x, y, plusMinus * acc);
874}
875
885template<typename _Tp, typename _Abi>
886constexpr basic_vec<_Tp, _Abi>
887fmsubadd(generic_tag, const basic_vec<_Tp, _Abi>& x, const basic_vec<_Tp, _Abi>& y, const basic_vec<_Tp, _Abi>& acc)
888{
889 // Not the most efficient way to do this but it is meant to be overriden by
890 // target-specific improvements.
891 constexpr auto minusPlus = basic_vec<_Tp, _Abi>([](auto idx) -> _Tp { return (idx % 2 == 0) ? +1 : -1; });
892 return fma(x, y, minusPlus * acc);
893}
894
904template<typename _Tp, typename _Abi>
905inline auto fma(generic_tag,
906 const basic_vec<std::complex<_Tp>, _Abi>& x,
907 const basic_vec<std::complex<_Tp>, _Abi>& y,
908 const basic_vec<std::complex<_Tp>, _Abi>& acc)
909{
910 // Convert incoming arguments into raw elements.
911 const auto ex = simd_bit_cast<_Tp>(x);
912 const auto ey = simd_bit_cast<_Tp>(y);
913
914 // Generate real/imag duplicates and swaps
915 const auto dupXReal = permute(ex, [](simd_size_type i) -> simd_size_type { return (i / 2) * 2; });
916 const auto dupXImag = permute(ex, [](simd_size_type i) -> simd_size_type { return i | 1; });
917 const auto swapY = permute(ey, [](simd_size_type i) -> simd_size_type { return i ^ 1; });
918
919 // Do the actual maths.
920 const auto tmp = fmaddsub(dupXImag, swapY, simd_bit_cast<_Tp>(acc));
921 return simd_bit_cast<std::complex<_Tp>>(fmaddsub(dupXReal, simd_bit_cast<_Tp>(y), tmp));
922}
923
930template<typename _Tp, typename _Abi>
931constexpr auto conj(generic_tag, const basic_vec<std::complex<_Tp>, _Abi>& v) {
932 // Synthesise a conjugation by setting the MSB of the complex value's container, and use that
933 // to flip the imaginary sign bit.
934 static_assert(detail::is_floating_point_v<_Tp>,
935 "Uses floating-point trick for conjugation - need to specialise for integral later if needed");
936
937 // XOR the floating-point value with the sign bit to flip the sign bit of the imaginaries.
938 using _C = container_for_type<_Tp>;
939 const auto msbImag = vec<_C, _Abi::num_elements * 2>([](auto idx) { return idx % 2 == 0 ? _C() : std::rotr(_C(1), 1); });
940 const auto t = simd_bit_cast<_C>(v) ^ msbImag;
941
942 return simd_bit_cast<std::complex<_Tp>>(t);
943}
944
955template<typename _Tp, std::integral _Sp, typename _Abi>
956constexpr basic_vec<_Tp, _Abi> rotate_left(generic_tag, const basic_vec<_Tp, _Abi>& value, const basic_vec<_Sp, _Abi>& s) {
957 constexpr int numBits = sizeof(_Tp) * 8;
958 const auto sm = basic_vec<_Tp, _Abi>(s).to_builtin() % numBits;
959 const auto r = (value.to_builtin() << sm) | (value.to_builtin() >> ((numBits - sm) % numBits));
960 return basic_vec<_Tp, _Abi>(sm != 0 ? r : value.to_builtin());
961}
962
963template<typename _Tp, std::integral _Sp, typename _Abi>
964constexpr basic_vec<_Tp, _Abi> rotate_right(generic_tag, const basic_vec<_Tp, _Abi>& value, const basic_vec<_Sp, _Abi>& s) {
965 constexpr int numBits = sizeof(_Tp) * 8;
966 const auto sm = basic_vec<_Tp, _Abi>(s).to_builtin() % numBits;
967 const auto r = (value.to_builtin() >> sm) | (value.to_builtin() << ((numBits - sm) % numBits));
968 return basic_vec<_Tp, _Abi>(sm != 0 ? r : value.to_builtin());
969}
971
984template<vec_unsigned_integer _Vp, vec_integral _Sp>
985 requires (sizeof(typename _Vp::value_type) == sizeof(typename _Sp::value_type))
986constexpr _Vp fsr(generic_tag, const _Vp& high, const _Vp& low, const _Sp& s) {
987 constexpr std::uint8_t numBits = sizeof(typename _Vp::value_type) * 8;
988 const auto sm = _Vp(s) % numBits;
989 const _Vp r = (low >> sm) | (high << ((numBits - sm) % numBits));
990 return select(sm != _Vp(), r, low);
991}
992
993template<vec_unsigned_integer _Vp, vec_integral _Sp>
994 requires (sizeof(typename _Vp::value_type) == sizeof(typename _Sp::value_type))
995constexpr _Vp fsl(generic_tag, const _Vp& high, const _Vp& low, const _Sp& s) {
996 constexpr std::uint8_t numBits = sizeof(typename _Vp::value_type) * 8;
997 const auto sm = _Vp(s) % numBits;
998 const _Vp r = (high << sm) | (low >> ((numBits - sm) % numBits));
999 return select(sm != _Vp(), r, high);
1000}
1002
1012template<typename _Tp, typename _Abi>
1013constexpr basic_vec<_Tp, _Abi> abs(generic_tag, const basic_vec<_Tp, _Abi>& v)
1014{
1015 if constexpr (detail::is_floating_point_v<_Tp>)
1016 {
1017 // Mask off the sign bit for floating-point. This is done explicitly so that FP16 emulation gets a fast
1018 // implementation. Otherwise, the conditional version below generate awful code during FP16 emulation.
1019 using _C = detail::container_for_type<_Tp>;
1020 constexpr _C msb = _C(~_C()) >> 1;
1021 const auto c = simd_bit_cast<_C>(v);
1022 auto r = basic_vec<_C, _Abi>(c & msb);
1023 return simd_bit_cast<_Tp>(r);
1024 }
1025 else
1026 {
1027 const auto vd = v.to_builtin();
1028 return basic_vec<_Tp, _Abi>(vd < _Tp() ? -vd : vd);
1029 }
1030}
1032
1044template<vec_type _Vp>
1045constexpr _Vp minimum(generic_tag, const _Vp& lhs, const _Vp& rhs) { return select(rhs < lhs, rhs, lhs); }
1046
1047template<vec_type _Vp>
1048constexpr _Vp maximum(generic_tag, const _Vp& lhs, const _Vp& rhs) { return select(lhs < rhs, rhs, lhs); }
1050
1059template<std::integral _To, vec_integral _From>
1060constexpr auto
1061saturating_cast(generic_tag, const _From& input) noexcept
1062{
1063 using _FromT = typename _From::value_type;
1064 auto value = input;
1065
1066 // Clamp to the destination's minimum only when that bound is representable by
1067 // the source element type.Example: int8_t -> uint8_t. The destination
1068 // minimum, 0, fits in int8_t, so negative source values are clamped to 0.
1069 // Conversely, for uint8_t -> int16_t, -32768 does not fit in uint8_t and is
1070 // below every possible source value, so no lower clamp is needed.
1071 if constexpr (std::in_range<_FromT>(std::numeric_limits<_To>::min()))
1072 value = max(value, _From(static_cast<_FromT>(std::numeric_limits<_To>::min())));
1073
1074 // Clamp to the destination's maximum only when that bound is representable by
1075 // the source element type. For example: uint8_t -> int8_t. The destination
1076 // maximum, 127, fits in uint8_t, so source values greater than 127 are
1077 // clamped to 127. Conversely, for int8_t -> uint8_t, 255 does not fit in
1078 // int8_t and is above every possible source value, so no upper clamp is
1079 // needed.
1080 if constexpr (std::in_range<_FromT>(std::numeric_limits<_To>::max()))
1081 value = min(value, _From(static_cast<_FromT>(std::numeric_limits<_To>::max())));
1082
1083 // After the necessary clamps, every element is representable in _To, making
1084 // the element-wise conversion non-saturating and safe.
1085 return rebind_cast<_To>(value);
1086}
1087
1088template<typename _Vendor, typename _Tp, typename _Abi>
1089constexpr basic_vec<_Tp, _Abi> byteswap(_Vendor, const basic_vec<_Tp, _Abi>& x) noexcept {
1090
1091 // Break the incoming value down into register-sized pieces and reverse the
1092 // bytes in each piece. This is necessary because the alternative - to reverse
1093 // the bytes across the entire value of `x' in one permute - creates a
1094 // byte-element vec which could overflow gcc's limits (no more than 256
1095 // elements).
1096 auto impl = [](auto piece)
1097 {
1098 const auto asBytes = simd_bit_cast<std::uint8_t>(piece);
1099 const auto rev = permute(asBytes, [](simd_size_type idx) -> simd_size_type { return idx ^ (sizeof(_Tp) - 1); });
1100 return simd_bit_cast<_Tp>(rev);
1101 };
1102
1103 return chunked_invoke(impl, x);
1104}
1105
1110template<vec_integral _Vp> // :TODO: Should be unsigned_integral once bitswap is removed.
1111constexpr _Vp bitreverse(generic_tag, const _Vp& x) noexcept {
1112
1113 auto doBrev = []<vec_type X>(X bits) {
1114 auto asBytes = simd_bit_cast<std::uint8_t>(bits);
1115 using BM = decltype(asBytes);
1116
1117 constexpr BM m1(cw<0x55u>);
1118 constexpr BM m2(cw<0x33u>);
1119 constexpr BM m3(cw<0x0Fu>);
1120
1121 asBytes = ((asBytes & m1) << 1u) | ((asBytes >> 1u) & m1);
1122 asBytes = ((asBytes & m2) << 2u) | ((asBytes >> 2u) & m2);
1123 asBytes = ((asBytes & m3) << 4u) | ((asBytes >> 4u) & m3);
1124
1125 return byteswap(simd_bit_cast<X>(asBytes));
1126 };
1127
1128 // Break it into small pieces to avoid creating simd objects which have too
1129 // many elements (e.g., a vec with 128 64-bit elements would get mapped to a
1130 // byte-wise permute with 1024 elements, which gcc doesn't like).
1131 return chunked_invoke(doBrev, x);
1132}
1133
1144template<int _Value, typename _Tp, std::size_t _Bytes, typename _Abi>
1145constexpr basic_vec<_Tp, _Abi> mask_to_vec(generic_tag, const basic_mask<_Bytes, _Abi>& mask)
1146{
1147 static_assert(_Value == 1 || _Value == -1, "Only some values are permitted for mask to vec output");
1148 static_assert(_Bytes == sizeof(_Tp), "Invalid mask or destination - differing sizes");
1149
1150 using _To = basic_vec<_Tp, _Abi>;
1151 using _B = typename _To::builtin_type;
1152
1153 if constexpr (std::integral<_Tp> && _Value == -1)
1154 // The mask is already entirely 0 or 1 bits, so nothing more to do
1155 return _To(_B(mask.to_builtin()));
1156 else if constexpr (std::integral<_Tp> && _Value == 1)
1157 // The mask is entirely 0 or 1 bits so a logical shift will convert the MSB into an LSB.
1158 return _To(_B(mask.to_builtin() >> (sizeof(_Tp) * 8 - 1)));
1159 else
1160 {
1161 constexpr auto _vs = _To(_Tp(_Value));
1162 return select(mask, _vs, _To());
1163 }
1164}
1165
1166template<int _Value, typename _Tp, std::size_t _Bytes, typename _Abi>
1167constexpr basic_vec<_Tp, _Abi> mask_to_vec(compact_mask_tag, const basic_mask<_Bytes, _Abi>& m)
1168{
1169 static_assert(_Value == 1 || _Value == -1, "Only some values are permitted for mask to vec output");
1170 constexpr auto _vs = basic_vec<_Tp, _Abi>(_Tp(_Value));
1171 return select(m, _vs, basic_vec<_Tp, _Abi>());
1172}
1174
1181template<typename _Tp, typename _Abi>
1182constexpr auto has_single_bit(generic_tag, const basic_vec<_Tp, _Abi>& x) noexcept {
1183 return (x != std::uint8_t(0)) && ((x & (x - std::uint8_t(1))) == std::uint8_t(0));
1184}
1185
1191constexpr bool mask_all_of(generic_tag, const mask_type auto& mask) noexcept {
1192 bool allSet = true; // Assume until otherwise
1193 auto processBlock = [&]<mask_type _S>(_S m) {
1194 constexpr std::uint64_t allBits = (std::uint64_t(1) << _S::size) - 1;
1195 allSet &= (allBits == m.to_ullong());
1196 };
1197
1198 chunked_invoke(processBlock, mask);
1199 return allSet;
1200}
1201
1202template<mask_type _Mp>
1203constexpr bool mask_all_of(compact_mask_tag, const _Mp& mask) noexcept {
1204 return mask.to_builtin() == _Mp(true).to_builtin();
1205}
1206
1207constexpr bool mask_any_of(generic_tag, const mask_type auto& mask) noexcept {
1208 // Note that real reduction (divide-and-conquor) would be faster, but only for
1209 // really big masks. Note also that it would be more efficient to OR the
1210 // values together as vecs and call to_ullong at the end, but the
1211 // compiler (clang/oneAPI) figures this out for itself anyway.
1212 std::uint64_t reducedBits = 0;
1213 chunked_invoke([&](auto m) { reducedBits |= m.to_ullong(); }, mask);
1214 return reducedBits != 0;
1215}
1216
1217template<std::size_t _Bytes, typename _Abi>
1218constexpr bool mask_any_of(compact_mask_tag, const basic_mask<_Bytes, _Abi>& mask) noexcept {
1219 return mask.to_builtin() != 0; // Compact mask can be directly compared to 0.
1220}
1221
1222constexpr bool mask_none_of(generic_tag, const mask_type auto& mask) noexcept {
1223 // Note that real reduction (divide-and-conquor) would be faster, but only for
1224 // really big masks. Note also that it would be more efficient to OR the
1225 // values together as vecs and call to_ullong at the end, but the
1226 // compiler (clang/oneAPI) figures this out for itself anyway.
1227 std::uint64_t reducedBits = 0;
1228 chunked_invoke([&](auto m) { reducedBits |= m.to_ullong(); }, mask);
1229 return reducedBits == 0;
1230}
1231
1232template<std::size_t _Bytes, typename _Abi>
1233constexpr bool mask_none_of(compact_mask_tag, const basic_mask<_Bytes, _Abi>& mask) noexcept {
1234 return mask.to_builtin() == 0; // Compact mask can be directly compared to 0.
1235}
1237
1242constexpr simd_size_type mask_reduce_count(generic_tag, const mask_type auto& mask) noexcept {
1243 // Note that real reduction (divide-and-conquor) would be faster, but only for
1244 // really big masks.
1245 simd_size_type numBits = 0;
1246 chunked_invoke([&](auto m) { numBits += std::popcount(m.to_ullong()); }, mask);
1247 return numBits;
1248}
1249
1250constexpr simd_size_type mask_reduce_count(compact_mask_tag, const mask_type auto& mask) noexcept {
1251 // The bitset route generates code that is just as fast as anything that could be hand-written.
1252 return mask.to_bitset().count();
1253}
1255
1260constexpr simd_size_type mask_reduce_min_index(generic_tag, mask_type auto& mask) noexcept {
1261 simd_size_type lowestSetBit = mask.size();
1262
1263 // Break the mask into individual register-sized pieces and compute the lowest
1264 // in each. Then merge all the results together.
1265 auto processBlock = [&](mask_type auto m, auto idx)
1266 {
1267 auto bits = m.to_ullong();
1268 if (bits != 0)
1269 lowestSetBit = std::min<int>(std::countr_zero(bits) + idx, lowestSetBit);
1270 };
1271
1272 chunked_invoke(processBlock, mask);
1273 return lowestSetBit;
1274}
1275
1276template<mask_type _Mp>
1277constexpr simd_size_type mask_reduce_min_index(compact_mask_tag, const _Mp& mask) noexcept {
1278 // Handling arbitrary length masks is possible, but for now the library has a
1279 // limit of 128 bits anyway, so exploit that to make the code simpler.
1280 static_assert(_Mp::size <= 128, "This function only handles up to 128 bits");
1281
1282 auto lowResult = std::countr_zero((std::uint64_t)mask.to_builtin());
1283
1284 if constexpr (_Mp::size <= 64)
1285 return lowResult;
1286 else
1287 {
1288 auto highResult = std::countr_zero((std::uint64_t)(mask.to_builtin() >> 64));
1289 if (lowResult < 64) return lowResult;
1290 else return highResult + 64;
1291 }
1292}
1294
1299constexpr simd_size_type mask_reduce_max_index(generic_tag, const mask_type auto& mask) noexcept {
1300 simd_size_type highestSetBit = 0;
1301
1302 // Break the mask into individual register-sized pieces and compute the highest
1303 // in each. Then merge all the results together.
1304 auto processBlock = [&](mask_type auto m, auto idx) {
1305 auto bits = m.to_ullong();
1306 if (bits != 0)
1307 highestSetBit = std::max<int>((63 - std::countl_zero(bits)) + idx, highestSetBit);
1308 };
1309
1310 chunked_invoke(processBlock, mask);
1311 return highestSetBit;
1312}
1313
1314template<mask_type _Mp>
1315constexpr simd_size_type mask_reduce_max_index(compact_mask_tag, const _Mp& mask) noexcept {
1316 // Handling arbitrary length masks is possible, but for now the library has a
1317 // limit of 128 bits anyway, so exploit that to make the code simpler.
1318 static_assert(_Mp::size <= 128, "This function only handles up to 128 bits");
1319
1320 auto lowResult = 63 - std::countl_zero((std::uint64_t)mask.to_builtin());
1321
1322 if constexpr (_Mp::size <= 64)
1323 return lowResult;
1324 else
1325 {
1326 auto upperBits = (std::uint64_t)(mask.to_builtin() >> 64);
1327 if (upperBits != 0)
1328 return 127 - std::countl_zero(upperBits);
1329 else
1330 return lowResult;
1331 }
1332}
1334
1343template<mask_type _Mp>
1344constexpr auto get_n_bit_mask(generic_tag, const _Mp&, simd_size_type _n) {
1345 static_assert(_Mp::size <= 256, "The code below won't work for more than 256 elements");
1346 if constexpr (simd_mask_element_size_v<_Mp> == 1 && _Mp::size >= 128)
1347 {
1348 // Special case. The mask's vec_type is normally signed, which for a 1 byte
1349 // element type is only good for up to 127 values. Handling a 128 element
1350 // mask means switching to unsigned bytes. Even then it can only go to 256
1351 // elements, but for now this will do.
1352 auto unsignedIota = iota<typename _Mp::traits::unsigned_vec_for_mask>;
1353 return unsignedIota < std::uint8_t(std::min<std::size_t>(_n, 255));
1354 } else {
1355 using _X = typename _Mp::traits::signed_vec_for_mask;
1356 return iota<_X> < (typename _X::value_type)std::min(_n, _Mp::size());
1357 }
1358}
1359
1360template<mask_type _M>
1361constexpr auto get_n_bit_mask(compact_mask_tag, const _M&, simd_size_type _n)
1362{
1363 static_assert(_M::size <= 255);
1364
1365 if constexpr (_M::size <= 64)
1366 {
1367 const auto clamped = std::min(_M::size(), _n);
1368 return _M::from_builtin(typename _M::builtin_type(clamped == 64 ? ~0ULL : (1ULL << clamped) - 1));
1369 }
1370 else
1371 return _M::from_builtin(_M(true).to_builtin() >> (_M::size - std::min(_n, _M::size())));
1372}
1374
1379template<vec_unsigned_integral _Vp>
1380constexpr _Vp clz(generic_tag, const _Vp& x) noexcept
1381 { return _Vp([=](auto i) -> typename _Vp::value_type { return std::countl_zero(x[i]); }); }
1382
1387template<vec_unsigned_integral _Vp>
1388constexpr _Vp popcount(generic_tag, const _Vp& x) noexcept
1389 { return _Vp([=](auto i) -> typename _Vp::value_type { return std::popcount(x[i]); }); }
1390
1398template<std::integral _Tp, typename _Abi>
1399constexpr basic_vec<_Tp, _Abi> saturating_add(generic_tag, const basic_vec<_Tp, _Abi>& lhs, const basic_vec<_Tp, _Abi>& rhs) noexcept
1400{
1401 #if __has_builtin(__builtin_elementwise_add_sat)
1402 return __builtin_elementwise_add_sat(lhs.to_builtin(), rhs.to_builtin());
1403 #else
1404 // Manually synthesise if the compiler can't do it. this might not generate
1405 // such good code as the compiler can do itself.
1406 if constexpr (std::unsigned_integral<_Tp>)
1407 {
1408 auto t = lhs + rhs;
1409 return select(t < lhs, _Tp(~_Tp()), t); // The condition is true when overflow happens (i.e., when it should saturate).
1410 }
1411 else
1412 {
1413 // Adapted from https://web.archive.org/web/20190213215419/https://locklessinc.com/articles/sat_arithmetic/
1414 using asUnsigned = std::make_unsigned_t<_Tp>;
1415 auto ux = simd_bit_cast<asUnsigned>(lhs);
1416 auto uy = simd_bit_cast<asUnsigned>(rhs);
1417 auto res = ux + uy;
1418
1419 // Create an overflow value. the top bit is shifted to give 0/1, and then
1420 // add the max to get min/max.
1421 ux = (ux >> (sizeof(_Tp) * 8 - 1)) + asUnsigned(std::numeric_limits<_Tp>::max());
1422
1423 auto t = simd_bit_cast<_Tp>((ux ^ uy) | ~(ux ^ res));
1424 return simd_bit_cast<_Tp>(select(t >= _Tp(), ux, res));
1425 }
1426 #endif
1427}
1428
1436template <std::integral _Tp, typename _Abi>
1437constexpr basic_vec<_Tp, _Abi> saturating_sub(generic_tag, const basic_vec<_Tp, _Abi>& lhs, const basic_vec<_Tp, _Abi>& rhs) noexcept
1438{
1439 #if __has_builtin(__builtin_elementwise_sub_sat)
1440 return __builtin_elementwise_sub_sat(lhs.to_builtin(), rhs.to_builtin());
1441 #else
1442 // Manually synthesise if the compiler can't do it. this might not generate
1443 // such good code as the compiler can do itself.
1444 if constexpr (std::unsigned_integral<_Tp>)
1445 {
1446 auto diff = lhs - rhs;
1447 return select(diff > lhs, basic_vec<_Tp, _Abi>{}, diff);
1448 }
1449 else
1450 {
1451 // Adapted from https://web.archive.org/web/20190213215419/https://locklessinc.com/articles/sat_arithmetic/
1452 using asUnsigned = std::make_unsigned_t<_Tp>;
1453 auto ux = simd_bit_cast<asUnsigned>(lhs);
1454 auto uy = simd_bit_cast<asUnsigned>(rhs);
1455 auto res = ux - uy;
1456
1457 // Create an overflow value. The top bit is shifted to give 0/1, and then
1458 // add the max to get min/max.
1459 ux = (ux >> (sizeof(_Tp) * 8 - 1)) + asUnsigned(std::numeric_limits<_Tp>::max());
1460
1461 auto t = simd_bit_cast<_Tp>((ux ^ uy) & (ux ^ res));
1462 return simd_bit_cast<_Tp>(select(t < _Tp(), ux, res));
1463 }
1464 #endif
1465}
1466
1473template<mask_type _To, mask_type _From>
1474constexpr _To convert_mask(generic_tag, const _From& v)
1475{
1476 constexpr auto fromSize = simd_mask_element_size_v<_From>;
1477 constexpr auto toSize = simd_mask_element_size_v<_To>;
1478
1479 auto c = vec_as_container(v);
1480
1481 using _ToType = container_for_num_bytes<toSize>;
1482
1483 if constexpr (fromSize == toSize)
1484 return _To(v); // Same size, so the mask doesn't change, only its type.
1485 else if constexpr (toSize < fromSize)
1486 // Mask elements get smaller, so all ones will be truncated to still be all ones.
1487 return _To::from_builtin(rebind_cast<_ToType>(c).to_builtin());
1488 else
1489 // Mask elements get bigger, so do a != comparison to 0 to recover a full element.
1490 return rebind_cast<_ToType>(c) != _ToType();
1491}
1492
1493template<mask_type _To, mask_type _From>
1494constexpr _To convert_mask(compact_mask_tag, const _From& m) { return _To::from_builtin(m.to_builtin()); }
1496
1502template<typename _Result>
1503constexpr auto generate_mask_from_unsigned(generic_tag, std::unsigned_integral auto mi) {
1504
1505 auto processBlock = [&]<mask_type _Mask>(_Mask, simd_size_type idx) -> _Mask {
1506 using _Mp = container_for_num_bytes<simd_mask_element_size_v<_Mask>>;
1507 using _Vp = vec<_Mp, _Mask::size>;
1508
1509 auto bits = mi >> idx;
1510
1511 // This code works by putting a copy of the appropriate bit into each
1512 // element position, and then testing that the bit is set to form a mask.
1513 if constexpr (sizeof(_Mp) != 1 || _Result::size <= 8)
1514 {
1515 // The simplest case can do this by broadcasting the basic bit to every
1516 // element. For example:
1517 // abcdefgh abcdefgh abcdefgh abcdefgh abcdefgh abcdefgh abcdefgh abcdefgh
1518 // 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000
1519 // Comparing the result of the AND against zero will then set all the bits in each element.
1520 // This works for all sizes except bytes.
1521 return ((_Mp(1) << iota<_Vp>) & _Mp(bits)) != _Vp();
1522 }
1523 else
1524 {
1525 // Bytes don't work when there are more than 8 elements because you run
1526 // out of bits in each element to play the trick above. Instead, use the
1527 // following slightly more tricky code instead. This works by broadcasting
1528 // the lower 8 bits to the first iota lane, and the upper 8-bits to the
1529 // second iota lane, and so on. So 0xabcdef would become
1530 // [ab ab ab ab ab ab ab ab cd cd cd cd cd cd cd cd cd ef ef ef ef ef ef ef ef ef]
1531 const auto bytesAs64BitVec = dependent_type<_Result, vec<std::uint64_t, 1>>(bits);
1532 auto t = simd_bit_cast<std::uint8_t>(bytesAs64BitVec);
1533 auto expandedBytes = permute<_Mask::size>(t, [](int idx) { return idx / 8; });
1534 return ((_Mp(1) << (iota<_Vp> % _Mp(8))) & expandedBytes) != _Vp();
1535 }
1536 };
1537
1538 // Create the lower elements from the incoming bits and then the upper bits
1539 // are filled with zeros using a resize.
1540 constexpr int numBits = std::min<int>(sizeof(mi) * 8, _Result::size());
1541 auto r = chunked_invoke(processBlock, resize_t<numBits, _Result>());
1542 return fit_to_size<_Result::size()>(r);
1543}
1544
1545template<mask_type _Result>
1546constexpr _Result generate_mask_from_unsigned(compact_mask_tag, std::unsigned_integral auto mi) {
1547 // Two stage approach required to call builtin constructor. Can't just call the same
1548 // integral constructor for _Result since that is what called this (and which leads to recursion).
1549 return _Result::from_builtin(typename _Result::builtin_type(mi));
1550}
1552
1553template<vec_floating_point _Vp>
1554constexpr _Vp rcp(generic_tag, const _Vp& x)
1555 { return std::uint8_t(1) / x; }
1556
1557template<vec_floating_point _Vp>
1558constexpr _Vp sqrt(generic_tag, const _Vp& x) {
1559 using _Tp = typename _Vp::value_type;
1560 if constexpr (is_fp16_v<_Tp>)
1561 return _Vp([=](auto i) { return typename _Vp::value_type(std::sqrt(float(x[i]))); });
1562 else
1563 return _Vp([=](auto i) { return typename _Vp::value_type(std::sqrt(x[i])); });
1564}
1565
1566template<vec_floating_point _Vp>
1567constexpr _Vp rsqrt(generic_tag, const _Vp& x)
1568 { return std::uint8_t(1) / sqrt(x); }
1569
1570enum class RoundOp{CEIL, FLOOR, TRUNC, ROUND, RINT};
1571
1572template<RoundOp _Op, vec_floating_point _Vp>
1573constexpr _Vp round_op(generic_tag, const _Vp& v)
1574{
1575 auto doOp = [](auto f) {
1576 switch (_Op) {
1577 case RoundOp::CEIL: return std::ceil(f);
1578 case RoundOp::FLOOR: return std::floor(f);
1579 case RoundOp::TRUNC: return std::trunc(f);
1580 case RoundOp::ROUND: return std::round(f);
1581 case RoundOp::RINT: return std::rint(f);
1582 }
1583 };
1584
1585 if constexpr (is_fp16_v<typename _Vp::value_type>)
1586 return _Vp([=](auto i) { return typename _Vp::value_type(doOp(float(v[i]))); });
1587 else
1588 return _Vp([=](auto i) { return typename _Vp::value_type(doOp(v[i])); });
1589}
1590
1591} // namespace detail
1592
1593} // namespace _XVEC_NAMESPACE::simd
constexpr _Vp min(const _Vp &lhs, const _Vp &rhs) noexcept
Compute the smaller of each respective vec element from the two input arguments.
Definition std_algorithm.hpp:34
constexpr _Vp max(const _Vp &lhs, const _Vp &rhs) noexcept
Compute the larger of each respective vec element from the two input arguments.
Definition std_algorithm.hpp:48
constexpr auto select(const _M &m, const _T &if_true, const _U &if_false) noexcept -> decltype(simd_select_impl(m, if_true, if_false))
Select between corresponding elements according to a mask.
Definition std_algorithm.hpp:88
constexpr auto popcount(const _Vp &x) noexcept
Count the one bits in each element.
Definition std_bit.hpp:133
constexpr _Up rotr(const _Up &u, const _Sp &s)
Rotate each unsigned element right by its corresponding offset. Bits rotated off one end reappear at ...
Definition std_bit.hpp:68
constexpr _Vp::mask_type has_single_bit(const _Vp &x) noexcept
Identify elements that are integral powers of two.
Definition std_bit.hpp:146
constexpr _Vp byteswap(const _Vp &v) noexcept
Reverse the bytes within each integral element.
Definition std_bit.hpp:226
constexpr auto abs(const _Vc &value)
Compute the magnitude of each complex element.
Definition std_complex.hpp:55
constexpr _Vc conj(const _Vc &value)
Compute the conjugate of each complex element.
Definition std_complex.hpp:31
constexpr auto vec_as_container(const _Vp &v)
Generate a variable containing the iota sequence (i.e., 0, 1, 2, ... size-1) for the given simd::vec ...
Definition generic_impl.hpp:116
constexpr void unchecked_store(const basic_vec< _Tp, _Abi > &value, _Range &&r, flags< _Flags... > f={})
Unchecked store into a contiguous sized range.
Definition loadstore.hpp:332
constexpr std::bitset< size()> to_bitset() const noexcept
Return a bitset containing the value of each mask element.
Definition mask.hpp:292
basic_mask< sizeof(_Tp), simd_fixed_size_abi< _Np > > mask
Definition core.hpp:383
static constexpr auto size
Return the number of elements contained in the mask.
Definition mask.hpp:148
constexpr detail::deduced_vec_t< _Vp > sqrt(const _Vp &v)
Compute the square root of each element.
Definition std_math.hpp:102
constexpr detail::deduced_vec_t< _Vp > fma(const _Vp &lhs, const _Vp &rhs, const _Vp &acc)
Compute (lhs * rhs) + acc element-wise with a single rounding.
Definition std_math.hpp:111
constexpr _V compress(const _V &v, const typename _V::mask_type &selector)
Compress a basic_vec or basic_mask value by a mask. The elements corresponding to any set mask bits w...
Definition permute.hpp:145
constexpr _V expand(const _V &v, const typename _V::mask_type &selector, const _V &original={})
Expand consecutive input elements into positions selected by a mask.
Definition permute.hpp:214
constexpr auto permute(const _Vp &v, detail::index_generator_function auto &&fn)
Permute the elements in a vec value using a generator function to compute the indexing.
Definition permute.hpp:94
constexpr auto partial_gather_from(_Range &&r, const _Indexes &indexes, flags< _Flags... > f={})
Gather the contents of a contiguous range using the integral indexes supplied in another vec value.
Definition permute.hpp:258
constexpr auto chunked_invoke(_Fn fn, const _Arg0 &arg0, const _Args &... args)
Invoke a callable on corresponding vector or mask chunks.
Definition invoke.hpp:61
constexpr detail::deduced_vec_t< _Vp > fmaddsub(const _Vp &lhs, const _Vp &rhs, const _Vp &acc)
Compute lhs * rhs - acc at even indexes and lhs * rhs + acc at odd indexes. Each expression is comput...
Definition std_math.hpp:136
constexpr detail::deduced_vec_t< _Vp > rcp(const _Vp &v)
Compute the approximate reciprocal of each element.
Definition std_math.hpp:85
constexpr detail::deduced_vec_t< _Vp > fmsubadd(const _Vp &lhs, const _Vp &rhs, const _Vp &acc)
Compute lhs * rhs + acc at even indexes and lhs * rhs - acc at odd indexes. Each expression is comput...
Definition std_math.hpp:147
constexpr detail::deduced_vec_t< _Vp > rsqrt(const _Vp &v)
Compute the approximate reciprocal square root of each element.
Definition std_math.hpp:91
constexpr auto to_builtin() const
Return the builtin mask value directly.
Definition mask.hpp:275
constexpr auto simd_bit_cast(const _From &v) noexcept
A simd-generic version of simd_bit_cast which directly invokes std::bit_cast.
Definition core.hpp:585
constexpr T saturating_sub(T lhs, T rhs) noexcept
Subtract two integers, clamping an unrepresentable mathematical result.
Definition compat.hpp:90
constexpr T saturating_add(T lhs, T rhs) noexcept
Add two integers, clamping an unrepresentable mathematical result.
Definition compat.hpp:68
constexpr To saturating_cast(From value) noexcept
Definition compat.hpp:50
Special type used to indicate that the compress operation will not fill unused values.
Definition generic_impl.hpp:577
Definition target_config.hpp:23