Intel xvec
Experimental C++ simd class library
Loading...
Searching...
No Matches
x86_impl.hpp
1//===----------------------------------------------------------------------===//
2//
3// Copyright (C) 2021 Intel Corporation
4//
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#pragma once
10
11#include <array>
12#include <immintrin.h>
13#include <bitset>
14#include <cstdint>
15#include <cstddef> // std::size_t
16#include <concepts> // std::floating_point, std::unsigned_integral
17#include <complex>
18#include <type_traits>
19#include <algorithm> // std::min
20#include <functional> // std::equal_to, std::less, etc.
21#include <bit> // std::popcount
22#include <limits> // std::numeric_limits
23#include <utility> // std::pair
24
25#include <xvec/detail/config.hpp>
26#include <xvec/detail/utilities.hpp>
27#include <xvec/detail/core.hpp>
28
29#include <xvec/x86/permute.hpp>
30
31namespace _XVEC_NAMESPACE::simd {
32
33namespace detail
34{
35
36#if defined(__AVX512F__)
37template<typename _Tp, typename _Abi>
38constexpr basic_vec<_Tp, _Abi>
39select_if_else(x86_avx512_tag, const typename basic_vec<_Tp, _Abi>::mask_type& if_mask,
40 const basic_vec<_Tp, _Abi>& true_value, const basic_vec<_Tp, _Abi>& false_value)
41{
42 target_overloads impl {
43 [=]<xmm_register<std::uint8_t> _Vec>(_Vec if_true, _Vec if_false, auto m)
44 { return _mm_mask_blend_epi8(__mmask16(m.to_register()), if_false.to_register(), if_true.to_register()); },
45 [=]<ymm_register<std::uint8_t> _Vec>(_Vec if_true, _Vec if_false, auto m)
46 { return _mm256_mask_blend_epi8(__mmask32(m.to_register()), if_false.to_register(), if_true.to_register()); },
47 [=]<zmm_register<std::uint8_t> _Vec>(_Vec if_true, _Vec if_false, auto m)
48 { return _mm512_mask_blend_epi8(__mmask64(m.to_register()), if_false.to_register(), if_true.to_register()); },
49
50 [=]<xmm_register<std::uint16_t> _Vec>(_Vec if_true, _Vec if_false, auto m)
51 { return _mm_mask_blend_epi16(__mmask8(m.to_register()), if_false.to_register(), if_true.to_register()); },
52 [=]<ymm_register<std::uint16_t> _Vec>(_Vec if_true, _Vec if_false, auto m)
53 { return _mm256_mask_blend_epi16(__mmask16(m.to_register()), if_false.to_register(), if_true.to_register()); },
54 [=]<zmm_register<std::uint16_t> _Vec>(_Vec if_true, _Vec if_false, auto m)
55 { return _mm512_mask_blend_epi16(__mmask32(m.to_register()), if_false.to_register(), if_true.to_register()); },
56
57 [=]<xmm_register<std::uint32_t> _Vec>(_Vec if_true, _Vec if_false, auto m)
58 { return _mm_mask_blend_epi32(__mmask8(m.to_register()), if_false.to_register(), if_true.to_register()); },
59 [=]<ymm_register<std::uint32_t> _Vec>(_Vec if_true, _Vec if_false, auto m)
60 { return _mm256_mask_blend_epi32(__mmask8(m.to_register()), if_false.to_register(), if_true.to_register()); },
61 [=]<zmm_register<std::uint32_t> _Vec>(_Vec if_true, _Vec if_false, auto m)
62 { return _mm512_mask_blend_epi32(__mmask16(m.to_register()), if_false.to_register(), if_true.to_register()); },
63
64 [=]<xmm_register<std::uint64_t> _Vec>(_Vec if_true, _Vec if_false, auto m)
65 { return _mm_mask_blend_epi64(__mmask8(m.to_register()), if_false.to_register(), if_true.to_register()); },
66 [=]<ymm_register<std::uint64_t> _Vec>(_Vec if_true, _Vec if_false, auto m)
67 { return _mm256_mask_blend_epi64(__mmask8(m.to_register()), if_false.to_register(), if_true.to_register()); },
68 [=]<zmm_register<std::uint64_t> _Vec>(_Vec if_true, _Vec if_false, auto m)
69 { return _mm512_mask_blend_epi64(__mmask8(m.to_register()), if_false.to_register(), if_true.to_register()); },
70
71 [=]<xmm_register<unsigned __int128> _Vec>(_Vec if_true, _Vec if_false, auto m)
72 { return (m.to_register() & 1) == 0 ? if_false.to_register() : if_true.to_register(); },
73 [=]<ymm_register<unsigned __int128> _Vec>(_Vec if_true, _Vec if_false, auto m)
74 { return _mm256_mask_blend_epi64(__mmask8(dupBits(m.to_register())), if_false.to_register(), if_true.to_register()); },
75 [=]<zmm_register<unsigned __int128> _Vec>(_Vec if_true, _Vec if_false, auto m)
76 { return _mm512_mask_blend_epi64(__mmask8(dupBits(m.to_register())), if_false.to_register(), if_true.to_register()); },
77
78 [=](auto unhandled, auto, auto) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); return unhandled; }
79 };
80
81 auto wrapper = [=]<typename _Vec>(_Vec ift, _Vec iff, auto ifm) {
82 auto r = _Vec(impl(ift, iff, ifm));
83 return simd_bit_cast<_Tp>(r);
84 };
85
86 constexpr int _Np = basic_vec<_Tp, _Abi>::size;
87 using _C = container_for_type<_Tp>;
88
89 auto trueAsC = vec_as_container(true_value);
90 auto falseAsC = vec_as_container(false_value);
91
92 if (std::is_constant_evaluated())
93 {
94 // operator? doesn't work in constexpr for Clang so implement in terms of
95 // bitwise operations instead with a wide mask (all 0/1 bits). Note that
96 // `-mask` is defined in terms of select, so the mask must be built
97 // explicitly here.
98 auto wideM = vec<_C, _Np>([=](auto i) -> _C { if (if_mask[i]) return ~_C(); else return _C(); }).to_builtin();
99 basic_vec<_C, _Abi> r = (wideM & trueAsC.to_builtin()) | ((~wideM) & falseAsC.to_builtin());
100 return simd_bit_cast<_Tp>(r);
101 }
102 else
103 return chunked_invoke(wrapper, trueAsC, falseAsC, mask<_C, _Np>(if_mask));
104}
105#endif
106
107inline target_overloads msb_to_bitmask_impl {
108 // Intel SSE
109 [](xmm_register<std::uint8_t> auto m) -> std::uint16_t { return _mm_movemask_epi8(m.to_register()); },
110 [](xmm_register<std::uint16_t> auto m) -> std::uint8_t {
111 // No 16-bit equivalent of movemask in Intel SSE so put all the upper bits of the mask
112 // elements into the lower 64-bits and grab those. All the other elements
113 // are zeroed.
114 const auto B = simd_bit_cast<std::uint8_t>(m);
115 const auto reqBits = permute<16>(B, [](auto idx) { return idx < decltype(m)::size() ? idx * 2 : zero_element; });
116 return _mm_movemask_epi8(reqBits.to_register());
117 },
118 [](xmm_register<std::uint32_t> auto m) -> std::uint8_t { return _mm_movemask_ps(__m128(m.to_register()));},
119 [](xmm_register<std::uint64_t> auto m) -> std::uint8_t { return _mm_movemask_pd(__m128d(m.to_register()));},
120 [](xmm_register<unsigned __int128> auto m) -> std::uint8_t { return _mm_movemask_pd(__m128d(m.to_register())) >> 1;},
121
122#if defined(__AVX2__)
123 // Intel AVX2
124 [](ymm_register<std::uint8_t> auto m) -> std::uint32_t { return _mm256_movemask_epi8(m.to_register()); },
125 [](ymm_register<std::uint16_t> auto m) -> std::uint16_t { return _pext_u64(_mm256_movemask_epi8(m.to_register()), 0xAAAAAAAA); },
126#elif defined(__AVX__)
127 // Intel AVX - doesn't have 16-bit movemask, or 8-bit beyond the first 128-bit lane.
128 [](ymm_register<std::uint8_t> auto m) -> std::uint32_t {
129 // Process upper and lower halves separately and then combine them.
130 auto lower = _mm_movemask_epi8(_mm256_castsi256_si128(m.to_register()));
131 auto upper = _mm_movemask_epi8(_mm256_extractf128_si256(m.to_register(), 1));
132 return std::uint32_t(lower) | (std::uint32_t(upper) << 16);
133 },
134 [](ymm_register<std::uint16_t> auto m) -> std::uint16_t {
135 // Convert each the top byte of each 16-bit value into an 8-bit value and extract their sign bits.
136 auto to8 = rebind_cast<std::uint8_t>(m >> 8);
137 return _mm_movemask_epi8(to8.to_register());
138 },
139#endif
140
141#if defined(__AVX__)
142 [](ymm_register<std::uint32_t> auto m) -> std::uint8_t { return _mm256_movemask_ps(__m256(m.to_register()));},
143 [](ymm_register<std::uint64_t> auto m) -> std::uint8_t { return _mm256_movemask_pd(__m256d(m.to_register()));},
144 [](ymm_register<unsigned __int128> auto m) -> std::uint8_t {
145 auto fourMsbs = _mm256_movemask_pd(__m256d(m.to_register()));
146 // Only need bits 1 and 3.
147 #if defined(__BMI2__)
148 return _pext_u64(fourMsbs, 0b1010);
149 #else
150 auto masked4 = (fourMsbs & 0b1010) >> 1;
151 return ((masked4 >> 1) | masked4) & 0b11;
152 #endif
153 },
154#endif
155
156 [](auto unhandled) -> unsigned long { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); return 0;}
157};
158
159template<mask_type _Mp>
160constexpr std::bitset<_Mp::size>
161mask_to_bitset(x86_sse_tag, const _Mp& mask) noexcept
162{
163 constexpr int _Np = _Mp::size;
164 using _R = std::bitset<_Np>;
165
166 // Given a subset of a vec_impl, extract the raw bits and insert into the correct result position.
167 _R result = {};
168 auto processBlock = [&](auto v, simd_size_type idx) {
169 auto msbs = msb_to_bitmask_impl(vec_as_container(v)) & ((1ULL << v.size()) - 1);
170 result |= _R(msbs) << idx;
171 };
172 chunked_invoke(processBlock, mask);
173 return result;
174}
175
176template<mask_type _Mp>
177constexpr std::bitset<_Mp::size>
178mask_to_bitset(x86_avx512_tag, const _Mp& mask) noexcept
179{
180 constexpr int _Np = _Mp::size;
181 std::bitset<_Np> result;
182
183 // Build the bitset up 64-bits at a time. Each 64-bit group is extracted from the extended int and
184 // then inserted into the bitset.
185 for (simd_size_type i = 0; i<((_Np + 63) / 64); ++i)
186 {
187 const auto b = std::bitset<_Np>(std::uint64_t(mask.to_builtin() >> (64 * i)));
188 result |= (b << (64 * i));
189 }
190
191 return result;
192}
193
194
195template<std::floating_point _Tp, typename _Abi>
196constexpr basic_vec<_Tp, _Abi> fma(x86_avx2_tag, const basic_vec<_Tp, _Abi>& x, const basic_vec<_Tp, _Abi>& y, const basic_vec<_Tp, _Abi>& acc)
197{
198 target_overloads impl {
199 // 128-bit (not Intel SSE - only Intel AVX2 onwards has FMA)
200 [=](xmm_register<float> auto x, auto y, auto acc) { return _mm_fmadd_ps(x.to_register(), y.to_register(), acc.to_register()); },
201 [=](xmm_register<double> auto x, auto y, auto acc) { return _mm_fmadd_pd(x.to_register(), y.to_register(), acc.to_register()); },
202
203 // 256-bit
204 [=](ymm_register<float> auto x, auto y, auto acc) { return _mm256_fmadd_ps(x.to_register(), y.to_register(), acc.to_register()); },
205 [=](ymm_register<double> auto x, auto y, auto acc) { return _mm256_fmadd_pd(x.to_register(), y.to_register(), acc.to_register()); },
206
207 // 512-bit
208#if defined(__AVX512F__)
209 [=](zmm_register<float> auto x, auto y, auto acc) { return _mm512_fmadd_ps(x.to_register(), y.to_register(), acc.to_register()); },
210 [=](zmm_register<double> auto x, auto y, auto acc) { return _mm512_fmadd_pd(x.to_register(), y.to_register(), acc.to_register()); },
211#endif
212
213 [=](auto unhandled, auto, auto) -> decltype(unhandled) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); }
214 };
215
216 auto wrapper = [=](auto a, auto b, auto c) { return decltype(a)(impl(a, b, c)); }; // Return a vec from an intrinsic.
217 return chunked_invoke(wrapper, x, y, acc);
218}
219
220template<std::floating_point _Tp, typename _Abi>
221constexpr
222basic_vec<_Tp, _Abi>
223fmaddsub(x86_avx2_tag, const basic_vec<_Tp, _Abi>& x, const basic_vec<_Tp, _Abi>& y, const basic_vec<_Tp, _Abi>& acc)
224{
225 target_overloads impl {
226 // 128-bit (not Intel SSE - only Intel AVX2 onwards has FMA)
227 [=](xmm_register<float> auto x, auto y, auto acc) { return _mm_fmaddsub_ps(x.to_register(), y.to_register(), acc.to_register()); },
228 [=](xmm_register<double> auto x, auto y, auto acc) { return _mm_fmaddsub_pd(x.to_register(), y.to_register(), acc.to_register()); },
229
230 // 256-bit
231 [=](ymm_register<float> auto x, auto y, auto acc) { return _mm256_fmaddsub_ps(x.to_register(), y.to_register(), acc.to_register()); },
232 [=](ymm_register<double> auto x, auto y, auto acc) { return _mm256_fmaddsub_pd(x.to_register(), y.to_register(), acc.to_register()); },
233
234 // 512-bit
235#if defined(__AVX512F__)
236 [=](zmm_register<float> auto x, auto y, auto acc) { return _mm512_fmaddsub_ps(x.to_register(), y.to_register(), acc.to_register()); },
237 [=](zmm_register<double> auto x, auto y, auto acc) { return _mm512_fmaddsub_pd(x.to_register(), y.to_register(), acc.to_register()); },
238#endif
239
240 [=](auto unhandled, auto, auto) -> decltype(unhandled) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); }
241 };
242
243 auto wrapper = [=](auto a, auto b, auto c) { return decltype(a)(impl(a, b, c)); }; // Return a vec from an intrinsic.
244 return chunked_invoke(wrapper, x, y, acc);
245}
246
247template<std::floating_point _Tp, typename _Abi>
248constexpr
249basic_vec<_Tp, _Abi>
250fmsubadd(x86_avx2_tag, const basic_vec<_Tp, _Abi>& x, const basic_vec<_Tp, _Abi>& y, const basic_vec<_Tp, _Abi>& acc)
251{
252 target_overloads impl {
253 // 128-bit (not Intel SSE - only Intel AVX2 onwards has FMA)
254 [=](xmm_register<float> auto x, auto y, auto acc) { return _mm_fmsubadd_ps(x.to_register(), y.to_register(), acc.to_register()); },
255 [=](xmm_register<double> auto x, auto y, auto acc) { return _mm_fmsubadd_pd(x.to_register(), y.to_register(), acc.to_register()); },
256
257 // 256-bit
258 [=](ymm_register<float> auto x, auto y, auto acc) { return _mm256_fmsubadd_ps(x.to_register(), y.to_register(), acc.to_register()); },
259 [=](ymm_register<double> auto x, auto y, auto acc) { return _mm256_fmsubadd_pd(x.to_register(), y.to_register(), acc.to_register()); },
260
261 // 512-bit
262#if defined(__AVX512F__)
263 [=](zmm_register<float> auto x, auto y, auto acc) { return _mm512_fmsubadd_ps(x.to_register(), y.to_register(), acc.to_register()); },
264 [=](zmm_register<double> auto x, auto y, auto acc) { return _mm512_fmsubadd_pd(x.to_register(), y.to_register(), acc.to_register()); },
265#endif
266
267 [=](auto unhandled, auto, auto) -> decltype(unhandled) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); }
268 };
269
270 auto wrapper = [=](auto a, auto b, auto c) { return decltype(a)(impl(a, b, c)); }; // Return a vec from an intrinsic.
271 return chunked_invoke(wrapper, x, y, acc);
272}
273
274#if defined(__AVX512FP16__)
275
276template<vec_of<_Float16> _Vp>
277constexpr _Vp fma(x86_avxspr_tag, const _Vp& x, const _Vp& y, const _Vp& acc) {
278 target_overloads impl {
279 [=](xmm_register<_Float16> auto x, auto y, auto acc) { return _mm_fmadd_ph(x.to_register(), y.to_register(), acc.to_register()); },
280 [=](ymm_register<_Float16> auto x, auto y, auto acc) { return _mm256_fmadd_ph(x.to_register(), y.to_register(), acc.to_register()); },
281 [=](zmm_register<_Float16> auto x, auto y, auto acc) { return _mm512_fmadd_ph(x.to_register(), y.to_register(), acc.to_register()); },
282 [=](auto unhandled, auto, auto) -> decltype(unhandled) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); }
283 };
284
285 auto wrapper = [=](auto a, auto b, auto c) { return decltype(a)(impl(a, b, c)); }; // Return a vec from an intrinsic.
286 return chunked_invoke(wrapper, x, y, acc);
287}
288
289template<vec_of<std::complex<_Float16>> _Vp>
290constexpr _Vp fma(x86_avxspr_tag, const _Vp& x, const _Vp& y, const _Vp& acc) {
291 target_overloads impl {
292 [=](xmm_register<_Float16> auto x, auto y, auto acc) { return _mm_fmadd_pch(x.to_register(), y.to_register(), acc.to_register()); },
293 [=](ymm_register<_Float16> auto x, auto y, auto acc) { return _mm256_fmadd_pch(x.to_register(), y.to_register(), acc.to_register()); },
294 [=](zmm_register<_Float16> auto x, auto y, auto acc) { return _mm512_fmadd_pch(x.to_register(), y.to_register(), acc.to_register()); },
295 [=](auto unhandled, auto, auto) -> decltype(unhandled) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); }
296 };
297
298 auto wrapper = [=](auto a, auto b, auto c) { return decltype(a)(impl(a, b, c)); }; // Return a vec from an intrinsic.
299 auto t = chunked_invoke(wrapper, simd_bit_cast<_Float16>(x), simd_bit_cast<_Float16>(y), simd_bit_cast<_Float16>(acc));
300 return simd_bit_cast<std::complex<_Float16>>(t);
301}
302
303template<vec_of<_Float16> _Vp>
304constexpr _Vp fmaddsub(x86_avxspr_tag, const _Vp& x, const _Vp& y, const _Vp& acc) {
305 target_overloads impl {
306 [=](xmm_register<_Float16> auto x, auto y, auto acc) { return _mm_fmaddsub_ph(x.to_register(), y.to_register(), acc.to_register()); },
307 [=](ymm_register<_Float16> auto x, auto y, auto acc) { return _mm256_fmaddsub_ph(x.to_register(), y.to_register(), acc.to_register()); },
308 [=](zmm_register<_Float16> auto x, auto y, auto acc) { return _mm512_fmaddsub_ph(x.to_register(), y.to_register(), acc.to_register()); },
309 [=](auto unhandled, auto, auto) -> decltype(unhandled) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); }
310 };
311
312 auto wrapper = [=](auto a, auto b, auto c) { return decltype(a)(impl(a, b, c)); }; // Return a vec from an intrinsic.
313 return chunked_invoke(wrapper, x, y, acc);
314}
315
316template<vec_of<_Float16> _Vp>
317constexpr _Vp
318fmsubadd(x86_avxspr_tag, const _Vp& x, const _Vp& y, const _Vp& acc)
319{
320 target_overloads impl {
321 [=](xmm_register<_Float16> auto x, auto y, auto acc) { return _mm_fmsubadd_ph(x.to_register(), y.to_register(), acc.to_register()); },
322 [=](ymm_register<_Float16> auto x, auto y, auto acc) { return _mm256_fmsubadd_ph(x.to_register(), y.to_register(), acc.to_register()); },
323 [=](zmm_register<_Float16> auto x, auto y, auto acc) { return _mm512_fmsubadd_ph(x.to_register(), y.to_register(), acc.to_register()); },
324 [=](auto unhandled, auto, auto) -> decltype(unhandled) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); }
325 };
326
327 auto wrapper = [=](auto a, auto b, auto c) { return decltype(a)(impl(a, b, c)); }; // Return a vec from an intrinsic.
328 return chunked_invoke(wrapper, x, y, acc);
329}
330
331template<vec_of<std::complex<_Float16>> _Vp>
332constexpr _Vp builtin_operator(x86_avxspr_tag, const _Vp& x, const _Vp& y, std::multiplies<>)
333{
334 target_overloads impl {
335 [=](xmm_register<_Float16> auto x, auto y) { return _mm_fmul_pch(x.to_register(), y.to_register()); },
336 [=](ymm_register<_Float16> auto x, auto y) { return _mm256_fmul_pch(x.to_register(), y.to_register()); },
337 [=](zmm_register<_Float16> auto x, auto y) { return _mm512_fmul_pch(x.to_register(), y.to_register()); },
338 [=](auto unhandled, auto) -> decltype(unhandled) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); }
339 };
340
341 auto wrapper = [=](auto a, auto b) { return decltype(a)(impl(a, b)); }; // Return a vec from an intrinsic.
342 auto t = chunked_invoke(wrapper, simd_bit_cast<_Float16>(x), simd_bit_cast<_Float16>(y));
343 return simd_bit_cast<std::complex<_Float16>>(t);
344}
345
346#endif
347
350
351template<RoundOp _Op, typename _Tp, typename _Abi>
352requires (detail::is_floating_point_v<_Tp>)
354{
355 [[maybe_unused]] constexpr auto sseOp = std::array<int, 5>{
356 _MM_FROUND_TO_POS_INF, _MM_FROUND_TO_NEG_INF, _MM_FROUND_TO_ZERO,
357 _MM_FROUND_TO_NEAREST_INT, _MM_FROUND_CUR_DIRECTION
358 }[int(_Op)];
359 [[maybe_unused]] constexpr auto avx512Op = std::array<int, 5>{
360 _MM_FROUND_CEIL, _MM_FROUND_FLOOR, _MM_FROUND_TO_ZERO,
361 _MM_FROUND_TO_NEAREST_INT, _MM_FROUND_CUR_DIRECTION
362 }[int(_Op)];
363
364 target_overloads impl {
365 // Intel SSE
366 [=](xmm_register<float> auto x) { return _mm_round_ps(x.to_register(), sseOp); },
367 [=](xmm_register<double> auto x) { return _mm_round_pd(x.to_register(), sseOp); },
368
369 // Intel AVX
370 [=](ymm_register<float> auto x) { return _mm256_round_ps(x.to_register(), sseOp); },
371 [=](ymm_register<double> auto x) { return _mm256_round_pd(x.to_register(), sseOp); },
372
373 // Intel AVX-512
374#if defined(__AVX512F__)
375 [=](zmm_register<float> auto x) { return _mm512_roundscale_ps(x.to_register(), sseOp); },
376 [=](zmm_register<double> auto x) { return _mm512_roundscale_pd(x.to_register(), sseOp); },
377#endif
378
379#if defined(__AVX512FP16__)
380 [=](xmm_register<_Float16> auto x) { return _mm_roundscale_ph(x.to_register(), avx512Op); },
381 [=](ymm_register<_Float16> auto x) { return _mm256_roundscale_ph(x.to_register(), avx512Op); },
382 [=](zmm_register<_Float16> auto x) { return _mm512_roundscale_ph(x.to_register(), avx512Op); },
383#endif
384
385 [=](auto unhandled) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); return unhandled; }
386 };
387
388 if constexpr (_Op == detail::RoundOp::ROUND)
389 {
390 // Unfortunately the round/roundscale intrinsics don't have a mode which
391 // matches C++ round function, so it needs to be built manually.
392 constexpr _Tp almost_half = [] {
393 if constexpr (sizeof(_Tp) == 2) return static_cast<_Tp>(0x1.ffcp-2);
394 else if constexpr (sizeof(_Tp) == 4) return 0x1.fffffep-2f;
395 else return 0x1.fffffffffffffp-2;
396 }();
397
398 return trunc(v + copysign(basic_vec<_Tp, _Abi>(almost_half), v));
399 }
400 else if constexpr (is_fp16_v<_Tp> && !hasAvxFp16)
401 {
402 // When FP16 rounding is needed in emulation, do the rounding in FP32
403 // instead. All FP16 values can be converted to a valid FP32 which will round in the same way.
404 const auto rounded32 = round_op<_Op>(x86_tag{}, rebind_cast<float>(v));
405 return rebind_cast<_Float16>(rounded32);
406 }
407 else
408 {
409 auto wrapper = [=](auto x) { return decltype(x)(impl(x)); }; // Return a vec from an intrinsic.
410 return chunked_invoke(wrapper, v);
411 }
412}
413
414
416
426template<vec_type _Vp, typename _Up, std::size_t _Extent, typename _Flags>
427constexpr void store_masked(x86_tag, const _Vp& value_original, std::span<_Up, _Extent> to,
428 const typename _Vp::mask_type& mask_original, _Flags flags)
429{
430 mandates_for_store(value_original, to, flags);
431
432 using _ToVec = rebind_t<_Up, _Vp>;
433 auto from = _ToVec(value_original);
434 auto from_mask = typename _ToVec::mask_type(mask_original);
435
436 // This function uses the Intel SSE instruction to write only selected bytes
437 // to memory. Bigger registers must be broken down into Intel SSE-sized
438 // pieces.
439 constexpr auto numSseElements = 16 / sizeof(_Up);
440 auto impl = [=]<typename _Vec>(_Vec x, auto m, auto idx) {
441 // Note that the mask register is zero extended to ensure no extra mask elements appear in it during `to_register'.
442 _mm_maskmoveu_si128(x.to_register(), grow<numSseElements>(m).to_register(), (char*)(to.data() + idx));
443 };
444
445 // Use the mask to limit the bounds of the write if requested by the user.
446 auto boundedMask = from_mask;
448 checkStaticMemoryBounds<decltype(from)>("store", to, from_mask);
449 else
450 boundedMask &= from_mask.__mask_from_count(to.size());
451
452 chunked_invoke<numSseElements>(impl, vec_as_container(from), boundedMask);
453}
454
456#if defined(__AVX2__)
457template<vec_type _Vp, typename _Up, std::size_t _Extent, typename _Flags>
458requires (sizeof(_Up) == 4 || sizeof(_Up) == 8)
459constexpr void store_masked(x86_avx2_tag, const _Vp& value_original, std::span<_Up, _Extent> to,
460 const typename _Vp::mask_type& mask_original, _Flags flags)
461{
462 mandates_for_store(value_original, to, flags);
463
464 using _ToVec = rebind_t<_Up, _Vp>;
465 auto from = _ToVec(value_original);
466 auto from_mask = typename _ToVec::mask_type(mask_original);
467
468 auto impl = [=]<typename _Vec>(_Vec s, auto m, auto idx) {
469 // Resize the mask with zeroes to allow partial registers to be written
470 // back. Can't use initialised resize because that could introduce unwanted
471 // mask elements.
472 constexpr auto numAvxElements = 32 / sizeof(_Up);
473
474 const auto rm = grow<numAvxElements>(m);
475
476 // Resize the values to a whole Intel AVX2 register because a partial register can't be allowed into an Intel SSE.
477 const auto rc = permute<numAvxElements>(s, perm_uninitResize);
478
479 if constexpr (sizeof(_Up) == 4)
480 _mm256_maskstore_epi32(reinterpret_cast<int*>(to.data() + idx), rm.to_register(), rc.to_register());
481 else if constexpr (sizeof(_Up) == 8)
482 _mm256_maskstore_epi64(reinterpret_cast<long long*>(to.data() + idx), rm.to_register(), rc.to_register());
483 else
484 static_assert(dependent_false<_Vec>, "Unimplemented masked store");
485 };
486
487 // Use the mask to limit the bounds of the write if requested by the user.
488 auto boundedMask = from_mask;
489 if constexpr (contains_flag<flag_unchecked>(flags))
490 checkStaticMemoryBounds<_ToVec>("store", to, from_mask);
491 else
492 boundedMask &= from_mask.__mask_from_count(to.size());
493
494 chunked_invoke(impl, vec_as_container(from), boundedMask);
495}
496#endif // __AVX2__
497
498#if defined(__AVX512F__)
500template<vec_type _Vp, typename _Up, std::size_t _Extent, typename _Flags>
501constexpr void store_masked(x86_avx512_tag, const _Vp& value_original, std::span<_Up, _Extent> to,
502 const typename _Vp::mask_type& mask_original, _Flags flags)
503{
504 mandates_for_store(value_original, to, flags);
505
506 target_overloads call_insn {
507 [=](xmm_register<std::uint8_t> auto v, auto m, auto p) { _mm_mask_storeu_epi8(p, m, v.to_register()); },
508 [=](xmm_register<std::uint16_t> auto v, auto m, auto p) { _mm_mask_storeu_epi16(p, m, v.to_register()); },
509 [=](xmm_register<std::uint32_t> auto v, auto m, auto p) { _mm_mask_storeu_epi32(p, m, v.to_register()); },
510 [=](xmm_register<std::uint64_t> auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, m, v.to_register()); },
511
512 [=](ymm_register<std::uint8_t> auto v, auto m, auto p) { _mm256_mask_storeu_epi8(p, m, v.to_register()); },
513 [=](ymm_register<std::uint16_t> auto v, auto m, auto p) { _mm256_mask_storeu_epi16(p, m, v.to_register()); },
514 [=](ymm_register<std::uint32_t> auto v, auto m, auto p) { _mm256_mask_storeu_epi32(p, m, v.to_register()); },
515 [=](ymm_register<std::uint64_t> auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, m, v.to_register()); },
516
517 [=](zmm_register<std::uint8_t> auto v, auto m, auto p) { _mm512_mask_storeu_epi8(p, m, v.to_register()); },
518 [=](zmm_register<std::uint16_t> auto v, auto m, auto p) { _mm512_mask_storeu_epi16(p, m, v.to_register()); },
519 [=](zmm_register<std::uint32_t> auto v, auto m, auto p) { _mm512_mask_storeu_epi32(p, m, v.to_register()); },
520 [=](zmm_register<std::uint64_t> auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, m, v.to_register()); },
521
522 [=](xmm_register<unsigned __int128> auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, dupBits(m), v.to_register()); },
523 [=](ymm_register<unsigned __int128> auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, dupBits(m), v.to_register()); },
524 [=](zmm_register<unsigned __int128> auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, dupBits(m), v.to_register()); },
525
526 [=](auto unhandled, auto, auto) { static_assert(dependent_false<decltype(unhandled)>, "Unimplemented partial masked store"); }
527 };
528
529 using _ToVec = rebind_t<_Up, _Vp>;
530 auto from = _ToVec(value_original);
531 auto from_mask = typename _ToVec::mask_type(mask_original);
532
533 auto impl = [=]<typename _Vec>(_Vec x, auto m, auto idx) {
534 // Create a mask which both truncates to the correct number of values and also applies the incoming mask.
535 constexpr auto partialMask = typename _Vec::mask_type(true);
536 const auto tm = m.to_builtin() & partialMask.to_builtin();
537 call_insn(x, tm, to.data() + idx);
538 };
539
540 // Use the mask to limit the bounds of the write if requested by the user.
541 auto boundedMask = from_mask;
542 if constexpr (contains_flag<flag_unchecked>(flags))
543 checkStaticMemoryBounds<_ToVec>("store", to, from_mask);
544 else
545 boundedMask &= from_mask.__mask_from_count(to.size());
546
547 chunked_invoke(impl, vec_as_container(from), boundedMask);
548}
549#endif
550
552template<vec_type _Vp, typename _Up, std::size_t _Extent, typename _Flags>
553constexpr void store(x86_tag, const _Vp& value_original, std::span<_Up, _Extent> to, _Flags flags)
554{
555 mandates_for_store(value_original, to, flags);
556
557 auto from = rebind_cast<_Up>(value_original);
558 auto ptr = to.data();
559
560 if constexpr (!contains_flag<flag_unchecked>(flags))
561 {
562 // Handle bounded stores using a masked store which already implements the
563 // bounded store efficiently.
564 partial_store(from, ptr, to.size(), typename _Vp::mask_type(true), flags);
565 }
566 else
567 {
568 checkStaticMemoryBounds<decltype(from)>("store", to);
569
570 // Break the store down into individual pieces which are as large as
571 // possible, and which can be written back in individual operations.
572 auto impl = [=]<typename _Vec>(_Vec s, auto idx)
573 {
574 constexpr auto numBytes = _Vec::size * sizeof(_Up);
575
576 // Try a whole register store to begin with.
577 if constexpr (numBytes == 16)
578 _mm_storeu_si128(reinterpret_cast<__m128i *>(ptr + idx), __m128i(s.to_register()));
579#if defined(__AVX__)
580 else if constexpr (numBytes == 32)
581 _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr + idx), __m256i(s.to_register()));
582#endif
583#if defined(__AVX512F__)
584 else if constexpr (numBytes == 64)
585 _mm512_storeu_si512(ptr + idx, __m512i(s.to_register()));
586#endif
587 else
588 {
589 // Partial register instead - create a mask which has just enough bits for
590 // the partial value to store and defer to the masked store.
591 constexpr auto mask = typename _Vec::mask_type(true);
592 unchecked_store(s, ptr + idx, _Vec::size, mask, flags);
593 }
594 };
595
596 chunked_invoke(impl, from);
597 }
598}
599
600#if defined(__AVX512F__)
602template<vec_type _V, typename _Up, std::size_t _Extent, typename _Flags>
603constexpr _V load_masked(x86_avx512_tag, std::span<_Up, _Extent> from, const typename _V::mask_type& mask, _Flags flags)
604{
605 mandates_for_load(from, _V{}, flags);
606
607 // Note that all of these loads use zero as the default mask value. We can't
608 // use the type's actual value because the type of the pointer could be
609 // different to the type of the output vec, and therefore be unrepresentable
610 // during the load.
611 target_overloads call_insn {
612 [=](xmm_register<std::uint8_t> auto, auto m, auto p) { return _mm_maskz_loadu_epi8(m, p); },
613 [=](xmm_register<std::uint16_t> auto, auto m, auto p) { return _mm_maskz_loadu_epi16(m, p); },
614 [=](xmm_register<std::uint32_t> auto, auto m, auto p) { return _mm_maskz_loadu_epi32(m, p); },
615 [=](xmm_register<std::uint64_t> auto, auto m, auto p) { return _mm_maskz_loadu_epi64(m, p); },
616
617 [=](ymm_register<std::uint8_t> auto, auto m, auto p) { return _mm256_maskz_loadu_epi8(m, p); },
618 [=](ymm_register<std::uint16_t> auto, auto m, auto p) { return _mm256_maskz_loadu_epi16(m, p); },
619 [=](ymm_register<std::uint32_t> auto, auto m, auto p) { return _mm256_maskz_loadu_epi32(m, p); },
620 [=](ymm_register<std::uint64_t> auto, auto m, auto p) { return _mm256_maskz_loadu_epi64(m, p); },
621
622 [=](zmm_register<std::uint8_t> auto, auto m, auto p) { return _mm512_maskz_loadu_epi8(m, p); },
623 [=](zmm_register<std::uint16_t> auto, auto m, auto p) { return _mm512_maskz_loadu_epi16(m, p); },
624 [=](zmm_register<std::uint32_t> auto, auto m, auto p) { return _mm512_maskz_loadu_epi32(m, p); },
625 [=](zmm_register<std::uint64_t> auto, auto m, auto p) { return _mm512_maskz_loadu_epi64(m, p); },
626
627 //:TODO: DUP all mask bits in a single operation, and only then extract? Cheaper than dupping all bits in small units.
628 [=](xmm_register<unsigned __int128> auto, auto m, auto p) { return _mm_maskz_loadu_epi64(dupBits(m), p); },
629 [=](ymm_register<unsigned __int128> auto, auto m, auto p) { return _mm256_maskz_loadu_epi64(dupBits(m), p); },
630 [=](zmm_register<unsigned __int128> auto, auto m, auto p) { return _mm512_maskz_loadu_epi64(dupBits(m), p); },
631
632 [=](auto unhandled, auto, auto) { static_assert(dependent_false<decltype(unhandled)>, "Unimplemented partial masked load"); return unhandled; }
633 };
634
635 using SourceType = rebind_t<_Up, _V>;
636 using _C = container_for_type<_Up>;
637
638 auto impl = [=]<typename _CS>(_CS x, auto m, auto idx) {
639 // Create a mask which both truncates to the correct number of values and
640 // also applies the incoming mask, and then cast to the container type.
641 constexpr auto partialMask = typename _CS::mask_type(true);
642 const auto tm = m.to_builtin() & partialMask.to_builtin();
643 return _CS(call_insn(x, tm, from.data() + idx));
644 };
645
646 // Use the mask to limit the bounds of the write if requested by the user.
647 auto boundedMask = mask;
648 if constexpr (contains_flag<flag_unchecked>(flags))
649 checkStaticMemoryBounds<SourceType>("load_masked", from, mask);
650 else
651 boundedMask &= mask.__mask_from_count(from.size());
652
653 // Load as the container type, and do the conversion as a secondary step.
654 // :TODO: Load in the best size to fit a register (i.e., to make for an efficient conversion without spilling into multiple registers).
655 auto t = chunked_invoke(impl, rebind_t<_C, _V>(), typename rebind_t<_C, _V>::mask_type(boundedMask));
656 return _V(simd_bit_cast<std::remove_cvref_t<_Up>>(t));
657}
658
659template<vec_type _V, typename _Up, std::size_t _Extent, typename _Flags>
660constexpr auto load(x86_avx512_tag, std::span<_Up, _Extent> from, _Flags flags)
661{
662 if (contains_flag<flag_unchecked>(flags)) // Unchecked can use the default generic implementation.
663 return load<_V>(generic_tag{}, from, flags);
664 else // Checked uses a mask to limit the load
665 return load_masked<_V>(target, from, typename _V::mask_type(true), flags);
666}
667#endif
668
676template<typename _Range, std::integral _Idx, typename _IdxAbi, typename... _Flags>
677// :TODO: 4/8 byte gathers can be done from either 4/8 byte indexes, so handle
678// those scenarios too to avoid having to cast unnecessarily. Also handle
679// smaller sizes by reordering the indexes into the gather position, rather than
680// converting up and down (which is more expensive).
681requires
682 std::ranges::contiguous_range<_Range> &&
683 std::ranges::sized_range<_Range> &&
684 (sizeof(std::ranges::range_value_t<_Range>) <= 8) // Can't handle 128-bit gathers yet.
685constexpr auto gather_from(x86_avx2_tag, const _Range& r,
688{
689 using _Tp = std::ranges::range_value_t<_Range>;
690 auto data_ptr = std::ranges::data(r);
691
692 target_overloads impl {
693
694#if defined(__AVX512F__)
695 // 32-bit Intel AVX-512
696 [=](xmm_register<std::int32_t> auto i, auto m) { return _mm_mmask_i32gather_epi32(__m128i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); },
697 [=](ymm_register<std::int32_t> auto i, auto m) { return _mm256_mmask_i32gather_epi32(__m256i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); },
698
699 [=](xmm_register<std::int64_t> auto i, auto m) { return _mm_mmask_i64gather_epi64(__m128i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); },
700 [=](ymm_register<std::int64_t> auto i, auto m) { return _mm256_mmask_i64gather_epi64(__m256i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); },
701
702 [=](zmm_register<std::int32_t> auto i, auto m) { return _mm512_mask_i32gather_epi32(__m512i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); },
703 [=](zmm_register<std::int64_t> auto i, auto m) { return _mm512_mask_i64gather_epi64(__m512i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); },
704
705#elif defined(__AVX2__)
706 // Note the explicit growth of the mask to the correct register size to
707 // avoid inadvertently leaving uninitialised elements active.
708
709 // 32-bit Intel AVX2
710 [=](xmm_register<std::int32_t> auto i, auto m)
711 { return _mm_mask_i32gather_epi32(__m128i(), reinterpret_cast<const int*>(data_ptr), i.to_register(), grow<4>(m).to_register(), sizeof(_Tp)); },
712 [=](ymm_register<std::int32_t> auto i, auto m)
713 { return _mm256_mask_i32gather_epi32(__m256i(), reinterpret_cast<const int*>(data_ptr), i.to_register(), grow<8>(m).to_register(), sizeof(_Tp)); },
714
715 // 64-bit Intel AVX2
716 [=](xmm_register<std::int64_t> auto i, auto m)
717 { return _mm_mask_i64gather_epi64(__m128i(), reinterpret_cast<const long long*>(data_ptr), i.to_register(), grow<2>(m).to_register(), sizeof(_Tp)); },
718 [=](ymm_register<std::int64_t> auto i, auto m)
719 { return _mm256_mask_i64gather_epi64(__m256i(), reinterpret_cast<const long long*>(data_ptr), i.to_register(), grow<4>(m).to_register(), sizeof(_Tp)); },
720
721#endif
722
723 [=](auto unhandled, auto) { static_assert(dependent_false<decltype(unhandled)>); }
724 };
725
726 // The indexes and the values to gether must be the same size, and at least
727 // 32-bit signed to match the hardware expectation.
728 constexpr auto idxNumBytes = std::max({sizeof(std::int32_t), sizeof(_Tp), sizeof(_Idx)});
729 using _IW = std::make_signed_t<container_for_num_bytes<idxNumBytes>>;
730 const auto rmax = std::min<_IW>(std::ranges::size(r), std::numeric_limits<_Idx>::max());
731
732 auto wrapper = [=]<typename _VecIdx>(_VecIdx idx, auto m) {
733 using RetypedIndex = rebind_t<_IW, _VecIdx>;
734 auto ri = RetypedIndex(idx);
735
736 // Note that the mask is actively resized to zero out unwanted positions.
737 // Converting to the register in the builtin will merely leave unused
738 // positions uninitialized, which might make them active with an invalid
739 // index.
740 auto rm = typename RetypedIndex::mask_type(m);
741
742 // Add a bounds check if necessary.
743 if constexpr (!contains_flag<flag_unchecked>(flags))
744 rm = rm && (ri >= _IW(0) && ri < rmax);
745
746 // Do the gather, reading back elements which are at least as big as the required type, and then downcasting to the types size if necessary.
747 auto t = RetypedIndex(impl(ri, rm));
748 auto r = vec<container_for_type<_Tp>, _VecIdx::size>(t);
749
750 // Return the values in their proper type form.
751 return simd_bit_cast<_Tp>(r);
752 };
753
755 checkStaticMemoryBounds("gather", indexes, rmax, mask);
756
757 return chunked_invoke<int(vec<_IW>::size)>(wrapper, indexes, mask);
758}
759
760template<detail::contiguous_sized_range _Range, vec_integral _Idx, typename... _Flags>
761constexpr auto gather_from(x86_avx2_tag, const _Range& r,
762 const _Idx& indexes, flags<_Flags...> flags)
763{
764 // Forward to the normal gather. All gathers are masked anyway (i.e., the
765 // compiler will create a mask anyway), so this doesn't make much difference.
766 constexpr auto m = typename _Idx::mask_type(true);
767 return partial_gather_from(r, m, indexes, flags);
768}
769
770#if defined(__AVX512F__)
771
772inline auto scatterImpl =
773 []<typename _Tp, typename _VT, typename _IT>(basic_vec<_Tp, _VT> v,
774 basic_vec<std::int32_t, _IT> idx, auto ptr, std::uint64_t m)
775{
776 constexpr std::uint64_t justMaskBits = (1ULL << basic_vec<_Tp, _VT>::size()) - 1;
777 const std::uint64_t nm = justMaskBits & m;
778
779 constexpr bool isTp32 = (sizeof(_Tp) == 4);
780
781 if constexpr (xmm_register<basic_vec<_Tp, _VT>>)
782 {
783 if constexpr (isTp32) _mm_mask_i32scatter_epi32(ptr, nm, idx.to_register(), v.to_register(), 4);
784 else _mm_mask_i32scatter_epi64(ptr, nm, idx.to_register(), v.to_register(), 8);
785 }
786 else if constexpr (ymm_register<basic_vec<_Tp, _VT>>)
787 {
788 if constexpr (isTp32) _mm256_mask_i32scatter_epi32(ptr, nm, idx.to_register(), v.to_register(), 4);
789 else _mm256_mask_i32scatter_epi64(ptr, nm, idx.to_register(), v.to_register(), 8);
790 }
791 else
792 {
793 if constexpr (isTp32) _mm512_mask_i32scatter_epi32(ptr, nm, idx.to_register(), v.to_register(), 4);
794 else _mm512_mask_i32scatter_epi64(ptr, nm, idx.to_register(), v.to_register(), 8);
795 }
796};
797
799template<typename _Range, typename _Tp, typename _TpAbi, typename _Idx, typename _IdxAbi, typename... _Flags>
800requires (sizeof(_Tp) == 4 || sizeof(_Tp) == 8) // Only 32/64-bit writes are natively supported in x86. For smaller or larger elements the
801 // generic loop implementation is faster.
802constexpr void
803scatter_to(x86_avx512_tag, const basic_vec<_Tp, _TpAbi>& values, _Range&& r,
804 const basic_vec<_Idx, _IdxAbi>& indexes, flags<_Flags...> flags)
805{
806 auto ptr = std::ranges::data(r);
807 const auto rmax = std::int32_t(std::min(std::ranges::size(r), std::size_t(std::numeric_limits<_Idx>::max())));
808
809 // Bind extra pointer and default mask parameters to the scatter lambda. Note
810 // that a range check is added if necessary too.
811 auto impl = [=](auto v, auto i) {
812 if constexpr (contains_flag<detail::flag_unchecked>(flags))
813 scatterImpl(v, i, ptr, ~0ULL);
814 else
815 scatterImpl(v, i, ptr, (i >= 0 && i < rmax).to_register());
816 };
817
818 if constexpr (contains_flag<flag_unchecked>(flags))
819 checkStaticMemoryBounds("scatter", indexes, rmax);
820
821 // All scatter instructions can be made to use 32-bit indexes.
822 // :TODO: A special case might be 64-bit values using 64-bit indexes
823 auto indexAs32 = vec<std::int32_t, basic_vec<_Idx, _IdxAbi>::size>(indexes);
824 chunked_invoke<(int)basic_vec<_Tp>::size>(impl, vec_as_container(values), indexAs32);
825}
826
827template<typename _Range, typename _Tp, typename _TpAbi, typename _Idx, typename _IdxAbi, typename... _Flags>
828requires (sizeof(_Tp) == 4 || sizeof(_Tp) == 8) // Only 32/64-bit writes are natively supported in x86. For smaller or larger elements the
829 // generic loop implementation is faster.
830constexpr void
831scatter_to(x86_avx512_tag, const basic_vec<_Tp, _TpAbi>& values, _Range&& r,
832 const typename basic_vec<_Idx, _IdxAbi>::mask_type& mask,
833 const basic_vec<_Idx, _IdxAbi>& indexes, flags<_Flags...> flags)
834{
835 auto ptr = std::ranges::data(r);
836
837 // Range check if required.
838 const auto rmax = _Idx(std::min(std::ranges::size(r), std::size_t(std::numeric_limits<_Idx>::max())));
839 auto rangeCheckedMask =
840 contains_flag<detail::flag_unchecked>(flags) ? mask : (indexes >= _Idx(0) && indexes < rmax) && mask;
841
842 if constexpr (contains_flag<flag_unchecked>(flags))
843 checkStaticMemoryBounds("scatter", indexes, rmax, mask);
844
845 // Bind an extra pointer parameter to the scatter lambda
846 auto impl = [=](auto v, auto i, auto m) { scatterImpl(v, i, ptr, m.to_ullong()); };
847
848 // All scatter instructions can be made to use 32-bit indexes.
849 // :TODO: A special case might be 64-bit values using 64-bit indexes
850 auto indexAs32 = vec<std::int32_t, basic_vec<_Idx, _IdxAbi>::size>(indexes);
851 chunked_invoke<(int)basic_vec<_Tp>::size()>(impl, vec_as_container(values), indexAs32, rangeCheckedMask);
852}
853#endif
854
855// x86 binary operators to generate a mask using the underlying compiler builtin type.
856template<typename _Tp, typename _Abi, mask_operator _Op> // Concept - binary operator of some type. :TODO: differentiate between binary and unary?
857constexpr auto builtin_operator(x86_avx512_tag, basic_vec<_Tp, _Abi> x, basic_vec<_Tp, _Abi> y, _Op)
858{
859 using binop_type = std::remove_cvref_t<_Op>;
860 using value_type = std::remove_cvref_t<_Tp>;
861
862 // Dispatch to the correct operation.
863 constexpr bool is_int = std::is_integral_v<value_type> || std::is_enum_v<value_type>;
864 constexpr auto EQ_OP = is_int ? _MM_CMPINT_EQ : _CMP_EQ_OQ;
865 constexpr auto NEQ_OP = is_int ? _MM_CMPINT_NE : _CMP_NEQ_OQ;
866 constexpr auto LT_OP = is_int ? _MM_CMPINT_LT : _CMP_LT_OQ;
867 constexpr auto LE_OP = is_int ? _MM_CMPINT_LE : _CMP_LE_OQ;
868 constexpr auto GT_OP = is_int ? _MM_CMPINT_NLE : _CMP_GT_OQ;
869 constexpr auto GE_OP = is_int ? _MM_CMPINT_NLT : _CMP_GE_OQ;
870
871 constexpr auto op = (std::is_same_v<binop_type, std::equal_to<>>) ? EQ_OP :
872 (std::is_same_v<binop_type, std::not_equal_to<>>) ? NEQ_OP :
873 (std::is_same_v<binop_type, std::less<>>) ? LT_OP :
874 (std::is_same_v<binop_type, std::less_equal<>>) ? LE_OP :
875 (std::is_same_v<binop_type, std::greater<>>) ? GT_OP :
876 GE_OP;
877
878 static_assert(op != GE_OP || std::is_same_v<binop_type, std::greater_equal<>>, "Unknown binary operation");
879
880 auto impl = [=](auto lhs, auto rhs) {
881
882 constexpr bool isSigned = std::is_signed_v<value_type>;
883
884 auto lhsr = lhs.to_register();
885 auto rhsr = rhs.to_register();
886
887 if constexpr (is_int)
888 {
889 if constexpr (sizeof(value_type) == 1) {
890 if constexpr (sizeof(lhs) <= 16) return isSigned ? _mm_cmp_epi8_mask(lhsr, rhsr, op) : _mm_cmp_epu8_mask(lhsr, rhsr, op);
891 else if constexpr (sizeof(lhs) <= 32) return isSigned ? _mm256_cmp_epi8_mask(lhsr, rhsr, op) : _mm256_cmp_epu8_mask(lhsr, rhsr, op);
892 else if constexpr (sizeof(lhs) <= 64) return isSigned ? _mm512_cmp_epi8_mask(lhsr, rhsr, op) : _mm512_cmp_epu8_mask(lhsr, rhsr, op);
893 }
894 else if constexpr (sizeof(value_type) == 2) {
895 if constexpr (sizeof(lhs) <= 16) return isSigned ? _mm_cmp_epi16_mask(lhsr, rhsr, op) : _mm_cmp_epu16_mask(lhsr, rhsr, op);
896 else if constexpr (sizeof(lhs) <= 32) return isSigned ? _mm256_cmp_epi16_mask(lhsr, rhsr, op) : _mm256_cmp_epu16_mask(lhsr, rhsr, op);
897 else if constexpr (sizeof(lhs) <= 64) return isSigned ? _mm512_cmp_epi16_mask(lhsr, rhsr, op) : _mm512_cmp_epu16_mask(lhsr, rhsr, op);
898 }
899 else if constexpr (sizeof(value_type) == 4) {
900 if constexpr (sizeof(lhs) <= 16) return isSigned ? _mm_cmp_epi32_mask(lhsr, rhsr, op) : _mm_cmp_epu32_mask(lhsr, rhsr, op);
901 else if constexpr (sizeof(lhs) <= 32) return isSigned ? _mm256_cmp_epi32_mask(lhsr, rhsr, op) : _mm256_cmp_epu32_mask(lhsr, rhsr, op);
902 else if constexpr (sizeof(lhs) <= 64) return isSigned ? _mm512_cmp_epi32_mask(lhsr, rhsr, op) : _mm512_cmp_epu32_mask(lhsr, rhsr, op);
903 }
904 if constexpr (sizeof(value_type) == 8) {
905 if constexpr (sizeof(lhs) <= 16) return isSigned ? _mm_cmp_epi64_mask(lhsr, rhsr, op) : _mm_cmp_epu64_mask(lhsr, rhsr, op);
906 else if constexpr (sizeof(lhs) <= 32) return isSigned ? _mm256_cmp_epi64_mask(lhsr, rhsr, op) : _mm256_cmp_epu64_mask(lhsr, rhsr, op);
907 else if constexpr (sizeof(lhs) <= 64) return isSigned ? _mm512_cmp_epi64_mask(lhsr, rhsr, op) : _mm512_cmp_epu64_mask(lhsr, rhsr, op);
908 }
909 }
910 else if constexpr (is_fp16_v<value_type>)
911 {
912 #if defined (__AVX512FP16__)
913 if constexpr (sizeof(lhs) <= 16) return _mm_cmp_ph_mask(lhsr, rhsr, op);
914 else if constexpr (sizeof(lhs) <= 32) return _mm256_cmp_ph_mask(lhsr, rhsr, op);
915 else if constexpr (sizeof(lhs) <= 64) return _mm512_cmp_ph_mask(lhsr, rhsr, op);
916 #else
917 static_assert(dependent_false<decltype(lhs)>, "Emulated support for _Float16 handled below");
918 #endif
919 }
920 else if constexpr (std::is_same_v<value_type, float>)
921 {
922 if constexpr (sizeof(lhs) <= 16) return _mm_cmp_ps_mask(lhsr, rhsr, op);
923 else if constexpr (sizeof(lhs) <= 32) return _mm256_cmp_ps_mask(lhsr, rhsr, op);
924 else if constexpr (sizeof(lhs) <= 64) return _mm512_cmp_ps_mask(lhsr, rhsr, op);
925 }
926 else if constexpr (std::is_same_v<value_type, double>)
927 {
928
929 if constexpr (sizeof(lhs) <= 16) return _mm_cmp_pd_mask(lhsr, rhsr, op);
930 else if constexpr (sizeof(lhs) <= 32) return _mm256_cmp_pd_mask(lhsr, rhsr, op);
931 else if constexpr (sizeof(lhs) <= 64) return _mm512_cmp_pd_mask(lhsr, rhsr, op);
932 }
933 else
934 static_assert(dependent_false<decltype(lhs)>);
935 };
936
937 // Provide a wrapper to return the correct type of mask from whatever the
938 // intrinsic returns above. It is easiest to do that once here, than to have
939 // to make every intrinsic above do it. Same with the data member - unwrap
940 // once here.
941 if constexpr (is_fp16_v<_Tp> && !hasAvxFp16)
942 {
943 // When FP16 comparison is needed in emulation, do the comparison using their
944 // FP32 equivalents. All FP16 values can be converted to FP32 while
945 // retaining their relationships with each other.
946 auto m32 = builtin_operator(x86_avx512_tag{}, rebind_cast<float>(x), rebind_cast<float>(y), _Op{});
948 }
949 else
950 {
951 // Wrapper to put the correct return type on a plain integer from an intrinsic.
952 auto wrapper = [=](auto l, auto r) { return mask<_Tp, decltype(l.size)::value>(impl(l, r)); };
953 return chunked_invoke(wrapper, x, y);
954 }
955}
956
972template<typename _Tp, typename _Abi, mask_operator _Op> // :TODO: Or specifically equal-not-equal?
973requires (detail::is_floating_point_v<_Tp>)
974constexpr auto builtin_operator(x86_avx512_tag, basic_vec<std::complex<_Tp>, _Abi> lhs, basic_vec<std::complex<_Tp>, _Abi> rhs, _Op cmp_op)
975{
976 using binop_type = std::remove_cvref_t<_Op>;
977 constexpr bool isEqualOp = std::is_same_v<binop_type, std::equal_to<>>;
978 constexpr bool isNotEqualOp = std::is_same_v<binop_type, std::not_equal_to<>>;
979 static_assert(isEqualOp || isNotEqualOp, "Complex values only allow equality comparisons");
980
982 const auto reduceBitPair = [](auto bits) {
983 if constexpr (isEqualOp) // AND bits
984 return _mm256_cmp_epi16_mask(_mm256_movm_epi8(bits), _mm256_set1_epi16(-1), _MM_CMPINT_EQ);
985 else // OR bits
986 return _mm256_cmp_epi16_mask(_mm256_movm_epi8(bits), __m256i(), _MM_CMPINT_NE);
987 };
988
989 auto impl = [=]<typename _Vec>(_Vec x, _Vec y)
990 {
991 const auto bits = builtin_operator(x86_avx512_tag{}, simd_bit_cast<_Tp>(x), simd_bit_cast<_Tp>(y), cmp_op);
992 auto rb = reduceBitPair(bits.to_builtin());
993 return typename _Vec::mask_type(rb);
994 };
995
996 return chunked_invoke(impl, lhs, rhs);
997}
998
1000// :COMPILER: Perhaps the constants should be defined in immintrin?
1002{
1003 QUIET_NAN = 0x01,
1004 POS_ZERO = 0x02,
1005 NEG_ZERO = 0x04,
1006 POS_INF = 0x08,
1007 NEG_INF = 0x10,
1008 DENORM = 0x20,
1009 NEG = 0x40,
1010 SIGNAL_NAN = 0x80
1011};
1012
1022template<int _Op, typename _Tp, typename _Abi>
1024
1025 target_overloads impl {
1026#if defined (__AVX512FP16__)
1027 // GCC has a bug in its implementation of the fpclass intrinsic so use the
1028 // underlying builtin directly. Clang doesn't care which implementation is
1029 // used so there is no need to have gcc-specific conditional code.
1030 [=]<xmm_register<_Float16> _Vec>(_Vec v) -> __mmask8 {
1031 return __builtin_ia32_fpclassph128_mask(v.to_register(), _Op, (__mmask64(1) << _Vec::size) - 1);
1032 },
1033 [=]<ymm_register<_Float16> _Vec>(_Vec v) -> __mmask16 {
1034 return __builtin_ia32_fpclassph256_mask(v.to_register(), _Op, (__mmask64(1) << _Vec::size) - 1);
1035 },
1036 [=]<zmm_register<_Float16> _Vec>(_Vec v) -> __mmask32 {
1037 return __builtin_ia32_fpclassph512_mask(v.to_register(), _Op, (__mmask64(1) << _Vec::size) - 1);
1038 },
1039#endif
1040 [=](xmm_register<float> auto v) { return _mm_fpclass_ps_mask(v.to_register(), _Op); },
1041 [=](ymm_register<float> auto v) { return _mm256_fpclass_ps_mask(v.to_register(), _Op); },
1042
1043 [=](xmm_register<double> auto v) { return _mm_fpclass_pd_mask(v.to_register(), _Op); },
1044 [=](ymm_register<double> auto v) { return _mm256_fpclass_pd_mask(v.to_register(), _Op); },
1045
1046 [=](zmm_register<float> auto v) { return _mm512_fpclass_ps_mask(v.to_register(), _Op); },
1047 [=](zmm_register<double> auto v) { return _mm512_fpclass_pd_mask(v.to_register(), _Op); },
1048
1049 [=](auto unhandled) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); return 0; }
1050 };
1051
1052 // Put a suitable mask around the bare integer from impl and then invoke.
1053 auto wrapper = [=](auto _vec) {
1054 auto dwt = impl(_vec);
1055 return mask<_Tp, decltype(_vec.size)::value>(dwt);
1056 };
1057 return chunked_invoke(wrapper, value);
1058}
1059
1063template<typename _Tp> constexpr bool has_native_support_for_classify =
1064 detail::is_floating_point_v<_Tp> && (hasAvxFp16 || !is_fp16_v<_Tp>);
1065
1067template<typename _Tp, typename _Abi>
1069constexpr typename basic_vec<_Tp, _Abi>::mask_type
1074
1075template<typename _Tp, typename _Abi>
1076requires (has_native_support_for_classify<_Tp>)
1077constexpr typename basic_vec<_Tp, _Abi>::mask_type
1082
1083template<typename _Tp, typename _Abi>
1084requires (has_native_support_for_classify<_Tp>)
1085constexpr typename basic_vec<_Tp, _Abi>::mask_type
1086isfinite(x86_avx512_tag, const basic_vec<_Tp, _Abi>& v)
1087{
1088 constexpr int invalidClasses = X86_FP_CLASSIFY::QUIET_NAN | X86_FP_CLASSIFY::SIGNAL_NAN | X86_FP_CLASSIFY::POS_INF | X86_FP_CLASSIFY::NEG_INF;
1090}
1091
1092template<typename _Tp, typename _Abi>
1093requires (has_native_support_for_classify<_Tp>)
1094constexpr typename basic_vec<_Tp, _Abi>::mask_type
1095isnormal(x86_avx512_tag, const basic_vec<_Tp, _Abi>& v)
1096{
1097 constexpr int invalidClasses =
1098 X86_FP_CLASSIFY::QUIET_NAN | X86_FP_CLASSIFY::SIGNAL_NAN | X86_FP_CLASSIFY::POS_INF | X86_FP_CLASSIFY::NEG_INF |
1099 X86_FP_CLASSIFY::POS_ZERO | X86_FP_CLASSIFY::NEG_ZERO | X86_FP_CLASSIFY::DENORM;
1101}
1102
1103#if defined(__AVX512F__)
1104template<std::integral _Up, std::integral _Tp, typename _Abi>
1105requires (std::signed_integral<_Up> == std::signed_integral<_Tp> && // Only same-sign conversions have ISA support
1106 sizeof(_Up) < sizeof(_Tp)) // Type must be getting smaller. Same size or bigger can be handled by generic cast
1107constexpr auto
1108saturating_cast(x86_avx512_tag, const basic_vec<_Tp, _Abi>& value) noexcept
1109{
1110 target_overloads impl {
1111 // S16 -> S8
1112 [=](xmm_register<std::int16_t> auto in, std::int8_t) { return _mm_cvtsepi16_epi8(in.to_register()); },
1113 [=](ymm_register<std::int16_t> auto in, std::int8_t) { return _mm256_cvtsepi16_epi8(in.to_register()); },
1114 [=](zmm_register<std::int16_t> auto in, std::int8_t) { return _mm512_cvtsepi16_epi8(in.to_register()); },
1115
1116 // U16 -> U8
1117 [=](xmm_register<std::uint16_t> auto in, std::uint8_t) { return _mm_cvtusepi16_epi8(in.to_register()); },
1118 [=](ymm_register<std::uint16_t> auto in, std::uint8_t) { return _mm256_cvtusepi16_epi8(in.to_register()); },
1119 [=](zmm_register<std::uint16_t> auto in, std::uint8_t) { return _mm512_cvtusepi16_epi8(in.to_register()); },
1120
1121 // S32 -> S8
1122 [=](xmm_register<std::int32_t> auto in, std::int8_t) { return _mm_cvtsepi32_epi8(in.to_register()); },
1123 [=](ymm_register<std::int32_t> auto in, std::int8_t) { return _mm256_cvtsepi32_epi8(in.to_register()); },
1124 [=](zmm_register<std::int32_t> auto in, std::int8_t) { return _mm512_cvtsepi32_epi8(in.to_register()); },
1125
1126 // U32 -> U8
1127 [=](xmm_register<std::uint32_t> auto in, std::uint8_t) { return _mm_cvtusepi32_epi8(in.to_register()); },
1128 [=](ymm_register<std::uint32_t> auto in, std::uint8_t) { return _mm256_cvtusepi32_epi8(in.to_register()); },
1129 [=](zmm_register<std::uint32_t> auto in, std::uint8_t) { return _mm512_cvtusepi32_epi8(in.to_register()); },
1130
1131 // S32 -> S16
1132 [=](xmm_register<std::int32_t> auto in, std::int16_t) { return _mm_cvtsepi32_epi16(in.to_register()); },
1133 [=](ymm_register<std::int32_t> auto in, std::int16_t) { return _mm256_cvtsepi32_epi16(in.to_register()); },
1134 [=](zmm_register<std::int32_t> auto in, std::int16_t) { return _mm512_cvtsepi32_epi16(in.to_register()); },
1135
1136 // U32 -> U16
1137 [=](xmm_register<std::uint32_t> auto in, std::uint16_t) { return _mm_cvtusepi32_epi16(in.to_register()); },
1138 [=](ymm_register<std::uint32_t> auto in, std::uint16_t) { return _mm256_cvtusepi32_epi16(in.to_register()); },
1139 [=](zmm_register<std::uint32_t> auto in, std::uint16_t) { return _mm512_cvtusepi32_epi16(in.to_register()); },
1140
1141 // S64 -> S8
1142 [=](xmm_register<std::int64_t> auto in, std::int8_t) { return _mm_cvtsepi64_epi8(in.to_register()); },
1143 [=](ymm_register<std::int64_t> auto in, std::int8_t) { return _mm256_cvtsepi64_epi8(in.to_register()); },
1144 [=](zmm_register<std::int64_t> auto in, std::int8_t) { return _mm512_cvtsepi64_epi8(in.to_register()); },
1145
1146 // U64 -> U8
1147 [=](xmm_register<std::uint64_t> auto in, std::uint8_t) { return _mm_cvtusepi64_epi8(in.to_register()); },
1148 [=](ymm_register<std::uint64_t> auto in, std::uint8_t) { return _mm256_cvtusepi64_epi8(in.to_register()); },
1149 [=](zmm_register<std::uint64_t> auto in, std::uint8_t) { return _mm512_cvtusepi64_epi8(in.to_register()); },
1150
1151 // S64 -> S16
1152 [=](xmm_register<std::int64_t> auto in, std::int16_t) { return _mm_cvtsepi64_epi16(in.to_register()); },
1153 [=](ymm_register<std::int64_t> auto in, std::int16_t) { return _mm256_cvtsepi64_epi16(in.to_register()); },
1154 [=](zmm_register<std::int64_t> auto in, std::int16_t) { return _mm512_cvtsepi64_epi16(in.to_register()); },
1155
1156 // U64 -> U16
1157 [=](xmm_register<std::uint64_t> auto in, std::uint16_t) { return _mm_cvtusepi64_epi16(in.to_register()); },
1158 [=](ymm_register<std::uint64_t> auto in, std::uint16_t) { return _mm256_cvtusepi64_epi16(in.to_register()); },
1159 [=](zmm_register<std::uint64_t> auto in, std::uint16_t) { return _mm512_cvtusepi64_epi16(in.to_register()); },
1160
1161 // S64 -> S32
1162 [=](xmm_register<std::int64_t> auto in, std::int32_t) { return _mm_cvtsepi64_epi32(in.to_register()); },
1163 [=](ymm_register<std::int64_t> auto in, std::int32_t) { return _mm256_cvtsepi64_epi32(in.to_register()); },
1164 [=](zmm_register<std::int64_t> auto in, std::int32_t) { return _mm512_cvtsepi64_epi32(in.to_register()); },
1165
1166 // U64 -> U32
1167 [=](xmm_register<std::uint64_t> auto in, std::uint32_t) { return _mm_cvtusepi64_epi32(in.to_register()); },
1168 [=](ymm_register<std::uint64_t> auto in, std::uint32_t) { return _mm256_cvtusepi64_epi32(in.to_register()); },
1169 [=](zmm_register<std::uint64_t> auto in, std::uint32_t) { return _mm512_cvtusepi64_epi32(in.to_register()); },
1170
1171 [=](auto unhandled, auto out)
1172 { static_assert(dependent_false<std::pair<decltype(unhandled), decltype(out)>>, "No target overload"); return out;}
1173 };
1174
1175 auto wrapper = [=]<typename _VT, typename _VAbi>(basic_vec<_VT, _VAbi> v) {
1176 auto r = impl(v, _Up()); // Pass in dummy parameter to be able to use overloading.
1177 return vec<_Up, basic_vec<_VT, _VAbi>::size>(r);
1178 };
1179 return chunked_invoke(wrapper, value);
1180}
1181#endif
1182
1185inline __m128i shuffle_epi8(__m128i values, __m128i indexes) { return _mm_shuffle_epi8(values, indexes); }
1186
1187inline __m256i shuffle_epi8(__m256i values, __m256i indexes) {
1188#if defined (__AVX2__)
1189 return _mm256_shuffle_epi8(values, indexes);
1190#else
1191 // Intel AVX doesn't have a 256-bit shuffle so handle it using a pair of
1192 // 128-bit shuffles instead. Note that users of this function might consider
1193 // rewriting the call to improve efficiency instead.
1194 const auto lowerShuf = _mm_shuffle_epi8(_mm256_castsi256_si128(values), _mm256_castsi256_si128(indexes));
1195 const auto upperShuf = _mm_shuffle_epi8(_mm256_extractf128_si256(values, 1), _mm256_extractf128_si256(indexes, 1));
1196 return _mm256_insertf128_si256(_mm256_castsi128_si256(lowerShuf), upperShuf, 1);
1197#endif
1198}
1199
1200#if defined(__AVX512F__)
1201inline __m512i shuffle_epi8(__m512i values, __m512i indexes) { return _mm512_shuffle_epi8(values, indexes); }
1202#endif
1203
1212template<typename _Gen, typename _Abi>
1213constexpr basic_vec<std::uint8_t, _Abi>
1215{
1216 auto rawIndexes = indexes.to_register();
1217
1218 // Build a LUT which contains as many lanes as the indexes.
1219 constexpr auto lut = vec<std::uint8_t, sizeof(rawIndexes)>([](auto idx) { return _Gen{}(idx & 0xF); });
1220
1221 // Use the fast in-lane byte shuffle.
1222 return basic_vec<std::uint8_t, _Abi>(shuffle_epi8(lut.to_register(), rawIndexes));
1223}
1224
1226
1231template<typename _Tp, typename _Abi>
1232constexpr basic_vec<_Tp, _Abi> popcount(x86_sse_tag, const basic_vec<_Tp, _Abi>& x) noexcept {
1233
1234 const auto bitsInByte = []<typename _Vec>(_Vec piece) -> _Vec {
1235 const auto bytes = simd_bit_cast<std::uint8_t>(piece);
1236
1237 // Use a lookup table to compute the population count for 4-bits at a time
1238 // for lower and upper halves of a byte, then add together.
1239 constexpr auto gen = [](auto idx) -> std::uint8_t { return std::popcount(std::uint8_t(idx & 0xF)); };
1240
1241 const auto lowPopCount = shuffle_epi8_lut(gen, bytes & cw<0xF>);
1242 const auto highPopCount = shuffle_epi8_lut(gen, bytes >> 4);
1243
1244 return simd_bit_cast<_Tp>(lowPopCount + highPopCount);
1245 };
1246
1247 // Compute the population counts for the individual bytes.
1248 auto pc = chunked_invoke(bitsInByte, x);
1249
1250 if constexpr (sizeof(_Tp) == 1) return pc;
1251
1252 // Sum adjacent bytes for larger values.
1253 if constexpr (sizeof(_Tp) == 8) pc += (pc >> 32);
1254 if constexpr (sizeof(_Tp) >= 4) pc += (pc >> 16);
1255 if constexpr (sizeof(_Tp) >= 2) pc += (pc >> 8);
1256
1257 // Only the lowest byte of each element contains the correct count. The other bytes contain left-over debris.
1258 return pc & cw<0xFF>;
1259}
1260
1261#if defined(__AVX512VPOPCNTDQ__)
1262// Sunny Cove has native population count instructions.
1263template<typename _Tp, typename _Abi>
1264constexpr basic_vec<_Tp, _Abi> popcount(x86_avxsnc_tag, const basic_vec<_Tp, _Abi>& x) noexcept {
1265 target_overloads impl {
1266 [=](xmm_register<std::uint8_t> auto v) { return _mm_popcnt_epi8(v.to_register()); },
1267 [=](xmm_register<std::uint16_t> auto v) { return _mm_popcnt_epi16(v.to_register()); },
1268 [=](xmm_register<std::uint32_t> auto v) { return _mm_popcnt_epi32(v.to_register()); },
1269 [=](xmm_register<std::uint64_t> auto v) { return _mm_popcnt_epi64(v.to_register()); },
1270
1271 [=](ymm_register<std::uint8_t> auto v) { return _mm256_popcnt_epi8(v.to_register()); },
1272 [=](ymm_register<std::uint16_t> auto v) { return _mm256_popcnt_epi16(v.to_register()); },
1273 [=](ymm_register<std::uint32_t> auto v) { return _mm256_popcnt_epi32(v.to_register()); },
1274 [=](ymm_register<std::uint64_t> auto v) { return _mm256_popcnt_epi64(v.to_register()); },
1275
1276 [=](zmm_register<std::uint8_t> auto v) { return _mm512_popcnt_epi8(v.to_register()); },
1277 [=](zmm_register<std::uint16_t> auto v) { return _mm512_popcnt_epi16(v.to_register()); },
1278 [=](zmm_register<std::uint32_t> auto v) { return _mm512_popcnt_epi32(v.to_register()); },
1279 [=](zmm_register<std::uint64_t> auto v) { return _mm512_popcnt_epi64(v.to_register()); },
1280
1281 [=](auto unhandled) -> decltype(unhandled) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); }
1282 };
1283
1284 // Call inside wrapper which converts intrinsic into vec value.
1285 return chunked_invoke([=](auto v) { return decltype(v)(impl(v)); }, x);
1286}
1287#endif // __AVX512VPOPCNTDQ__
1288
1293template<typename _Tp, typename _Abi>
1294constexpr auto has_single_bit(x86_avxsnc_tag, const basic_vec<_Tp, _Abi>& x) noexcept {
1295 return popcount(x) == cw<1>;
1296}
1297
1302template<std::unsigned_integral _Tp, typename _Abi>
1303constexpr basic_vec<_Tp, _Abi> clz(x86_sse_tag, const basic_vec<_Tp, _Abi>& x) noexcept {
1304
1305 // Utility to compute clz for individual bytes using table lookup.
1306 const auto clzByte = []<typename _Vec>(_Vec piece) -> _Vec
1307 {
1308 const auto bytes = simd_bit_cast<std::uint8_t>(piece);
1309
1310 // Compute the 4-bit low and high clzs for each 8-bit value. Note that
1311 // shuffle uses a dirty index where the high bit means zero the output, so
1312 // the top bits must normally be masked away. There is no need to handle
1313 // that special case though because if the MSB is set then the select below
1314 // will remove it anyway. Note also that constant 4 is added to the low clz
1315 // to account for it being the bottom 4 bits to start with. The 4 could have
1316 // been added to the result of the clz but by putting it on the values to
1317 // lookup the compiler can hoist it.
1318 const auto lowClz =
1319 shuffle_epi8_lut([](auto idx) -> std::uint8_t { return std::countl_zero(std::uint8_t(idx & 0xF)); }, bytes & std::uint8_t(0xF));
1320 const auto highClz =
1321 shuffle_epi8_lut([](auto idx) -> std::uint8_t { return std::countl_zero(std::uint8_t(idx & 0xF)) - 4; }, bytes >> std::uint8_t(4));
1322
1323 // Select which clz actually applies here.
1324 return simd_bit_cast<_Tp>(select(bytes > std::uint8_t(15), highClz, lowClz));
1325 };
1326
1327 constexpr auto numElementBytes = sizeof(_Tp);
1328
1329 if constexpr (numElementBytes == 1)
1330 return chunked_invoke(clzByte, x);
1331 else if constexpr (numElementBytes == 2)
1332 {
1333 // Use the 8-bit variant to build the 16-bit variant.
1334 const auto clz8 = chunked_invoke(clzByte, x);
1335 // If there are any bits in the upper byte use the top byte, otherwise use
1336 // the bottom byte with an extra 8 added to account for the empty top bits.
1337 return select(x > cw<0xFF>, clz8 >> 8, (clz8 & cw<0xFF>) + cw<8>);
1338 }
1339 else if constexpr (numElementBytes == 4)
1340 {
1341 // Use the FP conversion trick described in Hackers Delight, page 82.
1342 // Floating-point format essentially stores the index of the first
1343 // leading bit in the mantissa so this can be extracted after converting
1344 // the number in question to FP. Care must be taken with rounding and
1345 // the special case of 0. Note also that a conversion from unsigned to
1346 // float is unsupported in Intel AVX and the compiler's synthesis is quite
1347 // expensive. It is cheaper to pretend it is signed int and test for the
1348 // special case of the MSB being set and blend in the correct answer
1349 // than do the conversion.
1350 const auto keepTopBit = rebind_cast<std::int32_t>(x & ~(x >> 1));
1351 const auto fp = rebind_cast<float>(keepTopBit) + 0.5f;
1352
1353 const auto t = (cw<158> - (simd_bit_cast<std::uint32_t>(fp) >> 23));
1354 const auto mt = min(t, basic_vec<_Tp, _Abi>(cw<32>));
1355
1356 return select((x & 0x80000000) == cw<0>, mt, basic_vec<_Tp, _Abi>());
1357 }
1358 else if constexpr (numElementBytes == 8)
1359 {
1360 // Use the 32-bit variant to build the 64-bit variant.
1361 const auto clz32 = countl_zero(simd_bit_cast<std::uint32_t>(x));
1362
1363 // Treat the individual clz values as parts of a 64-bit value.
1364 const auto tas64 = simd_bit_cast<std::uint64_t>(clz32);
1365
1366 // If there are any bits in the upper word use that count. If the top word
1367 // is empty use whatever the bottom word's count is but compensate for the
1368 // empty top word by adding another 32.
1369 return select(x > 0xFFFFFFFF, tas64 >> 32, (tas64 & 0xFFFFFFFF) + cw<32>);
1370
1371 // This is an alternative for machines which have fast std::uint64_t -> double conversion (which Intel AVX or Intel SSE don't have).
1372 // const auto keepTopBit = v & ~(v >> 1);
1373 // const auto fp = rebind_cast<double>(rebind_cast<int64_t>(keepTopBit)) + 0.5;
1374 // return select((v & 0x8000000000000000) != 0, 0, min(64, 1086 - (simd_bit_cast<std::uint64_t>(fp) >> 52)));
1375 }
1376 else
1377 static_assert(dependent_false<_Tp>, "Unsupported data type for clz");
1378}
1379
1384#if defined(__AVX512F__)
1385template<typename _Tp, typename _Abi>
1386requires (sizeof(_Tp) == 4 || sizeof(_Tp) == 8) // Only works on 4 or 8 byte values
1387constexpr basic_vec<_Tp, _Abi> clz(x86_avx512_tag, const basic_vec<_Tp, _Abi>& x) noexcept {
1388 target_overloads impl {
1389 [=](xmm_register<std::uint32_t> auto v) { return _mm_lzcnt_epi32(v.to_register()); },
1390 [=](xmm_register<std::uint64_t> auto v) { return _mm_lzcnt_epi64(v.to_register()); },
1391
1392 [=](ymm_register<std::uint32_t> auto v) { return _mm256_lzcnt_epi32(v.to_register()); },
1393 [=](ymm_register<std::uint64_t> auto v) { return _mm256_lzcnt_epi64(v.to_register()); },
1394
1395 [=](zmm_register<std::uint32_t> auto v) { return _mm512_lzcnt_epi32(v.to_register()); },
1396 [=](zmm_register<std::uint64_t> auto v) { return _mm512_lzcnt_epi64(v.to_register()); },
1397
1398 [=](auto unhandled) -> decltype(unhandled) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); }
1399 };
1400
1401 return chunked_invoke([=](auto v) { return decltype(v)(impl(v)); }, x);
1402}
1403#endif
1404
1405// :TODO: __AVX512VBMI2__ could handle byte and short by an epi8 double-register lookup.
1406
1411template<typename _Tp, typename _Abi>
1412requires (sizeof(_Tp) == 1) // Multi-byte reversal handled generically.
1413constexpr basic_vec<_Tp, _Abi> bitreverse(x86_sse_tag, const basic_vec<_Tp, _Abi>& x) noexcept {
1414
1415 // Reverse the bits within individual bytes.
1416 const auto byteBitReverser = []<typename _Vec>(_Vec piece) -> _Vec {
1417 const auto bytes = simd_bit_cast<std::uint8_t>(piece);
1418
1419 // Reverse upper and lower 4-bit blocks using a lut, and then combine them
1420 // together. Note that the lookup tables contain the data shifted into the
1421 // correct nibble, to avoid having to do that dynamically.
1422 constexpr std::uint8_t _revbytes[16] = {0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe, 0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf};
1423 const auto lowRev = shuffle_epi8_lut([](auto idx) -> std::uint8_t { return _revbytes[idx % 16] << 4; }, bytes & cw<0xF>);
1424 const auto highRev = shuffle_epi8_lut([](auto idx) -> std::uint8_t { return _revbytes[idx % 16]; }, bytes >> 4);
1425
1426 return _Vec(lowRev | highRev);
1427 };
1428
1429 return chunked_invoke(byteBitReverser, x);
1430}
1431
1436template<vec_integral _Vp>
1437requires (sizeof(typename _Vp::value_type) == 1) // Multi-byte reversal handled generically.
1438constexpr _Vp bitreverse(x86_avxsnc_tag, const _Vp& x) noexcept
1439 { return x86::gf2p8affine(x, std::uint64_t(0x8040201008040201)); }
1440
1441template<typename _Tp, typename _Abi>
1442requires (detail::is_floating_point_v<_Tp>)
1443constexpr basic_vec<_Tp, _Abi> rcp(x86_sse_tag, const basic_vec<_Tp, _Abi>& v)
1444{
1445 target_overloads impl {
1446 [=](xmm_register<float> auto x) { return _mm_rcp_ps(x.to_register());},
1447
1448#if defined(__AVX__)
1449 [=](ymm_register<float> auto x) { return _mm256_rcp_ps(x.to_register());},
1450#endif
1451
1452 // Generic version for unhandled cases.
1453 [=](auto x) { return _Tp(1) / x; }
1454 };
1455
1456 return chunked_invoke([=](auto v) { return decltype(v)(impl(v)); }, v);
1457}
1458
1459#if defined(__AVX512F__)
1460template<std::floating_point _Tp, typename _Abi>
1461constexpr basic_vec<_Tp, _Abi> rcp(x86_avx512_tag, const basic_vec<_Tp, _Abi>& v)
1462{
1463 target_overloads impl {
1464 [=](xmm_register<float> auto x) { return _mm_rcp14_ps(x.to_register());},
1465 [=](ymm_register<float> auto x) { return _mm256_rcp14_ps(x.to_register());},
1466
1467 [=](xmm_register<double> auto x) { return _mm_rcp14_pd(x.to_register());},
1468 [=](ymm_register<double> auto x) { return _mm256_rcp14_pd(x.to_register());},
1469
1470 [=](zmm_register<float> auto x) { return _mm512_rcp14_ps(x.to_register());},
1471 [=](zmm_register<double> auto x) { return _mm512_rcp14_pd(x.to_register());},
1472 };
1473
1474 return chunked_invoke([=](auto v) { return decltype(v)(impl(v)); }, v);
1475}
1476#endif
1477
1478#if defined(__AVX512FP16__)
1479template<typename _Abi>
1480constexpr basic_vec<_Float16, _Abi> rcp(x86_avxspr_tag, const basic_vec<_Float16, _Abi>& v)
1481{
1482 target_overloads impl {
1483 [=](xmm_register<_Float16> auto x) { return _mm_rcp_ph(x.to_register());},
1484 [=](ymm_register<_Float16> auto x) { return _mm256_rcp_ph(x.to_register());},
1485 [=](zmm_register<_Float16> auto x) { return _mm512_rcp_ph(x.to_register());},
1486 };
1487 return chunked_invoke([=](auto v) { return decltype(v)(impl(v)); }, v);
1488}
1489#endif
1490
1491template<typename _Tp, typename _Abi>
1492requires (detail::is_floating_point_v<_Tp>)
1493constexpr basic_vec<_Tp, _Abi> rsqrt(x86_sse_tag, const basic_vec<_Tp, _Abi>& v)
1494{
1495 target_overloads impl {
1496 [=](xmm_register<float> auto x) { return _mm_rsqrt_ps(x.to_register());},
1497 [=](xmm_register<double> auto x) { return 1.0 / _mm_sqrt_pd(x.to_register());},
1498
1499#if defined(__AVX__)
1500 [=](ymm_register<float> auto x) { return _mm256_rsqrt_ps(x.to_register());},
1501 [=](ymm_register<double> auto x) { return 1.0 / _mm256_sqrt_pd(x.to_register());},
1502#endif
1503
1504 // Generic version of all other cases.
1505 [=](auto x) { return _Tp(1) / sqrt(x); }
1506 };
1507
1508 return chunked_invoke([=](auto v) { return decltype(v)(impl(v)); }, v);
1509}
1510
1511#if defined(__AVX512F__)
1512template<std::floating_point _Tp, typename _Abi>
1513constexpr basic_vec<_Tp, _Abi> rsqrt(x86_avx512_tag, const basic_vec<_Tp, _Abi>& v)
1514{
1515 target_overloads impl {
1516 [=](xmm_register<float> auto x) { return _mm_rsqrt14_ps(x.to_register());},
1517 [=](ymm_register<float> auto x) { return _mm256_rsqrt14_ps(x.to_register());},
1518
1519 [=](xmm_register<double> auto x) { return _mm_rsqrt14_pd(x.to_register());},
1520 [=](ymm_register<double> auto x) { return _mm256_rsqrt14_pd(x.to_register());},
1521
1522 [=](zmm_register<float> auto x) { return _mm512_rsqrt14_ps(x.to_register());},
1523 [=](zmm_register<double> auto x) { return _mm512_rsqrt14_pd(x.to_register());},
1524 };
1525 return chunked_invoke([=](auto v) { return decltype(v)(impl(v)); }, v);
1526}
1527#endif
1528
1529#if defined(__AVX512FP16__)
1530template<typename _Abi>
1531constexpr basic_vec<_Float16, _Abi> rsqrt(x86_avxspr_tag, const basic_vec<_Float16, _Abi>& v)
1532{
1533 target_overloads impl {
1534 [=](xmm_register<_Float16> auto x) { return _mm_rsqrt_ph(x.to_register());},
1535 [=](ymm_register<_Float16> auto x) { return _mm256_rsqrt_ph(x.to_register());},
1536 [=](zmm_register<_Float16> auto x) { return _mm512_rsqrt_ph(x.to_register());},
1537 };
1538 return chunked_invoke([=](auto v) { return decltype(v)(impl(v)); }, v);
1539}
1540#endif
1541
1542template<typename _Tp, typename _Abi>
1543requires (detail::is_floating_point_v<_Tp>)
1544constexpr basic_vec<_Tp, _Abi> sqrt(x86_sse_tag, const basic_vec<_Tp, _Abi>& v)
1545{
1546 target_overloads impl {
1547 [=](xmm_register<float> auto x) { return _mm_sqrt_ps(x.to_register());},
1548 [=](xmm_register<double> auto x) { return _mm_sqrt_pd(x.to_register());},
1549
1550#if defined(__AVX__)
1551 [=](ymm_register<float> auto x) { return _mm256_sqrt_ps(x.to_register());},
1552 [=](ymm_register<double> auto x) { return _mm256_sqrt_pd(x.to_register());},
1553#endif
1554
1555#if defined(__AVX512F__)
1556 [=](zmm_register<float> auto x) { return _mm512_sqrt_ps(x.to_register());},
1557 [=](zmm_register<double> auto x) { return _mm512_sqrt_pd(x.to_register());},
1558#endif
1559
1560#if defined(__AVX512FP16__)
1561 [=](xmm_register<_Float16> auto x) { return _mm_sqrt_ph(x.to_register());},
1562 [=](ymm_register<_Float16> auto x) { return _mm256_sqrt_ph(x.to_register());},
1563 [=](zmm_register<_Float16> auto x) { return _mm512_sqrt_ph(x.to_register());},
1564#endif
1565
1566 [=](auto unhandled) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); return unhandled; }
1567 };
1568
1569 if constexpr (is_fp16_v<_Tp> && !hasAvxFp16)
1570 {
1571 // When sqrt is needed in FP16 use FP32 to do the work.
1572 const auto sqrt32 = sqrt(rebind_cast<float>(v));
1573 return rebind_cast<_Tp>(sqrt32);
1574 }
1575 else
1576 return chunked_invoke([=](auto v) { return decltype(v)(impl(v)); }, v);
1577}
1578
1586template<std::integral _Tp, typename _Abi>
1587requires (sizeof(_Tp) <= 2) // x86 only has support for 8 and 16-bit saturating operations. Defer everything else to the
1588 // generic synthesised version.
1589constexpr basic_vec<_Tp, _Abi> saturating_add(x86_tag, const basic_vec<_Tp, _Abi>& lhs, const basic_vec<_Tp, _Abi>& rhs) noexcept
1590{
1591 target_overloads impl {
1592 // 128-bit
1593 [=](xmm_register<std::uint8_t> auto x, auto y) { return _mm_adds_epu8(x.to_register(), y.to_register()); },
1594 [=](xmm_register<std::int8_t> auto x, auto y) { return _mm_adds_epi8(x.to_register(), y.to_register()); },
1595 [=](xmm_register<std::uint16_t> auto x, auto y) { return _mm_adds_epu16(x.to_register(), y.to_register()); },
1596 [=](xmm_register<std::int16_t> auto x, auto y) { return _mm_adds_epi16(x.to_register(), y.to_register()); },
1597
1598 // 256-bit (but only in Intel AVX2 - Intel AVX has to break into 128-bit pieces instead).
1599#if defined(__AVX2__)
1600 [=](ymm_register<std::uint8_t> auto x, auto y) { return _mm256_adds_epu8(x.to_register(), y.to_register()); },
1601 [=](ymm_register<std::int8_t> auto x, auto y) { return _mm256_adds_epi8(x.to_register(), y.to_register()); },
1602 [=](ymm_register<std::uint16_t> auto x, auto y) { return _mm256_adds_epu16(x.to_register(), y.to_register()); },
1603 [=](ymm_register<std::int16_t> auto x, auto y) { return _mm256_adds_epi16(x.to_register(), y.to_register()); },
1604#endif
1605
1606 // 512-bit
1607#if defined(__AVX512F__)
1608 [=](zmm_register<std::uint8_t> auto x, auto y) { return _mm512_adds_epu8(x.to_register(), y.to_register()); },
1609 [=](zmm_register<std::int8_t> auto x, auto y) { return _mm512_adds_epi8(x.to_register(), y.to_register()); },
1610 [=](zmm_register<std::uint16_t> auto x, auto y) { return _mm512_adds_epu16(x.to_register(), y.to_register()); },
1611 [=](zmm_register<std::int16_t> auto x, auto y) { return _mm512_adds_epi16(x.to_register(), y.to_register()); },
1612 #endif
1613
1614 [=](auto unhandled, auto) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); return unhandled;}
1615 };
1616
1617 // 256-bit saturating only exists in the Intel AVX2 ISA, not Intel AVX. If
1618 // Intel AVX2 is not available break the problem into 128-bit chunks instead
1619 // to use Intel SSE ISA.
1620 constexpr int numElements = (hasAvx && !hasAvx2) ? vec<_Tp>::size / 2 : vec<_Tp>::size;
1621 auto wrapper = [=](auto a, auto b) { return decltype(a)(impl(a, b)); }; // Return a vec from an intrinsic.
1622 return chunked_invoke<numElements>(wrapper, lhs, rhs);
1623}
1624
1632template<std::integral _Tp, typename _Abi>
1633requires (sizeof(_Tp) <= 2) // x86 only has support for 8 and 16-bit saturating operations. Defer everything else to the
1634 // generic synthesised version.
1635constexpr basic_vec<_Tp, _Abi> saturating_sub(x86_tag, const basic_vec<_Tp, _Abi>& lhs, const basic_vec<_Tp, _Abi>& rhs) noexcept
1636{
1637 target_overloads impl {
1638 // 128-bit
1639 [=](xmm_register<std::uint8_t> auto x, auto y) { return _mm_subs_epu8(x.to_register(), y.to_register()); },
1640 [=](xmm_register<std::int8_t> auto x, auto y) { return _mm_subs_epi8(x.to_register(), y.to_register()); },
1641 [=](xmm_register<std::uint16_t> auto x, auto y) { return _mm_subs_epu16(x.to_register(), y.to_register()); },
1642 [=](xmm_register<std::int16_t> auto x, auto y) { return _mm_subs_epi16(x.to_register(), y.to_register()); },
1643
1644 // 256-bit (but only in Intel AVX2 - Intel AVX has to break into 128-bit pieces instead).
1645#if defined(__AVX2__)
1646 [=](ymm_register<std::uint8_t> auto x, auto y) { return _mm256_subs_epu8(x.to_register(), y.to_register()); },
1647 [=](ymm_register<std::int8_t> auto x, auto y) { return _mm256_subs_epi8(x.to_register(), y.to_register()); },
1648 [=](ymm_register<std::uint16_t> auto x, auto y) { return _mm256_subs_epu16(x.to_register(), y.to_register()); },
1649 [=](ymm_register<std::int16_t> auto x, auto y) { return _mm256_subs_epi16(x.to_register(), y.to_register()); },
1650#endif
1651
1652 // 512-bit
1653#if defined(__AVX512F__)
1654 [=](zmm_register<std::uint8_t> auto x, auto y) { return _mm512_subs_epu8(x.to_register(), y.to_register()); },
1655 [=](zmm_register<std::int8_t> auto x, auto y) { return _mm512_subs_epi8(x.to_register(), y.to_register()); },
1656 [=](zmm_register<std::uint16_t> auto x, auto y) { return _mm512_subs_epu16(x.to_register(), y.to_register()); },
1657 [=](zmm_register<std::int16_t> auto x, auto y) { return _mm512_subs_epi16(x.to_register(), y.to_register()); },
1658 #endif
1659
1660 [=](auto unhandled, auto) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); return unhandled;}
1661 };
1662
1663 // 256-bit saturating only exists in the Intel AVX2 ISA, not Intel AVX. If
1664 // Intel AVX2 is not available break the problem into 128-bit chunks instead
1665 // to use Intel SSE ISA.
1666 constexpr int numElements = (hasAvx && !hasAvx2) ? vec<_Tp>::size / 2 : vec<_Tp>::size;
1667 auto wrapper = [=](auto a, auto b) { return decltype(a)(impl(a, b)); }; // Return a vec from an intrinsic.
1668 return chunked_invoke<numElements>(wrapper, lhs, rhs);
1669}
1670
1671#if defined(__AVX512VBMI2__)
1672template<vec_unsigned_integral _Vp, vec_integral _Sp>
1673 requires (sizeof(_Vp::value_type) >= 2) // __AVX512VBMI2__ only has variable shift intrinsics for 16, 32, and 64-bit integers.
1674constexpr _Vp fsr(x86_avxsnc_tag, const _Vp& high, const _Vp& low, const _Sp& shift)
1675{
1676 using namespace detail;
1677
1678 target_overloads impl {
1679
1680 [=]<xmm_register<std::uint16_t> _Vec>(_Vec x, _Vec y, _Vec s)
1681 { return _mm_shrdv_epi16(x.to_register(), y.to_register(), s.to_register()); },
1682 [=]<ymm_register<std::uint16_t> _Vec>(_Vec x, _Vec y, _Vec s)
1683 { return _mm256_shrdv_epi16(x.to_register(), y.to_register(), s.to_register()); },
1684 [=]<zmm_register<std::uint16_t> _Vec>(_Vec x, _Vec y, _Vec s)
1685 { return _mm512_shrdv_epi16(x.to_register(), y.to_register(), s.to_register()); },
1686
1687 [=]<xmm_register<std::uint32_t> _Vec>(_Vec x, _Vec y, _Vec s)
1688 { return _mm_shrdv_epi32(x.to_register(), y.to_register(), s.to_register()); },
1689 [=]<ymm_register<std::uint32_t> _Vec>(_Vec x, _Vec y, _Vec s)
1690 { return _mm256_shrdv_epi32(x.to_register(), y.to_register(), s.to_register()); },
1691 [=]<zmm_register<std::uint32_t> _Vec>(_Vec x, _Vec y, _Vec s)
1692 { return _mm512_shrdv_epi32(x.to_register(), y.to_register(), s.to_register()); },
1693
1694 [=]<xmm_register<std::uint64_t> _Vec>(_Vec x, _Vec y, _Vec s)
1695 { return _mm_shrdv_epi64(x.to_register(), y.to_register(), s.to_register()); },
1696 [=]<ymm_register<std::uint64_t> _Vec>(_Vec x, _Vec y, _Vec s)
1697 { return _mm256_shrdv_epi64(x.to_register(), y.to_register(), s.to_register()); },
1698 [=]<zmm_register<std::uint64_t> _Vec>(_Vec x, _Vec y, _Vec s)
1699 { return _mm512_shrdv_epi64(x.to_register(), y.to_register(), s.to_register()); },
1700
1701 [=](auto unhandled, auto, auto) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); return unhandled; }
1702 };
1703
1704 auto wrapper = [=]<typename _Vec>(_Vec x, _Vec y, _Vec s) { return _Vec(impl(x, y, s)); };
1705 return std::bit_cast<_Vp>(chunked_invoke(wrapper, low, high, shift));
1706}
1707
1708template<vec_unsigned_integral _Vp, vec_integral _Sp>
1709 requires (sizeof(_Vp::value_type) >= 2) // __AVX512VBMI2__ only has variable shift intrinsics for 16, 32, and 64-bit integers.
1710constexpr _Vp fsl(x86_avxsnc_tag, const _Vp& high, const _Vp& low, const _Sp& shift)
1711{
1712 using namespace detail;
1713
1714 target_overloads impl {
1715 [=]<xmm_register<std::uint16_t> _Vec>(_Vec x, _Vec y, _Vec s)
1716 { return _mm_shldv_epi16(x.to_register(), y.to_register(), s.to_register()); },
1717 [=]<ymm_register<std::uint16_t> _Vec>(_Vec x, _Vec y, _Vec s)
1718 { return _mm256_shldv_epi16(x.to_register(), y.to_register(), s.to_register()); },
1719 [=]<zmm_register<std::uint16_t> _Vec>(_Vec x, _Vec y, _Vec s)
1720 { return _mm512_shldv_epi16(x.to_register(), y.to_register(), s.to_register()); },
1721
1722 [=]<xmm_register<std::uint32_t> _Vec>(_Vec x, _Vec y, _Vec s)
1723 { return _mm_shldv_epi32(x.to_register(), y.to_register(), s.to_register()); },
1724 [=]<ymm_register<std::uint32_t> _Vec>(_Vec x, _Vec y, _Vec s)
1725 { return _mm256_shldv_epi32(x.to_register(), y.to_register(), s.to_register()); },
1726 [=]<zmm_register<std::uint32_t> _Vec>(_Vec x, _Vec y, _Vec s)
1727 { return _mm512_shldv_epi32(x.to_register(), y.to_register(), s.to_register()); },
1728
1729 [=]<xmm_register<std::uint64_t> _Vec>(_Vec x, _Vec y, _Vec s)
1730 { return _mm_shldv_epi64(x.to_register(), y.to_register(), s.to_register()); },
1731 [=]<ymm_register<std::uint64_t> _Vec>(_Vec x, _Vec y, _Vec s)
1732 { return _mm256_shldv_epi64(x.to_register(), y.to_register(), s.to_register()); },
1733 [=]<zmm_register<std::uint64_t> _Vec>(_Vec x, _Vec y, _Vec s)
1734 { return _mm512_shldv_epi64(x.to_register(), y.to_register(), s.to_register()); },
1735
1736 [=](auto unhandled, auto, auto) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); return unhandled; }
1737 };
1738
1739 auto wrapper = [=]<typename _Vec>(_Vec x, _Vec y, _Vec s) { return _Vec(impl(x, y, s)); };
1740 return std::bit_cast<_Vp>(chunked_invoke(wrapper, low, high, shift));
1741}
1742#endif // __AVX512VBMI2__
1743
1744} // End of detail namespace
1745
1746} // namespace _XVEC_NAMESPACE::simd
A mask containing boolean elements.
Definition mask.hpp:121
static constexpr basic_mask __mask_from_count(simd_size_type _n)
Definition mask.hpp:223
Vector of elements sharing an element type and ABI.
Definition simd:161
static constexpr auto size
Definition simd:187
basic_mask< sizeof(_Tp), _Abi > mask_type
Definition simd:184
Check for contiguous range with a finite range.
Definition core.hpp:170
Definition core.hpp:65
constexpr _Vp min(const _Vp &lhs, const _Vp &rhs) noexcept
Compute the smaller of each respective vec element from the two input arguments.
Definition std_algorithm.hpp:34
constexpr auto select(const _M &m, const _T &if_true, const _U &if_false) noexcept -> decltype(simd_select_impl(m, if_true, if_false))
Select between corresponding elements according to a mask.
Definition std_algorithm.hpp:88
constexpr auto countl_zero(const _Vp &v) noexcept
Count consecutive zero bits from the most significant bit of each element.
Definition std_bit.hpp:90
constexpr auto popcount(const _Vp &x) noexcept
Count the one bits in each element.
Definition std_bit.hpp:133
constexpr auto vec_as_container(const _Vp &v)
Generate a variable containing the iota sequence (i.e., 0, 1, 2, ... size-1) for the given simd::vec ...
Definition generic_impl.hpp:116
constexpr void unchecked_store(const basic_vec< _Tp, _Abi > &value, _Range &&r, flags< _Flags... > f={})
Unchecked store into a contiguous sized range.
Definition loadstore.hpp:332
constexpr void partial_store(const basic_vec< _Tp, _Abi > &value, _Range &&r, flags< _Flags... > f={})
Store the elements from a basic_vec value into a contiguous range, performing a suitable conversion i...
Definition loadstore.hpp:231
basic_mask< sizeof(_Tp), simd_fixed_size_abi< _Np > > mask
Definition core.hpp:383
constexpr detail::deduced_vec_t< _Vp > fma(const _Vp &lhs, const _Vp &rhs, const _Vp &acc)
Compute (lhs * rhs) + acc element-wise with a single rounding.
Definition std_math.hpp:111
constexpr detail::deduced_vec_t< _Vp > copysign(const _Vp &magnitude, const _Vp &sign)
Combine corresponding magnitudes and signs.
Definition std_math.hpp:64
basic_vec< _Tp, simd_fixed_size_abi< _Np > > vec
Definition core.hpp:379
constexpr auto permute(const _Vp &v, detail::index_generator_function auto &&fn)
Permute the elements in a vec value using a generator function to compute the indexing.
Definition permute.hpp:94
constexpr auto partial_gather_from(_Range &&r, const _Indexes &indexes, flags< _Flags... > f={})
Gather the contents of a contiguous range using the integral indexes supplied in another vec value.
Definition permute.hpp:258
constexpr auto chunked_invoke(_Fn fn, const _Arg0 &arg0, const _Args &... args)
Invoke a callable on corresponding vector or mask chunks.
Definition invoke.hpp:61
constexpr rebind_t< _Up, _Vp > saturating_cast(const _Vp &v) noexcept
Compute a basic_vec where every element is converted to the new type, or if the value can't be repres...
Definition std_numeric.hpp:77
constexpr auto grow(const vec_or_mask_type auto &in)
Grow the vec or mask input to be N elements.
Definition named_permute.hpp:243
constexpr detail::RebindCast< _Up > rebind_cast
Rebinding CPO.
Definition std_rebind.hpp:55
constexpr detail::deduced_vec_t< _Vp > fmaddsub(const _Vp &lhs, const _Vp &rhs, const _Vp &acc)
Compute lhs * rhs - acc at even indexes and lhs * rhs + acc at odd indexes. Each expression is comput...
Definition std_math.hpp:136
constexpr detail::deduced_vec_t< _Vp > rcp(const _Vp &v)
Compute the approximate reciprocal of each element.
Definition std_math.hpp:85
constexpr detail::deduced_vec_t< _Vp > fmsubadd(const _Vp &lhs, const _Vp &rhs, const _Vp &acc)
Compute lhs * rhs + acc at even indexes and lhs * rhs - acc at odd indexes. Each expression is comput...
Definition std_math.hpp:147
_Vc trunc(const _Vc &value)
Truncate the real and imaginary components of each element.
Definition std_complex.hpp:80
constexpr detail::deduced_vec_t< _Vp > rsqrt(const _Vp &v)
Compute the approximate reciprocal square root of each element.
Definition std_math.hpp:91
constexpr auto to_builtin() const
Return the builtin mask value directly.
Definition mask.hpp:275
constexpr traits::register_type to_register() const
Definition simd:344
constexpr auto simd_bit_cast(const _From &v) noexcept
A simd-generic version of simd_bit_cast which directly invokes std::bit_cast.
Definition core.hpp:585
Detect a scoped enum. This will be provided in C++23 onwards.
Definition compat.hpp:27
constexpr basic_vec< std::uint8_t, _Abi > shuffle_epi8_lut(_Gen, basic_vec< std::uint8_t, _Abi > indexes)
Definition x86_impl.hpp:1214
X86_FP_CLASSIFY
Constants which specify to the compiler intrinsic which FP classification operation to perform.
Definition x86_impl.hpp:1002
constexpr basic_vec< _Tp, _Abi > popcount(x86_sse_tag, const basic_vec< _Tp, _Abi > &x) noexcept
}@
Definition x86_impl.hpp:1232
constexpr detail::deduced_mask_t< _Vp > isnormal(generic_tag, const _Vp &v)
Definition std_math.hpp:263
constexpr basic_vec< _Tp, _Abi > clz(x86_sse_tag, const basic_vec< _Tp, _Abi > &x) noexcept
Count the number of leading zero bits in each element.
Definition x86_impl.hpp:1303
constexpr bool is_fp16_v
Definition utilities.hpp:90
constexpr basic_vec< _Tp, _Abi > rsqrt(x86_sse_tag, const basic_vec< _Tp, _Abi > &v)
Constants which specify to the compiler intrinsic which FP classification operation to perform.
Definition x86_impl.hpp:1493
constexpr T saturating_sub(T lhs, T rhs) noexcept
Subtract two integers, clamping an unrepresentable mathematical result.
Definition compat.hpp:90
__m128i shuffle_epi8(__m128i values, __m128i indexes)
Definition x86_impl.hpp:1185
constexpr void store(x86_tag, const _Vp &value_original, std::span< _Up, _Extent > to, _Flags flags)
Special case code for handling 4 and 8 byte masked stores on Intel AVX2. This is faster than using th...
Definition x86_impl.hpp:553
constexpr auto builtin_operator(generic_tag, _Vp value, _Op op)
General purpose unary operator. Passes through to the compiler's own builtin.
Definition operators.hpp:28
constexpr basic_vec< _Tp, _Abi > sqrt(x86_sse_tag, const basic_vec< _Tp, _Abi > &v)
Constants which specify to the compiler intrinsic which FP classification operation to perform.
Definition x86_impl.hpp:1544
constexpr detail::deduced_mask_t< _Vp > isinf(generic_tag tag, const _Vp &v)
Definition std_math.hpp:251
constexpr bool contains_flag(flags< _Flags... >)
Query whether the given flags contains the specified type.
Definition core.hpp:415
constexpr auto has_single_bit(x86_avxsnc_tag, const basic_vec< _Tp, _Abi > &x) noexcept
Query whether each element has exactly one bit set.
Definition x86_impl.hpp:1294
constexpr basic_vec< _Tp, _Abi > rcp(x86_sse_tag, const basic_vec< _Tp, _Abi > &v)
Constants which specify to the compiler intrinsic which FP classification operation to perform.
Definition x86_impl.hpp:1443
constexpr void store_masked(x86_tag, const _Vp &value_original, std::span< _Up, _Extent > to, const typename _Vp::mask_type &mask_original, _Flags flags)
Implement masked stores for Intel SSE and Intel AVX targets.
Definition x86_impl.hpp:427
auto x86_fp_classify_mask(const basic_vec< _Tp, _Abi > &value)
Classify the floating-point vec values using the fpclassify intrinsic.
Definition x86_impl.hpp:1023
constexpr deduced_mask_t< _Vp > isnan(generic_tag, const _Vp &v)
Definition std_math.hpp:228
constexpr bool dependent_false
Definition utilities.hpp:103
constexpr auto gather_from(x86_avx2_tag, const _Range &r, const typename basic_vec< _Idx, _IdxAbi >::mask_type &mask, const basic_vec< _Idx, _IdxAbi > &indexes, flags< _Flags... > flags)
Masked gather operation.
Definition x86_impl.hpp:685
constexpr T saturating_add(T lhs, T rhs) noexcept
Add two integers, clamping an unrepresentable mathematical result.
Definition compat.hpp:68
constexpr basic_vec< _Tp, _Abi > round_op(x86_tag, const basic_vec< _Tp, _Abi > &v)
Rounding operations (ceil, floor, etc.).
Definition x86_impl.hpp:353
constexpr basic_vec< _Tp, _Abi > bitreverse(x86_sse_tag, const basic_vec< _Tp, _Abi > &x) noexcept
Count the number of leading zero bits in each element.
Definition x86_impl.hpp:1413
constexpr bool has_native_support_for_classify
Detect whether native classification support is available for the given type. Emulated FP16 has no su...
Definition x86_impl.hpp:1063
constexpr detail::deduced_mask_t< _Vp > isfinite(generic_tag tag, const _Vp &v)
Definition std_math.hpp:257
Allow overloaded lambdas to be used for target-specific code. Described at https://www....
Definition utilities.hpp:48
Definition core.hpp:393