Intel xvec
Experimental C++ simd class library
Loading...
Searching...
No Matches
permute.hpp
1//===----------------------------------------------------------------------===//
2//
3// Copyright (C) 2021 Intel Corporation
4//
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#pragma once
10
11#include <xvec/detail/config.hpp>
12
13#include <cstdint> // std::uint8_t, std::uint16_t, std::uint32_t, std::uint64_t
14#include <utility> // std::index_sequence, std::make_index_sequence
15
16namespace _XVEC_NAMESPACE::simd {
17
18namespace detail
19{
20
21// Utility to duplicate the bits in the input (e.g., 0101 becomes 00110011).
22// this is primarily used in Intel AVX-512 for vec elements which are bigger
23// than 64-bit, and hence require several blocks of 64-bit elements to be used.
24// It is no great pain then to use pdef, since most of the Intel AVX-512
25// instructions will have that readily available.
26#if defined(__BMI2__)
27inline std::uint64_t dupBits (std::uint32_t bits) {
28 auto spreadBits = _pdep_u64(std::uint64_t(bits), 0x55555555);
29 return spreadBits | (spreadBits << 1);
30}
31#endif
32
37template<typename _Tp, typename _AbiT, typename _Ip, typename _AbiI>
38constexpr auto permute_large_elements(const basic_vec<_Tp, _AbiT>& values, const basic_vec<_Ip, _AbiI>& indexes)
39{
40 static_assert(sizeof(_Tp) == 16, "Assume 128-bit is the maximum to deal with for now");
41
42 // :TODO: It is likely there are special cases that can be done faster. Or
43 // rather than duplicating the indexes, we could make the element smaller
44 // chunks and then recombine. Lots of room to explore. Just do enough to get
45 // it working for now.
46
47 // :TODO: Work harder to make it an amenable format to begin with? Convert to 64-bit indexes for example?
48
49 // Convert each index into a pair of 64-bit indexes which permute the upper
50 // and lower parts of the input elements. Given an index [3, 5, 1] then the
51 // 64-bit indexes would be [6, 7, 10, 11, 2, 3] for example (n => [n*2,
52 // n*2+1]).
53 auto bigIdxs = chunked_invoke([]<typename _Vec>(_Vec s) {
54 // Duplicate every index -> [3 3 5 5 1 1]
55 auto dup = permute<_Vec::size * 2>(s, [](auto idx) { return idx / 2; });
56 constexpr auto oddOnes =
57 vec<typename _Vec::value_type, _Vec::size * 2>([](auto i) -> typename _Vec::value_type { return i & 1; });
58 return (dup * cw<2>) + oddOnes;
59 }, indexes);
60
61 // Now do the permute again using 64-bit elements with the new indexes.
62 auto r = permute(simd_bit_cast<std::uint64_t>(values), bigIdxs);
63
64 // Map back to the original 128-bit elements.
65 return simd_bit_cast<_Tp>(r);
66}
67
68
72template<typename _Tp, typename _AbiT, typename _Ip, typename _AbiI>
73constexpr auto permute_byte_shuffle (const basic_vec<_Tp, _AbiT>& v, const basic_vec<_Ip, _AbiI>& indexes)
74{
75 static_assert(sizeof(_Tp) == 1 && sizeof(_Ip) == 1);
76 static_assert(basic_vec<_Tp, _AbiT>::size <= 16 && basic_vec<_Ip, _AbiI>::size <= 16, "Data and indexes must fit first lane");
77 // :TODO: In future both indexes and values could extend beyond lanes through
78 // duplication or multi-lane index merging, which will be faster than using permutex[2]var_epi8.
79 return vec<_Tp, basic_vec<_Ip, _AbiI>::size>(_mm_shuffle_epi8(v.to_register(), (indexes & std::uint8_t(0xf)).to_register()));
80}
81
83template<std::unsigned_integral _Tp, typename _AbiT, std::integral _Ip, typename _AbiI>
84requires (sizeof(basic_vec<_Tp, _AbiT>) <= 32)
85inline auto permute_x86_register(x86_avx2_tag, const basic_vec<_Tp, _AbiT>& v, const basic_vec<_Ip, _AbiI>& indexes)
86{
87 // Bump the incoming values up to a whole register so that the _mm256_permute
88 // instruction can be used. Both the indexes and the values need to be the
89 // same size. There is no advantage to using 128-bit permute instructions.
90 const auto _rv = permute<vec<_Tp>::size()>(v, perm_uninitResize);
91
92 auto impl = [_rv]<typename _ImplT>(_ImplT i) {
93 auto block = permute<vec<_Tp>::size>(i, perm_uninitResize); // Bump to whole register
94
95 auto r = [=]{
96 if constexpr (sizeof(_Tp) == 4)
97 return _mm256_permutevar8x32_epi32(__m256i(_rv.to_register()), block.to_register());
98 else if constexpr (sizeof(_Tp) == 8)
99 {
100 // There is no easy dynamic pd permute, so translate it into a 32-bit shuffle of pair-wise values instead.
101 // :TODO: permutevar_pd could be used, but it is rather awkward. Probably faster though, even then.
102 const auto dup32 = _mm256_castps_si256(_mm256_moveldup_ps(_mm256_castsi256_ps(block.to_register()))); // Dup 32-bit index
103 const auto mulBy2 = _mm256_slli_epi32(dup32, 1);
104 const auto idx32 = _mm256_add_epi64(mulBy2, __v4di() + 0x100000000);
105 return _mm256_permutevar8x32_epi32(_rv.to_register(), idx32);
106 }
107 else
108 static_assert(dependent_false<_Tp>, "Unhandled source type for permute");
109 }();
110
111 return fit_to_size<_ImplT::size>(vec<_Tp>(r));
112 };
113
114 return chunked_invoke(impl, indexes);
115}
116
117#if defined(__AVX512BW__) && defined(__AVX512F__)
119inline __m512i
120_mm512_permutex2var_epi8_emulated(__m512i v_data_0,
121 __m512i v_shuf_idxs,
122 __m512i v_data_1)
123{
124 // No 8-bit shuffle available, so use 16-bit instead, and process the upper and lower bytes in
125 // each 16-bit block independently. Note that each index is 6-bit.
126
127 // Each pair of 8-bit indexes is used to find the 16-bit block which will hold the value of interest in either
128 // the upper or lower byte. The odd and even indexes each need their own permutation, and the index to use
129 // for that permutation comes from the upper 5-bits of the incoming index (i.e., discard the LSB). The permute
130 // instruction masks the index bits itself, so we only need to put the upper 5-bits of the index
131 // into the correct location.
132 auto evenPerm = _mm512_permutex2var_epi16(v_data_0, v_shuf_idxs >> 1, v_data_1);
133 auto oddPerm = _mm512_permutex2var_epi16(v_data_0, v_shuf_idxs >> 9, v_data_1);
134
135 // evenPerm and oddPerm now contain pairs of bytes. We need to select which byte of each pair to use,
136 // and that selection is driven by the LSB of each index (i.e., LSB == 0 means use the lower byte of the
137 // pair, and LSB==1 means use the upper byte of the pair). We can use an in-lane bytewise shuffle to do this.
138 // We start with the identity shuffle (i.e., every byte stays in the same place), and then copy in the LSB
139 // of the index. This has the effect of using the LSB to choose which of each pair of bytes to use.
140 // Ternary logic is used to effect this bitwise blend.
141 constexpr __v64qi shuffle_identity = {
142 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
143 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
144 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
145 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
146 };
147 auto lsbSelector = _mm512_ternarylogic_epi32(__m512i(__v64qi() + 0x1), __m512i(shuffle_identity), v_shuf_idxs, 0xAC);
148
149 // Use the selector to choose which byte to pick. We have to do this for the odd and even indexes independently
150 // which means we end up doing some redundant work. Note that the second shuffle_epi8 only overwrites
151 // the odd positions, accepting all the previous shuffle's even values.
152 auto evenResult = _mm512_shuffle_epi8(evenPerm, lsbSelector);
153 auto combinedResult = _mm512_mask_shuffle_epi8(evenResult, 0xAAAAAAAAAAAAAAAA, oddPerm, lsbSelector);
154
155 return combinedResult;
156}
157#endif
158
159#if defined(__AVX512F__)
160inline target_overloads permutexvar {
161 #if defined(__AVX512VBMI__)
162 []<ymm_register<std::uint8_t> _Vec>(_Vec v, _Vec i) { return _mm256_permutexvar_epi8(i.to_register(), v.to_register()); },
163 []<zmm_register<std::uint8_t> _Vec>(_Vec v, _Vec i) { return _mm512_permutexvar_epi8(i.to_register(), v.to_register()); },
164 #else
165 []<ymm_register<std::uint8_t> _Vec>(_Vec v, _Vec i) { return __m256i(_mm512_permutex2var_epi8_emulated(v.to_register(), i.to_register(), __m512i())); },
166 []<zmm_register<std::uint8_t> _Vec>(_Vec v, _Vec i) { return _mm512_permutex2var_epi8_emulated(v.to_register(), i.to_register(), __m512i()); },
167 #endif
168
169 []<ymm_register<std::uint16_t> _Vec>(_Vec v, _Vec i) { return _mm256_permutexvar_epi16(i.to_register(), v.to_register()); },
170 []<ymm_register<std::uint32_t> _Vec>(_Vec v, _Vec i) { return _mm256_permutexvar_epi32(i.to_register(), v.to_register()); },
171 []<ymm_register<std::uint64_t> _Vec>(_Vec v, _Vec i) { return _mm256_permutexvar_epi64(i.to_register(), v.to_register()); },
172
173 []<zmm_register<std::uint16_t> _Vec>(_Vec v, _Vec i) { return _mm512_permutexvar_epi16(i.to_register(), v.to_register()); },
174 []<zmm_register<std::uint32_t> _Vec>(_Vec v, _Vec i) { return _mm512_permutexvar_epi32(i.to_register(), v.to_register()); },
175 []<zmm_register<std::uint64_t> _Vec>(_Vec v, _Vec i) { return _mm512_permutexvar_epi64(i.to_register(), v.to_register()); },
176};
177
178inline target_overloads permutex2var {
179 #if defined(__AVX512VBMI__)
180 []<ymm_register<std::uint8_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm256_permutex2var_epi8(v0.to_register(), i.to_register(), v1.to_register()); },
181 []<zmm_register<std::uint8_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm512_permutex2var_epi8(v0.to_register(), i.to_register(), v1.to_register()); },
182 #else
183 []<ymm_register<std::uint8_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return __m256i(_mm512_permutex2var_epi8_emulated(v0.to_register(), i.to_register(), v1.to_register())); },
184 []<zmm_register<std::uint8_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm512_permutex2var_epi8_emulated(v0.to_register(), i.to_register(), v1.to_register()); },
185 #endif
186
187 []<ymm_register<std::uint16_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm256_permutex2var_epi16(v0.to_register(), i.to_register(), v1.to_register()); },
188 []<ymm_register<std::uint32_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm256_permutex2var_epi32(v0.to_register(), i.to_register(), v1.to_register()); },
189 []<ymm_register<std::uint64_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm256_permutex2var_epi64(v0.to_register(), i.to_register(), v1.to_register()); },
190
191 []<zmm_register<std::uint16_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm512_permutex2var_epi16(v0.to_register(), i.to_register(), v1.to_register()); },
192 []<zmm_register<std::uint32_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm512_permutex2var_epi32(v0.to_register(), i.to_register(), v1.to_register()); },
193 []<zmm_register<std::uint64_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm512_permutex2var_epi64(v0.to_register(), i.to_register(), v1.to_register()); },
194};
195
196#endif // AVX512F
197
199template<std::unsigned_integral _Tp, typename _AbiT, std::integral _Ip, typename _AbiI>
200requires (basic_vec<_Tp, _AbiT>::size <= vec<_Tp>::size) // One register's worth of values to permute. Note
201 // that the indexes could be multiple registers.
202inline auto permute_x86_register(x86_avx512_tag, const basic_vec<_Tp, _AbiT>& v, const basic_vec<_Ip, _AbiI>& indexes)
203{
204 const auto rv = permute<vec<_Tp>::size>(v, perm_uninitResize);
205
206 auto impl = [=]<typename _Vec>(_Vec i) {
207 // Permute instructions require the indexes to be the same size as the values being permute.
208 const auto ri = permute<vec<_Tp>::size>(i, perm_uninitResize);
209 auto t = vec<_Tp>(permutexvar(rv, rebind_cast<_Tp>(ri)));
210 return fit_to_size<_Vec::size()>(t); // Permute instructions return a whole register.
211 };
212
213 return chunked_invoke(impl, indexes);
214}
215
217template<std::unsigned_integral _Tp, typename _AbiT, std::integral _Ip, typename _AbiI>
220 sizeof(_Tp) == sizeof(_Ip))
221inline auto permute_x86_register(x86_avx512_tag, const basic_vec<_Tp, _AbiT>& v, const basic_vec<_Ip, _AbiI>& indexes)
222{
223 constexpr int nativeSize = vec<_Tp>::size;
224
225 // Extract the two incoming registers and the indexes too as full size
226 // registers. The registers can only be full size Intel AVX-512 since smaller
227 // sizes would not fill two registers.
228 const auto _lower = take<nativeSize>(v);
229 const auto _upper = permute<nativeSize>(drop<nativeSize>(v), perm_uninitResize);
230
231 // Resize the full size register back to the partial size. The intrinsics
232 // require all of their registers to be the same size. We could size
233 // everything to the common size (e.g., allowing a 256-bit permute on an
234 // Intel AVX-512 machine) but that doesn't make much difference to the code
235 // generated.
236 auto impl = [=]<typename _Vec>(_Vec idx) {
237 auto nativeIndexes = permute<nativeSize>(idx, perm_uninitResize);
238 auto t = permutex2var(_lower, rebind_cast<_Tp>(nativeIndexes), _upper);
239 return fit_to_size<_Vec::size>(vec<_Tp>(t));
240 };
241
242 return chunked_invoke(impl, indexes);
243}
244
245// Break a large set of values down into smaller groups which are permuted and
246// combined. This acts recursively, repeatedly calling back into the permute to
247// handle ever smaller pieces.
248template<std::unsigned_integral _Tp, typename _AbiT, vec_integral _Ip>
249inline auto permute_recurse(const basic_vec<_Tp, _AbiT>& values, const _Ip& indexes)
250{
251 constexpr auto nextPowerOf2 = std::bit_ceil(unsigned(basic_vec<_Tp, _AbiT>::size));
252 constexpr auto halfSize = nextPowerOf2 / 2;
253
254 // Split into two pieces, one a power-of-2, and the other the remainder.
255 const auto [lowerValues, upperValues] = chunk<halfSize>(values);
256
257 // Permute the lower and upper recursively. Note that the upper part could be
258 // much smaller than the lower part (even a single element) in which case it
259 // would transform into code which did just enough work to permute that
260 // smaller set of values. There is no need to replicate the entire full size
261 // recursion on both parts.
262 const auto indexedLower = permute(lowerValues, indexes);
263 const auto indexedUpper = permute(upperValues, indexes);
264
265 // Choose between lower and upper using the index bit for that power-of-2.
266 return select((indexes & cw<halfSize>) == cw<0>, indexedLower, indexedUpper);
267}
268
274template<typename _Tp, typename _AbiTp, std::integral _Ip, typename _AbiIp, std::derived_from<x86_avx2_tag> VENDOR>
275constexpr basic_vec<_Tp, _AbiIp>
276permute(VENDOR tag, const basic_vec<_Tp, _AbiTp>& v, const basic_vec<_Ip, _AbiIp>& indexes)
277{
278 using _C = container_for_type<_Tp>;
279 auto vc = simd_bit_cast<_C>(v);
280
281 // Find the best match to do the permute.
282 if constexpr (sizeof(_Tp) == 1 && sizeof(_Ip) == 1 &&
285 // Special case - both source and index fit into one lane, so use an in-lane byte shuffle.
286 return permute_byte_shuffle(v, indexes);
287 else if constexpr (sizeof(_Tp) > 8)
288 // Special case - large elements are dealt with as though they are pairs of smaller elements.
289 return permute_large_elements(v, indexes);
290 else if constexpr (std::same_as<VENDOR, x86_avx2_tag> && sizeof(_Tp) < 4)
291 {
292 // Special case - Intel AVX2 can't permute small element values so bump the size up to
293 // something that can be supported.
294 const auto r = permute(rebind_cast<std::uint32_t>(vc), indexes);
295 const auto resizeBack = rebind_cast<_C>(r);
296 return simd_bit_cast<_Tp>(resizeBack);
297 }
298 else if constexpr (sizeof(_Tp) != sizeof(_Ip))
299 // Intel AVX ISAs have elements of the same size, so any mismatches of value and index must make the index size match.
300 return permute(v, rebind_cast<container_for_type<_Tp>>(indexes));
301 else if constexpr (requires {permute_x86_register(tag, vc, indexes); })
302 // If a native register permute is available, use that to do the permute.
303 // This must be done after the special cases (e.g., small elements on Intel AVX2)
304 // are handled as it assumes that index and value are the same size.
305 return simd_bit_cast<_Tp>(permute_x86_register(tag, vc, indexes));
306 else
307 {
308 // Run out of options. Recursively break the problem down into smaller pieces which *can* be solved.
309 return simd_bit_cast<_Tp>(permute_recurse(vc, indexes));
310 }
311}
312
313// :TODO: Permute where contained in a single lane, or other special cases.
314// duplicate a lane into multiple lanes to do multi-lane shuffle?
315
316// :TODO: Special cases, like if the mask is less than 64-bits and the indexes
317// has less than 128, and byte level permute is available (__AVX512VBMI2__),
318// it will be a single instruction. Or it could use shuffle_epi8, and so on.
319template<std::size_t _BytesTp, typename _AbiTp, typename _Ip, typename _AbiIp>
321permute(x86_avx512_tag, const basic_mask<_BytesTp, _AbiTp>& m, const basic_vec<_Ip, _AbiIp>& indexes)
322{
323 constexpr int _Np = basic_mask<_BytesTp, _AbiTp>::size;
324
325 // Convert the mask to a vec with elements which are the same size as the
326 // indexes, permute that, and then convert back to bits. :TODO: For ICX
327 // onwards it may be faster to convert the indexes to bytes and then do a
328 // permutex2var_epi8, which will allow a permute of up to 128 bits in the
329 // mask.
330 constexpr auto zeroMaskElement = vec<_Ip, _Np>();
331 constexpr auto nonZeroMaskElement = vec<_Ip, _Np>(_Ip(~_Ip()));
332 const auto maskAsSimd = select(mask<_Ip, _Np>(m), nonZeroMaskElement, zeroMaskElement);
333
334 const auto r = permute(maskAsSimd, indexes) != cw<0>;
336}
337
339template<simd_size_type _OutSize, std::size_t _Bytes, typename _Abi, typename _Gp>
340constexpr auto
341generated_permute(x86_avx512_tag, const basic_mask<_Bytes, _Abi>& m, _Gp)
342{
343#if defined(__AVX512BMI__)
344 using _I = std::uint8_t;
345#else
346 // Not always a good idea to use 16-bit, as the instruction itself is quite
347 // slow, but using 32-bit could lead to multiple registers which is also slow.
348 // One special case is when there are fewer bits than a lane has bytes, in
349 // which case epi8 could be used instead.
350 using _I = std::conditional_t<basic_mask<_Bytes, _Abi>::size <= 16, std::uint8_t, std::uint16_t>;
351#endif
352
353 using _R = vec<_I, _OutSize>;
354
355 // :TODO: Provide constexpr version which permutes the bits directly.
356
357 // generate a set of indexes to do the permutation, and then use a real indexed permute with those.
358 constexpr auto indexes = [=]{
360 return generate<_R>([](auto idx) -> _I { return _I(_Gp{}(idx, _OutSize)); });
361 else
362 return generate<_R>([](auto idx) -> _I { return _I(_Gp{}(idx)); });
363 }();
364 auto p = permute(m, indexes);
365
366 // Some of the indexes might be `zero_element` which need to be zeroed
367 // out. Create a suitable mask for them. :TODO: Once constexpr operator== is
368 // available this can be replaced by `indexes == zero_element`
369 using OutputMask = resize_t<_OutSize, basic_mask<_Bytes, _Abi>>;
370 constexpr auto notZeroMask = [=]{
372 return OutputMask([=](auto i) { return _Gp{}(i, _OutSize) != zero_element; });
373 else
374 return OutputMask([=](auto i) { return _Gp{}(i) != zero_element; });
375 }();
376
377 // Unconditionally mask the zero elements as the compiler will elide this
378 // where it is unnecessary.
379 return select(notZeroMask, p, OutputMask());
380}
381
382// :TODO: if the mask has no more than 8 bits then use bitshuffle_epi64_mask
383
384#if defined(__AVX512F__)
385inline target_overloads compress_one_register {
386#if defined(__AVX512VBMI2__)
387 // __AVX512VBMI2__ has 8 and 16-bit compress instructions.
388
389 // int 8
390 []<xmm_register<std::uint8_t> _Vec>(_Vec v, auto m, auto f)
391 { return _Vec(_mm_mask_compress_epi8(f.to_register(), m.to_register(), v.to_register())); },
392 []<ymm_register<std::uint8_t> _Vec>(_Vec v, auto m, auto f)
393 { return _Vec(_mm256_mask_compress_epi8(f.to_register(), m.to_register(), v.to_register())); },
394 []<zmm_register<std::uint8_t> _Vec>(_Vec v, auto m, auto f)
395 { return _Vec(_mm512_mask_compress_epi8(f.to_register(), m.to_register(), v.to_register())); },
396
397 // int 16
398 []<xmm_register<std::uint16_t> _Vec>(_Vec v, auto m, auto f)
399 { return _Vec(_mm_mask_compress_epi16(f.to_register(), m.to_register(), v.to_register())); },
400 []<ymm_register<std::uint16_t> _Vec>(_Vec v, auto m, auto f)
401 { return _Vec(_mm256_mask_compress_epi16(f.to_register(), m.to_register(), v.to_register())); },
402 []<zmm_register<std::uint16_t> _Vec>(_Vec v, auto m, auto f)
403 { return _Vec(_mm512_mask_compress_epi16(f.to_register(), m.to_register(), v.to_register())); },
404#else
405 // In the absence of __AVX512VBMI2__ instructions for 8/16-bit, synthesise them by using their 32-bit counterparts.
406
407 []<xmm_register<std::uint8_t> _Vec>(_Vec v, auto m, auto f) {
408 auto compressed = _mm512_mask_compress_epi32(_mm512_cvtepi8_epi32(f.to_register()),
409 m.to_register(),
410 _mm512_cvtepi8_epi32(v.to_register()));
411 return _Vec(_mm512_cvtepi32_epi8(compressed));
412 },
413 []<xmm_register<std::uint16_t> _Vec>(_Vec v, auto m, auto f) {
414 auto compressed = _mm256_mask_compress_epi32(_mm256_cvtepi16_epi32(f.to_register()),
415 m.to_register(),
416 _mm256_cvtepi16_epi32(v.to_register()));
417 return _Vec(_mm256_cvtepi32_epi16(compressed));
418 },
419 []<ymm_register<std::uint16_t> _Vec>(_Vec v, auto m, auto f) {
420 auto compressed = _mm512_mask_compress_epi32(_mm512_cvtepi16_epi32(f.to_register()),
421 m.to_register(),
422 _mm512_cvtepi16_epi32(v.to_register()));
423 return _Vec(_mm512_cvtepi32_epi16(compressed));
424 },
425
426#endif
427
428 // int 32
429 []<xmm_register<std::uint32_t> _Vec>(_Vec v, auto m, auto f)
430 { return _Vec(_mm_mask_compress_epi32(f.to_register(), m.to_register(), v.to_register())); },
431 []<ymm_register<std::uint32_t> _Vec>(_Vec v, auto m, auto f)
432 { return _Vec(_mm256_mask_compress_epi32(f.to_register(), m.to_register(), v.to_register())); },
433 []<zmm_register<std::uint32_t> _Vec>(_Vec v, auto m, auto f)
434 { return _Vec(_mm512_mask_compress_epi32(f.to_register(), m.to_register(), v.to_register())); },
435
436 // int 64
437 []<xmm_register<std::uint64_t> _Vec>(_Vec v, auto m, auto f)
438 { return _Vec(_mm_mask_compress_epi64(f.to_register(), m.to_register(), v.to_register())); },
439 []<ymm_register<std::uint64_t> _Vec>(_Vec v, auto m, auto f)
440 { return _Vec(_mm256_mask_compress_epi64(f.to_register(), m.to_register(), v.to_register())); },
441 []<zmm_register<std::uint64_t> _Vec>(_Vec v, auto m, auto f)
442 { return _Vec(_mm512_mask_compress_epi64(f.to_register(), m.to_register(), v.to_register())); },
443
444 // int 128 - Double up each bit to allow 64-bit compress to be used to handle 128-bit elements as pairs of bits.
445 []<xmm_register<unsigned __int128> _Vec>(_Vec v, auto m, auto f)
446 { return _Vec(_mm_mask_compress_epi64(f.to_register(), __mmask8(dupBits(m.to_register())), v.to_register())); },
447 []<ymm_register<unsigned __int128> _Vec>(_Vec v, auto m, auto f)
448 { return _Vec(_mm256_mask_compress_epi64(f.to_register(), __mmask8(dupBits(m.to_register())), v.to_register())); },
449 []<zmm_register<unsigned __int128> _Vec>(_Vec v, auto m, auto f)
450 { return _Vec(_mm512_mask_compress_epi64(f.to_register(), __mmask8(dupBits(m.to_register())), v.to_register())); },
451
452 [](auto unhandled, auto, auto) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); return unhandled;}
453 };
454
455
456 template <typename _Tp, typename _Abi, typename VENDOR>
457 constexpr void
458 compress_by_mask_to_memory(VENDOR, _Tp* ptr, const typename basic_vec<_Tp, _Abi>::mask_type& selector,
459 const basic_vec<_Tp, _Abi>& values, _Tp fill_value)
460 {
461 // Given a block, compress each bit of the block into consecutive memory regions.
462 auto nextPtr = ptr;
463 auto compressBlock = [&]<typename _Vec>(_Vec x, typename _Vec::mask_type m)
464 {
465 const auto fillV = _Vec(fill_value);
466 auto compressed = compress_one_register(x, m, fillV);
467 unchecked_store(compressed, nextPtr, _Vec::size);
468 nextPtr += reduce_count(m);
469 };
470
471 // Break the vec into pieces no bigger than 16 elements so that the compression operation can be done
472 // entirely within a single ZMM register.
473 constexpr int numBlocks = (sizeof(_Tp) >= 4 || std::derived_from<VENDOR, x86_avxsnc_tag>) ? vec<_Tp>::size : 16;
474 chunked_invoke<numBlocks>(compressBlock, values, selector);
475 }
476
477 // Special case for compression on __AVX512VBMI2__ in which a set of 8-bit indexes is
478 // compressed, and then the indexes are used in a permute.
479 template<vec_type _V>
480constexpr _V
481compress_by_mask_snc(const _V& values, const typename _V::mask_type& selector, typename _V::value_type fill_value)
482{
483 constexpr int _Np = _V::size;
484 static_assert(_Np <= detail::maxBytesInVec, "Cannot compress more than one native register at once");
485
486 // Use a byte compression to turn the entire 64-bit mask into a set of 64
487 // indexes, and then use that to permute the input. The output needs to be
488 // masked to introduce zeros in unwritten elements.
489 constexpr auto indexes = iota<vec<std::uint8_t, _Np>>;
490 const auto compressedIndexes = chunked_invoke(compress_one_register, indexes, mask<std::uint8_t, _Np>(selector), vec<std::uint8_t, _Np>());
491 auto compressed = permute(values, compressedIndexes);
492 auto numSetBits = reduce_count(selector);
493 auto finalMask = _V::mask_type::__mask_from_count(numSetBits);
494 return select(finalMask, compressed, _V(fill_value));
495}
496
500template<vec_type _V, std::derived_from<x86_avx512_tag> VENDOR>
501constexpr _V
502compress_by_mask(VENDOR tag, const _V& values, const typename _V::mask_type& selector,
503 typename _V::value_type fill_value = {})
504{
505 constexpr int _Np = _V::size;
506 using _Tp = typename _V::value_type;
507 static_assert(vec_container_element<_Tp>, "Caller must convert to container");
508
509 // Work out the maximum number of expansion elements permitted by the processor for this target.
510 constexpr bool isSnc = std::derived_from<VENDOR, x86_avxsnc_tag>;
511 constexpr int maxElementsPerExpand = isSnc ? vec<_Tp>::size : std::min<int>(16, vec<_Tp>::size);
512
513 if constexpr (_Np <= maxElementsPerExpand)
514 {
515 // If the number of elements fits into what can be expanded in one
516 // instruction (even allowing for using larger granularity), then do it.
517 return compress_one_register(values, selector, _V(fill_value));
518 }
519 else if constexpr (isSnc && _Np <= detail::maxBytesInVec)
520 // Special case for __AVX512VBMI2__ where the number of elements is 64 or fewer.
521 return compress_by_mask_snc(values, selector, fill_value);
522 else
523 {
524 // No other option - compress directly into memory.
525 _V result(fill_value);
526 compress_by_mask_to_memory(tag, reinterpret_cast<_Tp*>(&result), selector, values, fill_value);
527 return result;
528 }
529}
530#endif // AVX512F
531
532#if defined(__BMI2__)
533template<mask_type _M>
534constexpr _M
535compress_mask_by_mask(x86_avx2_tag, const _M& values, const _M& selector)
536{
537 constexpr int _Np = _M::size;
538
539 // At time of implementation only 128 bits were supported in masks. If this changes we need to
540 // modify the code below to handle more chunks of data.
541 static_assert(_Np <= 128);
542
543 using _R = std::conditional_t<(_Np > 64), __uint128_t, std::uint64_t>;
544
545 // Process up to the first 64 bits.
546 constexpr int lowerN = std::min<int>(_Np, 64);
547 auto maskAsUll = detail::fit_to_size<lowerN>(selector).to_ullong();
548 _R resultInt = _pext_u64(detail::fit_to_size<lowerN>(values).to_ullong(), maskAsUll);
549
550 // Process any upper bits.
551 if constexpr (_Np > 64)
552 {
553 _R upperCompressed = _pext_u64(drop<64>(values).to_ullong(), drop<64>(selector).to_ullong());
554 resultInt |= (upperCompressed << std::popcount((std::uint64_t)maskAsUll));
555 }
556
557 // The result is in an integer. Convert that back into a mask.
558 _M resultMask((std::uint64_t)resultInt);
559 if constexpr (_Np > 64) {
560 auto upper = resize_t<_Np - 64, _M>(std::uint64_t(resultInt >> 64));
561 resultMask = cat(take<64>(resultMask), upper);
562 }
563
564 return resultMask;
565}
566
567template<mask_type _M>
568constexpr _M
569compress_mask_by_mask(x86_avx512_tag, const _M& v, const _M& selector)
570{
571
572 constexpr int _Np = _M::size;
573
574 auto vd = v.to_builtin();
575 auto md = selector.to_builtin();
576
577 // At time of implementation only 128 bits were supported in masks. If this changes we need to
578 // modify the code below to handle more chunks of data.
579 static_assert(_Np <= 128);
580
581 // Process the first 64 bits.
582 using _B = typename _M::builtin_type;
583 _B result = _pext_u64((std::uint64_t)vd, (std::uint64_t)md);
584
585 // Process the top 64-bits too and then combine them together.
586 if constexpr (_Np > 64)
587 {
588 _B upperCompressed = _pext_u64(vd >> 64, md >> 64);
589 result |= (upperCompressed << std::popcount((std::uint64_t)md));
590 }
591
592 return _M::from_builtin(result);
593}
594#endif
595
596#if defined(__AVX512F__)
597inline target_overloads expand_one_register {
598#if defined(__AVX512VBMI2__)
599 // int 8
600 []<xmm_register<std::uint8_t> _Vec>(_Vec v, auto m, auto alt)
601 { return _Vec(_mm_mask_expand_epi8(alt.to_register(), m.to_register(), v.to_register())); },
602 []<ymm_register<std::uint8_t> _Vec>(_Vec v, auto m, auto alt)
603 { return _Vec(_mm256_mask_expand_epi8(alt.to_register(), m.to_register(), v.to_register())); },
604 []<zmm_register<std::uint8_t> _Vec>(_Vec v, auto m, auto alt)
605 { return _Vec(_mm512_mask_expand_epi8(alt.to_register(), m.to_register(), v.to_register())); },
606
607 []<xmm_register<std::uint16_t> _Vec>(_Vec v, auto m, auto alt)
608 { return _Vec(_mm_mask_expand_epi16(alt.to_register(), m.to_register(), v.to_register())); },
609 []<ymm_register<std::uint16_t> _Vec>(_Vec v, auto m, auto alt)
610 { return _Vec(_mm256_mask_expand_epi16(alt.to_register(), m.to_register(), v.to_register())); },
611 []<zmm_register<std::uint16_t> _Vec>(_Vec v, auto m, auto alt)
612 { return _Vec(_mm512_mask_expand_epi16(alt.to_register(), m.to_register(), v.to_register())); },
613#else
614
615 // When VBMI2 is not available, small types can only be expanded by
616 // converting to a 4B type, expanding that, and converting back again. For the
617 // expand instruction to have only one register to work it must accept no
618 // more than 16 elements to start with (i.e., 128 or 256-bits respectively).
619 []<xmm_register<std::uint8_t> _Vec>(_Vec v, auto m, auto alt) {
620 auto compressed = _mm512_maskz_expand_epi32(m.to_register(),
621 _mm512_cvtepi8_epi32(v.to_register()));
622 return _Vec(_mm512_mask_cvtepi32_epi8(alt.to_register(), m.to_register(), compressed));
623 },
624 []<xmm_register<std::uint16_t> _Vec>(_Vec v, auto m, auto alt) {
625 auto compressed = _mm256_maskz_expand_epi32(m.to_register(),
626 _mm256_cvtepi16_epi32(v.to_register()));
627 return _Vec(_mm256_mask_cvtepi32_epi16(alt.to_register(), m.to_register(), compressed));
628 },
629 []<ymm_register<std::uint16_t> _Vec>(_Vec v, auto m, auto alt) {
630 auto compressed = _mm512_maskz_expand_epi32(m.to_register(),
631 _mm512_cvtepi16_epi32(v.to_register()));
632 return _Vec(_mm512_mask_cvtepi32_epi16(alt.to_register(), m.to_register(), compressed));
633 },
634#endif
635
636 // 32-bit
637 []<xmm_register<std::uint32_t> _Vec>(_Vec v, auto m, auto alt)
638 { return _Vec(_mm_mask_expand_epi32(alt.to_register(), m.to_register(), v.to_register())); },
639 []<ymm_register<std::uint32_t> _Vec>(_Vec v, auto m, auto alt)
640 { return _Vec(_mm256_mask_expand_epi32(alt.to_register(), m.to_register(), v.to_register())); },
641 []<zmm_register<std::uint32_t> _Vec>(_Vec v, auto m, auto alt)
642 { return _Vec(_mm512_mask_expand_epi32(alt.to_register(), m.to_register(), v.to_register())); },
643
644 // 64-bit
645 []<xmm_register<std::uint64_t> _Vec>(_Vec v, auto m, auto alt)
646 { return _Vec(_mm_mask_expand_epi64(alt.to_register(), m.to_register(), v.to_register())); },
647 []<ymm_register<std::uint64_t> _Vec>(_Vec v, auto m, auto alt)
648 { return _Vec(_mm256_mask_expand_epi64(alt.to_register(), m.to_register(), v.to_register())); },
649 []<zmm_register<std::uint64_t> _Vec>(_Vec v, auto m, auto alt)
650 { return _Vec(_mm512_mask_expand_epi64(alt.to_register(), m.to_register(), v.to_register())); },
651
652 // int 128 - Double up each bit to allow 64-bit compress to be used to handle 128-bit elements as pairs of bits.
653 []<xmm_register<unsigned __int128> _Vec>(_Vec v, auto m, auto alt)
654 { return _Vec(_mm_mask_expand_epi64(alt.to_register(), __mmask8(dupBits(m.to_register())), v.to_register())); },
655 []<ymm_register<unsigned __int128> _Vec>(_Vec v, auto m, auto alt)
656 { return _Vec(_mm256_mask_expand_epi64(alt.to_register(), __mmask8(dupBits(m.to_register())), v.to_register())); },
657 []<zmm_register<unsigned __int128> _Vec>(_Vec v, auto m, auto alt)
658 { return _Vec(_mm512_mask_expand_epi64(alt.to_register(), __mmask8(dupBits(m.to_register())), v.to_register())); },
659
660 [](auto unhandled, auto, auto) { static_assert(dependent_false<decltype(unhandled)>, "No target intrinsics"); return unhandled;}
661};
662
663template<vec_type _V, typename VENDOR>
664constexpr _V
665expand_by_mask_from_memory(VENDOR, const typename _V::value_type* ptr,
666 const typename _V::mask_type& selector, const _V& original)
667{
668 using _Tp = typename _V::value_type;
669
670 // Given a block, expand the indexes and move the output offset.
671 auto nextPos = ptr;
672 auto expandBlock = [&]<typename _Vec>(_Vec x, typename _Vec::mask_type m) {
673 auto t = detail::load<_Vec>(target, std::span<const _Tp, _Vec::size>(nextPos, _Vec::size), detail::add_unchecked_flag(flags<>{}));
674 nextPos += reduce_count(m);
675 return expand_one_register(t, m, x);
676 };
677
678 // Break the vec into pieces no bigger than 16 elements so that the expansion operation can be done
679 // entirely within a single ZMM register.
680 constexpr int numBlocks = (sizeof(_Tp) >= 4 || std::derived_from<VENDOR, x86_avxsnc_tag>) ? vec<_Tp>::size : 16;
681 return chunked_invoke<numBlocks>(expandBlock, original, selector);
682}
683
684template<vec_type _V>
685constexpr _V
686expand_by_mask_snc(const _V& value, const typename _V::mask_type& selector, const _V& original)
687{
688 constexpr int _Np = _V::size;
689
690 static_assert(_Np <= detail::maxBytesInVec);
691
692 // __AVX512VBMI2__ sizes greater than 64 values can be expanded by converting the mask into indexes
693 // by expanding iota, and then permuting using the indexes.
694 // :TODO: The permute should cope better with mixed sizes than it currently does.
695 constexpr auto indexes = iota<vec<std::uint8_t, _Np>>;
696 const auto expandedIndexes = chunked_invoke(expand_one_register, indexes, mask<std::uint8_t, _Np>(selector), vec<std::uint8_t, _Np>());
697 auto expanded = permute(value, expandedIndexes);
698 return select(selector, expanded, original);
699}
700
701template<vec_type _V, std::derived_from<x86_avx512_tag> VENDOR>
702constexpr _V
703expand_by_mask(VENDOR tag, const _V& values, const typename _V::mask_type& selector, const _V& original)
704{
705 using _Tp = typename _V::value_type;
706 static_assert(vec_container_element<_Tp>, "Caller must convert to container");
707 constexpr auto _Np = _V::size;
708
709 // Work out the maximum number of expansion elements permitted by the processor for this target.
710 constexpr bool isSnc = std::derived_from<VENDOR, x86_avxsnc_tag>;
711 const int maxElementsPerExpand = isSnc ? vec<_Tp>::size() : std::min<int>(16, vec<_Tp>::size());
712
713 if constexpr (_Np <= maxElementsPerExpand)
714 // If the number of elements fits into what can be expanded in one
715 // instruction (even allowing for using larger granularity), then do it.
716 return expand_one_register(values, selector, original);
717 else if constexpr (isSnc && _Np <= detail::maxBytesInVec)
718 // Special case for __AVX512VBMI2__ where the number of elements is less than 64.
719 return expand_by_mask_snc(values, selector, original);
720 else
721 {
722 // No other option - load from memory.
723 return expand_by_mask_from_memory(tag, reinterpret_cast<const _Tp*>(&values), selector, original);
724 }
725}
726#endif // AVX512F
727
728#if defined(__BMI2__)
729template<mask_type _M>
730constexpr _M
731expand_mask_by_mask(x86_avx_tag, const _M& values, const _M& selector, const _M& original)
732{
733 constexpr int _Np = _M::size;
734
735 // At time of implementation only 128 bits were supported in masks. If this changes we need to
736 // modify the code below to handle more chunks of data.
737 static_assert(_Np <= 128);
738
739 using _R = std::conditional_t<(_Np > 64), __uint128_t, std::uint64_t>;
740
741 // Process the first 64 bits.
742 constexpr int lowerN = std::min<int>(_Np, 64);
743 auto lowerSelector = detail::fit_to_size<lowerN>(selector).to_ullong();
744 auto lowerValue = detail::fit_to_size<lowerN>(values).to_ullong();
745 _R resultInt = _pdep_u64(lowerValue, lowerSelector);
746
747 // Merge in any upper bits for large masks.
748 if constexpr (_Np > 64)
749 {
750 _R upper64Value = drop<64>(values).to_ullong();
751 std::uint64_t upper64Selector = drop<64>(selector).to_ullong();
752 auto wholeValue = (upper64Value << 64) | _R(lowerValue);
753 auto bitsToExpand = std::uint64_t(wholeValue >> std::popcount(lowerSelector));
754 _R expandedBits = _pdep_u64(bitsToExpand, upper64Selector);
755 resultInt |= _R(expandedBits) << 64;
756 }
757
758 _M resultMask((std::uint64_t)resultInt);
759 if constexpr (_Np > 64)
760 resultMask = cat(take<64>(resultMask), resize_t<_Np - 64, _M>(std::uint64_t(resultInt >> 64)));
761
762 // Blend in any original bits and return the final result.
763 auto final = resultMask.to_builtin() | (original.to_builtin() & ~(selector.to_builtin()));
764 return _M::from_builtin(final);
765}
766
767template<mask_type _M>
768constexpr _M
769expand_mask_by_mask(x86_avx512_tag, const _M& values, const _M& selector, const _M& original)
770{
771 constexpr int _Np = _M::size;
772
773 auto vd = values.to_builtin();
774 auto sd = selector.to_builtin();
775
776 // At time of implementation only 128 bits were supported in masks. If this changes we need to
777 // modify the code below to handle more chunks of data.
778 static_assert(_Np <= 128);
779
780 // Process the first 64 bits.
781 typename _M::builtin_type result = _pdep_u64((std::uint64_t)vd, (std::uint64_t)sd);
782
783 // // If there are more than 64-bits, process the top 64-bits too and then combine them together.
784 if constexpr (_Np > 64)
785 {
786 std::uint64_t useBits = (std::uint64_t)(vd >> std::popcount((std::uint64_t)sd));
787 typename _M::builtin_type expandedBits = _pdep_u64(useBits, (std::uint64_t)(sd >> 64));
788 result |= expandedBits << 64;
789 }
790
791 // Blend in the original data in unmasked positions.
792 return _M::from_builtin(result | (original.to_builtin() & ~sd));
793}
794#endif
795
799
801template <int _Np, mask_type _Mp>
802constexpr auto permute_drop(x86_avx512_tag, const _Mp& in) noexcept
803{
804 static_assert(_Mp::size >= _Np, "Too short to drop N elements");
805 return resize_t<_Mp::size - _Np, _Mp>::from_builtin(in.to_register() >> _Np);
806}
807
809template <int _Np, mask_type _Mp>
810constexpr auto permute_take(x86_avx512_tag, const _Mp& in) noexcept
811{
812 static_assert(_Np <= _Mp::size, "Too short to drop N elements");
813 return resize_t<_Np, _Mp>::from_builtin(in.to_register());
814}
815
817template<int _Np, mask_type _Mp>
818constexpr auto permute_grow(x86_avx512_tag, const _Mp& in) noexcept {
819 static_assert(_Np >= _Mp::size, "Must grow to a size bigger than the input");
820 return resize_t<_Np, _Mp>::from_builtin(in.to_register());
821}
822
824template<int _Np, mask_type _Mp>
825constexpr auto permute_grow(x86_avx512_tag, const _Mp& in, bool fill) noexcept {
826 static_assert(_Np >= _Mp::size, "Must grow to a size bigger than the input");
827 using _NewMp = resize_t<_Np, _Mp>;
828 using _CM = typename _NewMp::builtin_type;
829
830 auto f = fill ? ~(_CM()) : _CM();
831 const auto lowerMask = ~(_CM()) << _Mp::size();
832
833 return _NewMp::from_builtin(in.to_register() | (f & lowerMask));
834}
835
840template<mask_type _Mp, int _Stride>
841constexpr auto every_nth_bit() {
842 using _Out = resize_t<_Mp::size * _Stride, _Mp>;
843 using _I = typename _Out::builtin_type;
844 return [=]<std::size_t... _Iota>(std::index_sequence<_Iota...>) {
845 return ((_I(1) << (_Iota * _Stride)) | ...);
846 }(std::make_index_sequence<_Mp::size>{});
847}
848
849#if defined(__AVX512F__)
851template<int _Np, mask_type _Mp>
852constexpr auto permute_stride(x86_avx512_tag, const _Mp& in) noexcept {
853 constexpr auto _OldSize = _Mp::size;
854
855 // If the mask size increases this code will need more work.
856 static_assert(_OldSize <= 128, "Hard-coded to a pair of 64-bit pext instructions");
857
858 // Easy special case.
859 if constexpr (_Np == 1) return in;
860
861 constexpr int _NewSize = (_OldSize + (_Np - 1)) / _Np;
862 using _NewMp = resize_t<_NewSize, _Mp>;
863
864 auto iv = in.to_register();
865
866 // Handle the lower 64-bits. Note that the output will never exceed 64 bits,
867 // since stride of 1 is handled above, and the maximum size of a mask is
868 // currently 128-bits, which means a stride of 2 will never generate more than
869 // 64 bits.
870 constexpr auto bits = every_nth_bit<_NewMp, _Np>();
871 std::uint64_t t = _pext_u64((std::uint64_t)iv, (std::uint64_t)bits);
872
873 // Handle the upper 64-bits if they are also present.
874 if constexpr (_OldSize > 64) {
875 auto upperBits = std::uint64_t(bits >> 64);
876 std::uint64_t b = _pext_u64((std::uint64_t)(iv >> 64), upperBits);
877 t |= b << std::popcount(std::uint64_t(bits & 0xFFFFFFFFFFFFFFFF));
878 }
879
880 return _NewMp::from_builtin(t);
881}
882#endif
883
884} // detail namespace
885
886} // namespace _XVEC_NAMESPACE::simd
A mask containing boolean elements.
Definition mask.hpp:121
Vector of elements sharing an element type and ABI.
Definition simd:161
static constexpr auto size
Return the number of elements contained in the vec.
Definition simd:187
constexpr _Vp min(const _Vp &lhs, const _Vp &rhs) noexcept
Compute the smaller of each respective vec element from the two input arguments.
Definition std_algorithm.hpp:34
constexpr auto select(const _M &m, const _T &if_true, const _U &if_false) noexcept -> decltype(simd_select_impl(m, if_true, if_false))
Select between corresponding elements according to a mask.
Definition std_algorithm.hpp:88
constexpr auto chunk(const _S &container) noexcept
Definition split_concat.hpp:107
constexpr auto cat(const basic_mask< _Bytes, _Abis > &... masks) noexcept
Concatenate several smaller mask values with the same element size into a single large mask value.
Definition split_concat.hpp:144
constexpr void unchecked_store(const basic_vec< _Tp, _Abi > &value, _Range &&r, flags< _Flags... > f={})
Unchecked store into a contiguous sized range.
Definition loadstore.hpp:332
basic_mask< sizeof(_Tp), simd_fixed_size_abi< _Np > > mask
Definition core.hpp:383
static constexpr auto size
Return the number of elements contained in the mask.
Definition mask.hpp:148
constexpr simd_size_type reduce_count(const mask_type auto &mask) noexcept
Count the true mask elements.
Definition mask.hpp:507
basic_vec< _Tp, simd_fixed_size_abi< _Np > > vec
Definition core.hpp:379
constexpr auto permute(const _Vp &v, detail::index_generator_function auto &&fn)
Permute the elements in a vec value using a generator function to compute the indexing.
Definition permute.hpp:94
constexpr auto chunked_invoke(_Fn fn, const _Arg0 &arg0, const _Args &... args)
Invoke a callable on corresponding vector or mask chunks.
Definition invoke.hpp:61
constexpr auto take(const vec_or_mask_type auto &in)
Definition named_permute.hpp:237
constexpr auto drop(const vec_or_mask_type auto &in)
Discard the first _Np elements of the input, returning the remainder.
Definition named_permute.hpp:240
constexpr detail::RebindCast< _Up > rebind_cast
Rebinding CPO.
Definition std_rebind.hpp:55
constexpr traits::register_type to_register() const
Definition simd:344
constexpr auto simd_bit_cast(const _From &v) noexcept
A simd-generic version of simd_bit_cast which directly invokes std::bit_cast.
Definition core.hpp:585
constexpr auto permute_grow(generic_tag, const _Sp &in) noexcept
A named permute which increases the number of elements in the vec or mask, placing a copy of the give...
Definition named_permute.hpp:63
constexpr auto every_nth_bit()
Definition permute.hpp:841
constexpr auto permute_large_elements(const basic_vec< _Tp, _AbiT > &values, const basic_vec< _Ip, _AbiI > &indexes)
Definition permute.hpp:38
constexpr basic_vec< _Tp, _AbiIp > permute(VENDOR tag, const basic_vec< _Tp, _AbiTp > &v, const basic_vec< _Ip, _AbiIp > &indexes)
Definition permute.hpp:276
constexpr auto permute_drop(generic_tag, const _Sp &in) noexcept
A named permute which discards the first _Np values of a vec or mask, returning the remainder.
Definition named_permute.hpp:30
auto permute_x86_register(x86_avx2_tag, const basic_vec< _Tp, _AbiT > &v, const basic_vec< _Ip, _AbiI > &indexes)
Permute one register of Intel AVX2 values.
Definition permute.hpp:85
constexpr auto generated_permute(x86_avx512_tag, const basic_mask< _Bytes, _Abi > &m, _Gp)
Use a generator to permute a compact bit mask.
Definition permute.hpp:341
constexpr bool dependent_false
Definition utilities.hpp:103
constexpr auto permute_take(generic_tag, const _Sp &in)
A named permute which keeps the first _Np values of a simd and discards the remainder.
Definition named_permute.hpp:22
constexpr auto permute_byte_shuffle(const basic_vec< _Tp, _AbiT > &v, const basic_vec< _Ip, _AbiI > &indexes)
Definition permute.hpp:73