Intel xvec
Experimental C++ simd class library
Loading...
Searching...
No Matches
permute.hpp
1//===----------------------------------------------------------------------===//
2//
3// Copyright (C) 2021 Intel Corporation
4//
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#pragma once
10
11#include <xvec/detail/config.hpp>
12
13#include <concepts>
14#include <span>
15#include <cstddef>
16
17namespace _XVEC_NAMESPACE::simd {
18
19namespace detail {
20
22template <simd_size_type _NewSize, simd_size_type _VecOrMaskSize>
23inline constexpr simd_size_type new_simd_size = (_NewSize == 0) ? _VecOrMaskSize : _NewSize;
24
43template<mask_type _M, vec_type _V>
44constexpr _M vec_to_mask(const _V& s) { return _M([=](auto i) { return s[i] != 0; }); }
45
49template<vec_type _V> constexpr auto to_container(const _V& v)
50{
51 using _C = detail::container_for_type<typename _V::value_type>;
52 return simd_bit_cast<_C>(v);
53}
54
56template<typename _Vp, typename _Range, typename _Indexes>
58 (std::same_as<_Vp, std::nullptr_t>
62
64template<typename _Vp, typename _Range, typename _Indexes>
67 std::ranges::output_range<_Range, typename _Vp::value_type> &&
69
70} // namespace detail
71
82
92template<simd_size_type _NewSize = 0, vec_type _Vp>
93constexpr auto
94permute(const _Vp& v, detail::index_generator_function auto&& fn) {
95 constexpr auto new_simd_size = detail::new_simd_size<_NewSize, _Vp::size()>;
96 return detail::generated_permute<new_simd_size>(target, v, std::forward<decltype(fn)>(fn)); }
97
104template<simd_size_type _NewSize = 0, mask_type _Mask>
105constexpr auto
107 constexpr auto new_simd_size = detail::new_simd_size<_NewSize, _Mask::size()>;
108
109 if (std::is_constant_evaluated())
110 {
111 auto asS = permute<new_simd_size>(+mask, std::forward<decltype(fn)>(fn));
112 using asM = resize_t<new_simd_size, _Mask>;
113 return detail::vec_to_mask<asM>(asS);
114 }
115 else
116 return detail::generated_permute<new_simd_size>(target, mask, std::forward<decltype(fn)>(fn));
117}
118
125template<vec_or_mask_type _V, vec_integral _I>
126constexpr resize_t<_I::size, _V> permute(const _V& v, const _I& indexes)
127{
128 if (std::is_constant_evaluated())
129 return resize_t<_I::size, _V>([=](auto i) { return v[indexes[i]]; });
130 else
131 return detail::permute(target, v, indexes);
132}
133
143template<vec_type _V>
144constexpr _V
145compress(const _V& v, const typename _V::mask_type& selector)
146{
147 using _Tp = typename _V::value_type;
148
149 if (std::is_constant_evaluated())
150 return compress(v, selector, _Tp()); // The fill-with-value overload handles consteval.
151 else
152 {
153 const auto out = detail::compress_by_mask(target, detail::to_container(v), selector);
154 return simd_bit_cast<_Tp>(out);
155 }
156}
157
163template<vec_type _V>
164constexpr _V
165compress(const _V& v, const typename _V::mask_type& selector, const typename _V::value_type& fill_value)
166{
167 using _Tp = typename _V::value_type;
168 if (std::is_constant_evaluated()) {
169 std::array<_Tp, _V::size()> r; // Can't write to a vec[] so use an array instead.
170 r.fill(fill_value);
171 int nextOutPos = 0;
172 for (int i=0; i<_V::size(); ++i)
173 if (selector[i])
174 r[nextOutPos++] = v[i];
175 return _V(r);
176 }
177 else
178 {
179 using _C = detail::container_for_type<_Tp>;
180 const auto fillAsC = std::bit_cast<_C>(fill_value);
181 auto t = detail::compress_by_mask(target, detail::to_container(v), selector, fillAsC);
182 return simd_bit_cast<_Tp>(t);
183 }
184}
185
187template<mask_type _M>
188constexpr _M compress(const _M& v, const std::type_identity_t<_M>& selector)
189{
190 if (std::is_constant_evaluated())
191 return detail::vec_to_mask<_M>(compress(+v, selector, false));
192 else
193 return detail::compress_mask_by_mask(target, v, selector);
194}
195
197template<mask_type _M>
198constexpr _M compress(const _M& v, const std::type_identity_t<_M>& selector, bool fill_value)
199{
200 if (std::is_constant_evaluated())
201 return detail::vec_to_mask<_M>(compress(+v, selector, fill_value));
202 else
203 return detail::compress_mask_by_mask(target, v, selector, fill_value);
204}
205
212template<vec_type _V>
213constexpr _V
214expand(const _V& v, const typename _V::mask_type& selector, const _V& original = {})
215{
216 using _Tp = typename _V::value_type;
217
218 if (std::is_constant_evaluated())
219 {
220 std::array<_Tp, _V::size()> r;
221 int nextInPos = 0;
222 for (int i=0; i<_V::size(); ++i)
223 r[i] = selector[i] ? v[nextInPos++] : original[i];
224 return _V(r);
225 }
226 else
227 {
228 const auto out = detail::expand_by_mask(target, detail::to_container(v), selector, detail::to_container(original));
229 return simd_bit_cast<_Tp>(out);
230 }
231}
232
237template<mask_type _M>
238constexpr _M expand(const _M& v, const std::type_identity_t<_M>& selector, const _M& original = {})
239{
240 if (std::is_constant_evaluated())
241 return detail::vec_to_mask<_M>(expand(+v, selector, +original));
242 else
243 return detail::expand_mask_by_mask(target, v, selector, original);
244}
245
255template<typename _Vp = std::nullptr_t, detail::contiguous_sized_range _Range, vec_integral _Indexes, typename... _Flags>
256requires detail::gather_from_requires<_Vp, _Range, _Indexes>
257constexpr auto
258partial_gather_from(_Range&& r, const _Indexes& indexes, flags<_Flags...> f = {})
259{
260 using _RangeSimd = rebind_t<std::ranges::range_value_t<_Range>, _Indexes>;
261 using _OutSimd = std::conditional_t<std::same_as<_Vp, std::nullptr_t>, _RangeSimd, _Vp>;
262
263 detail::mandates_for_load(r, _OutSimd{}, f);
264
265 if (std::is_constant_evaluated())
266 return _OutSimd([=](auto i) -> typename _OutSimd::value_type { return r[indexes[i]]; });
267 else
268 return _OutSimd(detail::gather_from(target, r, indexes, f));
269}
270
272template<typename _Vp = std::nullptr_t, detail::contiguous_sized_range _Range, vec_integral _Indexes, typename... _Flags>
273requires detail::gather_from_requires<_Vp, _Range, _Indexes>
274constexpr auto
275unchecked_gather_from(_Range&& r, const _Indexes& indexes, flags<_Flags...> f = {})
276 { return partial_gather_from<_Vp>(r, indexes, detail::add_unchecked_flag(f)); }
277
283template<typename _Vp = std::nullptr_t, detail::contiguous_sized_range _Range, vec_integral _Indexes, typename... _Flags>
284requires detail::gather_from_requires<_Vp, _Range, _Indexes>
285constexpr auto
286partial_gather_from(_Range&& r, const typename _Indexes::mask_type& mask,
287 const _Indexes& indexes, flags<_Flags...> f = {})
288{
289 using _RangeSimd = rebind_t<std::ranges::range_value_t<_Range>, _Indexes>;
290 using _OutSimd = std::conditional_t<std::same_as<_Vp, std::nullptr_t>, _RangeSimd, _Vp>;
291
292 detail::mandates_for_load(r, _OutSimd{}, f);
293
294 if (std::is_constant_evaluated())
295 return _OutSimd([=](auto i) -> typename _OutSimd::value_type { return mask[i] ? r[indexes[i]] : typename _OutSimd::value_type(); });
296 else
297 return _OutSimd(detail::gather_from(target, r, mask, indexes, f));
298}
299
301template<typename _Vp = std::nullptr_t, detail::contiguous_sized_range _Range, vec_integral _Indexes, typename... _Flags>
302requires detail::gather_from_requires<_Vp, _Range, _Indexes>
303constexpr auto
304unchecked_gather_from(_Range&& r, const typename _Indexes::mask_type& mask,
305 const _Indexes& indexes, flags<_Flags...> f = {})
306 { return partial_gather_from<_Vp>(r, mask, indexes, detail::add_unchecked_flag(f)); }
307
316template <detail::contiguous_sized_range _Range, vec_type _Vp, vec_integral _Indexes, typename... _Flags>
317requires detail::scatter_to_requires<_Vp, _Range, _Indexes>
318constexpr void
319partial_scatter_to(const _Vp& values, _Range&& r, const _Indexes& indexes, flags<_Flags...> f = {})
320{
321 detail::mandates_for_store(values, r, f);
322
323 if (std::is_constant_evaluated())
324 for (int i=0; i<_Vp::size; ++i) r[indexes[i]] = values[i];
325 else
326 detail::scatter_to(target, values, r, indexes, f);
327}
328
335template <detail::contiguous_sized_range _Range, vec_type _Vp, vec_integral _Indexes, typename... _Flags>
336requires detail::scatter_to_requires<_Vp, _Range, _Indexes>
337constexpr void
338partial_scatter_to(const _Vp& values, _Range&& r, const typename _Indexes::mask_type& mask, const _Indexes& indexes, flags<_Flags...> f = {})
339{
340 detail::mandates_for_store(values, r, f);
341
342 if (std::is_constant_evaluated())
343 {
344 for (int i=0; i<_Vp::size; ++i)
345 if (mask[i])
346 r[indexes[i]] = values[i];
347 }
348 else
349 detail::scatter_to(target, values, r, mask, indexes, f);
350}
351
353template <detail::contiguous_sized_range _Range, vec_type _Vp, vec_integral _Indexes, typename... _Flags>
354requires detail::scatter_to_requires<_Vp, _Range, _Indexes>
355constexpr void
356unchecked_scatter_to(const _Vp& values, _Range&& r, const _Indexes& indexes, flags<_Flags...> f = {})
357 { partial_scatter_to(values, r, indexes, detail::add_unchecked_flag(f)); }
358
360template <detail::contiguous_sized_range _Range, vec_type _Vp, vec_integral _Indexes, typename... _Flags>
361requires detail::scatter_to_requires<_Vp, _Range, _Indexes>
362constexpr void
363unchecked_scatter_to(const _Vp& values, _Range&& r, const typename _Indexes::mask_type& mask, const _Indexes& indexes, flags<_Flags...> f = {})
364 { partial_scatter_to(values, r, mask, indexes, detail::add_unchecked_flag(f)); }
365
367
368} // namespace _XVEC_NAMESPACE::simd
Requirements for the gather interfaces.
Definition permute.hpp:57
Requirements for the scatter interfaces.
Definition permute.hpp:65
Check that two simdable values (mask, simds, or a mixture) have the same number of elements.
Definition core.hpp:78
Definition core.hpp:61
basic_mask< sizeof(_Tp), simd_fixed_size_abi< _Np > > mask
Definition core.hpp:383
constexpr void partial_scatter_to(const _Vp &values, _Range &&r, const _Indexes &indexes, flags< _Flags... > f={})
Scatter the contents of a basic_vec into memory at the integral index positions supplied in another b...
Definition permute.hpp:319
constexpr _V compress(const _V &v, const typename _V::mask_type &selector)
Compress a basic_vec or basic_mask value by a mask. The elements corresponding to any set mask bits w...
Definition permute.hpp:145
constexpr auto unchecked_gather_from(_Range &&r, const _Indexes &indexes, flags< _Flags... > f={})
Unchecked variant of gather.
Definition permute.hpp:275
constexpr _V expand(const _V &v, const typename _V::mask_type &selector, const _V &original={})
Expand consecutive input elements into positions selected by a mask.
Definition permute.hpp:214
constexpr void unchecked_scatter_to(const _Vp &values, _Range &&r, const _Indexes &indexes, flags< _Flags... > f={})
Unchecked variant of scatter.
Definition permute.hpp:356
constexpr auto permute(const _Vp &v, detail::index_generator_function auto &&fn)
Permute the elements in a vec value using a generator function to compute the indexing.
Definition permute.hpp:94
constexpr auto partial_gather_from(_Range &&r, const _Indexes &indexes, flags< _Flags... > f={})
Gather the contents of a contiguous range using the integral indexes supplied in another vec value.
Definition permute.hpp:258
constexpr auto simd_bit_cast(const _From &v) noexcept
A simd-generic version of simd_bit_cast which directly invokes std::bit_cast.
Definition core.hpp:585
constexpr auto to_container(const _V &v)
Definition permute.hpp:49
constexpr basic_vec< _Tp, _AbiIp > permute(VENDOR tag, const basic_vec< _Tp, _AbiTp > &v, const basic_vec< _Ip, _AbiIp > &indexes)
Definition permute.hpp:276
constexpr simd_size_type new_simd_size
Utility to get new vec or mask size.
Definition permute.hpp:23
constexpr auto generated_permute(x86_avx512_tag, const basic_mask< _Bytes, _Abi > &m, _Gp)
Use a generator to permute a compact bit mask.
Definition permute.hpp:341
constexpr _M vec_to_mask(const _V &s)
Definition permute.hpp:44
Definition core.hpp:393