Intel xvec
Experimental C++ simd class library
Loading...
Searching...
No Matches
loadstore.hpp
1//===----------------------------------------------------------------------===//
2//
3// Copyright (C) 2021 Intel Corporation
4//
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#pragma once
10
11#include <xvec/detail/config.hpp>
12
13#include <cstddef> // std::size_t
14#include <iterator> // std::contiguous_iterator, std::iter_value_t
15
16namespace _XVEC_NAMESPACE::simd
17{
18
19namespace detail
20{
21 template<typename _Vp, std::ranges::range _Range>
22 using load_return_type_range = std::conditional_t<std::same_as<_Vp, std::nullptr_t>,
23 basic_vec<std::ranges::range_value_t<_Range>>, _Vp>;
24
25 template<typename _Vp, std::contiguous_iterator _It>
26 using load_return_type_iter = std::conditional_t<std::same_as<_Vp, std::nullptr_t>,
27 basic_vec<std::iter_value_t<_It>>, _Vp>;
28
29 template<typename _Vp, std::ranges::range _Range>
30 using load_mask_type_range = typename load_return_type_range<_Vp, _Range>::mask_type;
31
32 template<typename _Vp, std::contiguous_iterator _It>
33 using load_mask_type_iter = typename load_return_type_iter<_Vp, _It>::mask_type;
34}
35
46
63template<typename _Vp = std::nullptr_t, detail::contiguous_sized_range _Range, typename... _Flags>
64constexpr detail::load_return_type_range<_Vp, _Range> partial_load(_Range&& r, flags<_Flags...> f = {})
65{
66 using _R = detail::load_return_type_range<_Vp, _Range>;
67 if (std::is_constant_evaluated())
68 return detail::load<_R>(generic_tag{}, std::span(r), f);
69 else
70 {
71 constexpr auto sourceSize = detail::span_extent_v<_Range>;
72 if constexpr (sourceSize == std::dynamic_extent)
73 // Nothing known about the size of the source extent, so hand off to the
74 // target to dynamically load what is required.
75 return detail::load<_R>(target, std::span(r), f);
76 else if constexpr (sourceSize == 0)
77 // The source is known to be empty.
78 return _R();
79 else if constexpr (sourceSize <= _Vp::size())
80 {
81 // The source is smaller, so load what is available and default initialise
82 // the rest. No checks are required because it ends up exactly the correct
83 // size.
84 return grow<_R::size()>(resize_t<sourceSize, _R>(r));
85 }
86 else
87 // The source is bigger, so it is safe to load the entire vec without further checks.
88 return detail::load<_R>(target, std::span(r), detail::add_unchecked_flag(f));
89 }
90}
91
96template<typename _Vp = std::nullptr_t, detail::contiguous_sized_range _Range, typename... _Flags>
97constexpr detail::load_return_type_range<_Vp, _Range>
98partial_load(_Range&& r, const detail::load_mask_type_range<_Vp, _Range>& m, flags<_Flags...> f = {})
99{
100 using _R = detail::load_return_type_range<_Vp, _Range>;
101 if (std::is_constant_evaluated())
102 return detail::load_masked<_R>(generic_tag{}, std::span(r), m, f);
103 else
104 {
105 constexpr auto sourceSize = detail::span_extent_v<_Range>;
106 if constexpr (sourceSize == std::dynamic_extent)
107 // Nothing known about the source extents, so use the target's own version.
108 return detail::load_masked<_R>(target, std::span(r), m, f);
109 else if constexpr (sourceSize == 0)
110 return _R();
111 else if (sourceSize <= _Vp::size())
112 {
113 // The source is smaller, so use the mask to restrict the load size.
114 // Because the load has been restricted already, the range checks can be
115 // removed.
116 constexpr auto shorterMask = mask_from_count<_R>(sourceSize);
117 return detail::load_masked<_R>(target, std::span(r), m & shorterMask, detail::add_unchecked_flag(f));
118 }
119 else
120 // The source is bigger, so it is safe to load the entire vec without further checks.
121 return detail::load_masked<_R>(target, std::span(r), m, detail::add_unchecked_flag(f));
122 }
123}
124
129template<typename _Vp = std::nullptr_t, std::contiguous_iterator _It, typename... _Flags>
130constexpr detail::load_return_type_iter<_Vp, _It> partial_load(_It first, std::iter_difference_t<_It> n, flags<_Flags...> f = {})
131 { return partial_load<_Vp>(std::span(first, n), f); }
132
138template<typename _Vp = std::nullptr_t, std::contiguous_iterator _It, typename... _Flags>
139constexpr detail::load_return_type_iter<_Vp, _It>
140partial_load(_It first, std::iter_difference_t<_It> n,
141 const detail::load_mask_type_iter<_Vp, _It>& m, flags<_Flags...> f = {})
142 { return partial_load<_Vp>(std::span(first, n), m, f); }
143
148template<typename _Vp = std::nullptr_t, std::contiguous_iterator _It, std::sized_sentinel_for<_It> _Sent, typename... _Flags>
149constexpr detail::load_return_type_iter<_Vp, _It> partial_load(_It first, _Sent last, flags<_Flags...> f = {})
150 { return partial_load<_Vp>(std::span(first, static_cast<std::size_t>(last - first)), f); }
151
157template<typename _Vp = std::nullptr_t, std::contiguous_iterator _It, std::sized_sentinel_for<_It> _Sent, typename... _Flags>
158constexpr detail::load_return_type_iter<_Vp, _It>
159partial_load(_It first, _Sent last, const detail::load_mask_type_iter<_Vp, _It>& m, flags<_Flags...> f = {})
160 { return partial_load<_Vp>(std::span(first, static_cast<std::size_t>(last - first)), m, f); }
161
165template<typename _Vp = std::nullptr_t, detail::contiguous_sized_range _Range, typename... _Flags>
166constexpr detail::load_return_type_range<_Vp, _Range> unchecked_load(_Range&& r, flags<_Flags...> f = {})
167 { return partial_load<_Vp>(r, detail::add_unchecked_flag(f)); }
168
173template<typename _Vp = std::nullptr_t, detail::contiguous_sized_range _Range, typename... _Flags>
174constexpr detail::load_return_type_range<_Vp, _Range>
175unchecked_load(_Range&& r, const detail::load_mask_type_range<_Vp, _Range>& m, flags<_Flags...> f = {})
176 { return partial_load<_Vp>(r, m, detail::add_unchecked_flag(f)); }
177
182template<typename _Vp = std::nullptr_t, std::contiguous_iterator _It, typename... _Flags>
183constexpr detail::load_return_type_iter<_Vp, _It> unchecked_load(_It first, std::iter_difference_t<_It> n, flags<_Flags...> f = {})
184 { return partial_load<_Vp>(std::span(first, n), detail::add_unchecked_flag(f)); }
185
191template<typename _Vp = std::nullptr_t, std::contiguous_iterator _It, typename... _Flags>
192constexpr detail::load_return_type_iter<_Vp, _It>
193unchecked_load(_It first, std::iter_difference_t<_It> n,
194 const detail::load_mask_type_iter<_Vp, _It>& m, flags<_Flags...> f = {})
195 { return partial_load<_Vp>(std::span(first, n), m, detail::add_unchecked_flag(f)); }
196
201template<typename _Vp = std::nullptr_t, std::contiguous_iterator _It, std::sized_sentinel_for<_It> _Sent, typename... _Flags>
202constexpr detail::load_return_type_iter<_Vp, _It> unchecked_load(_It first, _Sent last, flags<_Flags...> f = {})
203 { return partial_load<_Vp>(std::span{first, static_cast<std::size_t>(last - first)}, detail::add_unchecked_flag(f)); }
204
210template<typename _Vp = std::nullptr_t, std::contiguous_iterator _It, std::sized_sentinel_for<_It> _Sent, typename... _Flags>
211constexpr detail::load_return_type_iter<_Vp, _It>
212unchecked_load(_It first, _Sent last, const detail::load_mask_type_iter<_Vp, _It>& m, flags<_Flags...> f = {})
213 { return partial_load<_Vp>(std::span(first, static_cast<std::size_t>(last - first)), m, detail::add_unchecked_flag(f)); }
214
229template<typename _Tp, typename _Abi, detail::contiguous_sized_range _Range, typename... _Flags>
230requires std::indirectly_writable<std::ranges::iterator_t<_Range>, _Tp>
231constexpr void partial_store(const basic_vec<_Tp, _Abi>& value, _Range&& r, flags<_Flags...> f = {})
232{
233 if (std::is_constant_evaluated())
234 detail::store(generic_tag(), value, std::span(r), f);
235 else
236 {
237 constexpr auto destSize = detail::span_extent_v<_Range>;
238 if constexpr (destSize == std::dynamic_extent)
239 // Nothing known about the destination extents, so use the target's own store to handle it at runtime.
240 detail::store(target, value, std::span(r), f);
241 else if constexpr (destSize == 0)
242 {} // Nothing to write to.
243 else if constexpr (destSize <= basic_vec<_Tp, _Abi>::size())
244 // The destination is smaller so restrict the values to only those that fit.
245 detail::store(target, take<destSize>(value), std::span(r), detail::add_unchecked_flag(f));
246 else
247 // The destination is bigger so it is safe to store the entire vec without further checks.
248 detail::store(target, value, std::span(r), detail::add_unchecked_flag(f));
249 }
250}
251
257template<typename _Tp, typename _Abi, detail::contiguous_sized_range _Range, typename... _Flags>
258requires std::indirectly_writable<std::ranges::iterator_t<_Range>, _Tp>
259constexpr void partial_store(const basic_vec<_Tp, _Abi>& value, _Range&& r,
260 const typename basic_vec<_Tp, _Abi>::mask_type& m, flags<_Flags...> f = {})
261{
262 if (std::is_constant_evaluated())
263 detail::store_masked(generic_tag{}, value, std::span(r), m, f);
264 else
265 {
266 constexpr auto destSize = detail::span_extent_v<_Range>;
267 if constexpr (destSize == std::dynamic_extent)
268 // Unknown destination size - hand off to the target to deal with the runtime behaviuor.
269 detail::store_masked(target, value, std::span(r), m, f);
270 else if constexpr (destSize == 0)
271 {} // Nothing to write to.
272 else if constexpr (destSize <= basic_vec<_Tp, _Abi>::size())
273 // The destination is smaller so restrict the values to only those that fit.
274 detail::store_masked(target, take<destSize>(value), std::span(r), take<destSize>(m), detail::add_unchecked_flag(f));
275 else
276 // The destination is bigger so it is safe to store the entire vec without further checks.
277 detail::store_masked(target, value, std::span(r), m, detail::add_unchecked_flag(f));
278 }
279 //
280}
281
287template<typename _Tp, typename _Abi, std::contiguous_iterator _It, typename... _Flags>
288requires std::indirectly_writable<_It, _Tp>
289constexpr void partial_store(const basic_vec<_Tp, _Abi>& value, _It first, std::iter_difference_t<_It> n, flags<_Flags...> f = {})
290 { partial_store(value, std::span(first, n), f); }
291
298template<typename _Tp, typename _Abi, std::contiguous_iterator _It, typename... _Flags>
299requires std::indirectly_writable<_It, _Tp>
300constexpr void partial_store(const basic_vec<_Tp, _Abi>& value, _It first, std::iter_difference_t<_It> n,
301 const typename basic_vec<_Tp, _Abi>::mask_type& m, flags<_Flags...> f = {})
302 { partial_store(value, std::span(first, n), m, f); }
303
309template<typename _Tp, typename _Abi, std::contiguous_iterator _It, std::sized_sentinel_for<_It> _Sent, typename... _Flags>
310requires std::indirectly_writable<_It, _Tp>
311constexpr void partial_store(const basic_vec<_Tp, _Abi>& value, _It first, _Sent last, flags<_Flags...> f = {})
312 { partial_store(value, std::span(first, static_cast<std::size_t>(last - first)), f); }
313
320template<typename _Tp, typename _Abi, std::contiguous_iterator _It, std::sized_sentinel_for<_It> _Sent, typename... _Flags>
321requires std::indirectly_writable<_It, _Tp>
322constexpr void partial_store(const basic_vec<_Tp, _Abi>& value, _It first, _Sent last,
323 const typename basic_vec<_Tp, _Abi>::mask_type& m, flags<_Flags...> f = {})
324 { partial_store(value, std::span(first, static_cast<std::size_t>(last - first)), m, f); }
325
330template<typename _Tp, typename _Abi, detail::contiguous_sized_range _Range, typename... _Flags>
331requires std::indirectly_writable<std::ranges::iterator_t<_Range>, _Tp>
332constexpr void unchecked_store(const basic_vec<_Tp, _Abi>& value, _Range&& r, flags<_Flags...> f = {})
333 { partial_store(value, std::span(r), detail::add_unchecked_flag(f)); }
334
340template<typename _Tp, typename _Abi, detail::contiguous_sized_range _Range, typename... _Flags>
341requires std::indirectly_writable<std::ranges::iterator_t<_Range>, _Tp>
342constexpr void unchecked_store(const basic_vec<_Tp, _Abi>& value, _Range&& r,
343 const typename basic_vec<_Tp, _Abi>::mask_type& m, flags<_Flags...> f = {})
344 { partial_store(value, std::span(r), m, detail::add_unchecked_flag(f)); }
345
351template<typename _Tp, typename _Abi, std::contiguous_iterator _It, typename... _Flags>
352requires std::indirectly_writable<_It, _Tp>
353constexpr void unchecked_store(const basic_vec<_Tp, _Abi>& value, _It first, std::iter_difference_t<_It> n, flags<_Flags...> f = {})
354 { partial_store(value, std::span(first, n), detail::add_unchecked_flag(f)); }
355
362template<typename _Tp, typename _Abi, std::contiguous_iterator _It, typename... _Flags>
363requires std::indirectly_writable<_It, _Tp>
364constexpr void unchecked_store(const basic_vec<_Tp, _Abi>& value, _It first, std::iter_difference_t<_It> n,
365 const typename basic_vec<_Tp, _Abi>::mask_type& m, flags<_Flags...> f = {})
366 { partial_store(value, std::span(first, n), m, detail::add_unchecked_flag(f)); }
367
373template<typename _Tp, typename _Abi, std::contiguous_iterator _It, std::sized_sentinel_for<_It> _Sent, typename... _Flags>
374requires std::indirectly_writable<_It, _Tp>
375constexpr void unchecked_store(const basic_vec<_Tp, _Abi>& value, _It first, _Sent last, flags<_Flags...> f = {})
376 { partial_store(value, std::span(first, static_cast<std::size_t>(last - first)), detail::add_unchecked_flag(f)); }
377
384template<typename _Tp, typename _Abi, std::contiguous_iterator _It, std::sized_sentinel_for<_It> _Sent, typename... _Flags>
385requires std::indirectly_writable<_It, _Tp>
386constexpr void unchecked_store(const basic_vec<_Tp, _Abi>& value, _It first, _Sent last,
387 const typename basic_vec<_Tp, _Abi>::mask_type& m, flags<_Flags...> f = {})
388 { partial_store(value, std::span(first, static_cast<std::size_t>(last - first)), m, detail::add_unchecked_flag(f)); }
389
391
392} // namespace _XVEC_NAMESPACE::simd
Vector of elements sharing an element type and ABI.
Definition simd:161
basic_mask< sizeof(_Tp), _Abi > mask_type
Definition simd:184
constexpr detail::load_return_type_range< _Vp, _Range > unchecked_load(_Range &&r, flags< _Flags... > f={})
Unchecked load from a contiguous sized range.
Definition loadstore.hpp:166
constexpr void unchecked_store(const basic_vec< _Tp, _Abi > &value, _Range &&r, flags< _Flags... > f={})
Unchecked store into a contiguous sized range.
Definition loadstore.hpp:332
constexpr void partial_store(const basic_vec< _Tp, _Abi > &value, _Range &&r, flags< _Flags... > f={})
Store the elements from a basic_vec value into a contiguous range, performing a suitable conversion i...
Definition loadstore.hpp:231
constexpr detail::load_return_type_range< _Vp, _Range > partial_load(_Range &&r, flags< _Flags... > f={})
Load the basic_vec elements from a contiguous sized range, performing a suitable conversion if necess...
Definition loadstore.hpp:64
Definition core.hpp:393