Intel xvec
Experimental C++ simd class library
Loading...
Searching...
No Matches
operators.hpp
1//===----------------------------------------------------------------------===//
2//
3// Copyright (C) 2021 Intel Corporation
4//
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#pragma once
10
11#include <xvec/detail/config.hpp>
12
13#include <bitset>
14#include <cstddef> // std::size_t
15#include <type_traits>
16#include <algorithm> // std::min, std::max
17#include <utility> // std::index_sequence
18#include <cstring> // for std::memcpy
19
20#include <xvec/detail/utilities.hpp>
21#include <xvec/detail/core.hpp>
22
23namespace _XVEC_NAMESPACE::simd::detail
24{
25
27template<vec_or_mask_type _Vp, typename _Op>
28constexpr auto builtin_operator(generic_tag, _Vp value, _Op op) {
29 auto t = value.to_builtin();
30 decltype(op(t)) r;
31
32 // Ensure that the target ISA for op() is embedded here.
33#if __clang_major__ > 14
34 // Clang 15+ uses a different attribute to force inlining.
35 [[clang::always_inline]] r = op(t);
36#else
37 #pragma forceinline
38 r = op(t);
39#endif
40
41 if constexpr (mask_operator<_Op>) return _Vp::mask_type::from_builtin(r);
42 else return _Vp::from_builtin(r);
43}
44
47template<vec_type _Vp>
48constexpr auto builtin_operator(generic_tag, _Vp value, std::logical_not<>) { return value == _Vp(); }
49
51template<vec_or_mask_type _Vp, typename _Op> // Concept - binary operator of some type
52constexpr auto builtin_operator(generic_tag, _Vp lhs, _Vp rhs, _Op op) {
53 auto lhsd = lhs.to_builtin();
54 auto rhsd = rhs.to_builtin();
55 decltype(op(lhsd, rhsd)) r;
56
57 // Ensure that the target ISA for op() is embedded here.
58#if __clang_major__ > 14
59 // Clang 15+ uses a different attribute to force inlining.
60 [[clang::always_inline]] r = op(lhsd, rhsd);
61#else
62 #pragma forceinline
63 r = op(lhsd, rhsd);
64#endif
65
66 if constexpr (mask_operator<_Op>) return _Vp::mask_type::from_builtin(typename _Vp::mask_type::builtin_type(r));
67 else return _Vp::from_builtin(r);
68}
69
73template<vec_integral _Vp, typename _Op>
74requires (any_same_v<_Op, std::divides<>, std::modulus<>>)
75constexpr _Vp builtin_operator(generic_tag, _Vp lhs, _Vp rhs, _Op)
76 { return _Vp([=](auto i) -> typename _Vp::value_type { return _Op{}(lhs[i], rhs[i]); }); }
77
79template<vec_complex _Vp>
80constexpr auto builtin_operator(generic_tag, const _Vp& lhs, const _Vp& rhs, std::multiplies<>)
81 { return fma(lhs, rhs, _Vp{});}
82
85template<vec_type _Vp, mask_operator _Op>
86constexpr auto builtin_operator(compact_mask_tag, const _Vp& x, const _Vp& y, _Op)
87 { return typename _Vp::mask_type{[=](auto i) -> bool { return _Op{}(x[i], y[i]); }}; }
88
92template<vec_complex _Vp, mask_operator _Op>
93constexpr typename _Vp::mask_type builtin_operator(generic_tag, _Vp lhs, _Vp rhs, _Op) {
94 using _Tp = typename _Vp::value_type::value_type;
95
96 static_assert(sizeof(_Tp) <= 8, "Can't deal with 128-bit elements like complex<double> yet");
97
98 using binop_type = std::remove_cvref_t<_Op>;
99 constexpr bool isEqualOp = std::is_same_v<binop_type, std::equal_to<>>;
100 constexpr bool isNotEqualOp = std::is_same_v<binop_type, std::not_equal_to<>>;
101 static_assert(isEqualOp || isNotEqualOp, "Complex values only allow equality comparisons");
102
103 // Element-wise comparison to generate a mask for each individual value.
104 const auto compareElements = _Op{}(simd_bit_cast<_Tp>(lhs), simd_bit_cast<_Tp>(rhs));
105
106 // Treat the mask as having half as many elements which are twice as big. Adjacent elements will be 0 or -1.
107 const auto cmplxMask = simd_bit_cast<container_for_type<typename _Vp::value_type>>(-compareElements);
108
109 // Compare wider elements to all 0 or all 1.
110 if constexpr (isEqualOp)
111 return typename _Vp::mask_type(cmplxMask == cw<-1>);
112 else
113 return typename _Vp::mask_type(cmplxMask != cw<0>);
114}
115
120template<typename FN, vec_type _Vp, typename... Args>
121 requires (std::same_as<Args, _Vp> && ...)
122constexpr auto make_mask_from_op(const _Vp& first, const Args&... args) noexcept {
123 using _UnsignedMask = typename _Vp::mask_type::traits::unsigned_vec_for_mask;
124 using _ME = typename _UnsignedMask::value_type;
125 auto m = _UnsignedMask{[=](auto i) -> _ME {
126 return FN{}(first[i], (args[i])...) ? ~_ME() : _ME();
127 }};
128 return typename _Vp::mask_type(m != _UnsignedMask());
129}
130
135template<typename FN, vec_type _Vp, vec_type... _Vps>
136constexpr auto do_operator(const _Vp& first, const _Vps&... rest) noexcept {
137 // Functions which return bool will return masks, otherwise return the same type as the input.
138 using _Rp = std::conditional_t<mask_operator<FN>, typename _Vp::mask_type, _Vp>;
139
140 if (std::is_constant_evaluated())
141 // Simplest to call the generic constructor. No need for target-specific.
142 return _Rp{[=](auto i) { return typename _Rp::value_type(FN{}(first[i], rest[i]...)); }};
144 // Builtin types can call the compiler's support directly.
145 return detail::builtin_operator(target, first, rest..., FN{});
146 else if constexpr (requires { {simd_operator(first, rest..., FN{})}; })
147 // Custom optimised function. The builtin above takes precedence to avoid
148 // the user overriding a target's own implementation.
149 return simd_operator(first, rest..., FN{});
150 else if constexpr (mask_operator<FN>)
151 // Auto-vectorise a mask generator. For performance, mask operators go
152 // through several steps which allow the compiler to auto-vectorise
153 // effectively.
154 return make_mask_from_op<FN>(first, rest...);
155 else
156 // Auto-vectorise a normal operator. This is the fallback for non-builtin
157 // types which don't have a custom implementation.
158 return _Rp{[=](auto i) { return static_cast<typename _Vp::value_type>(FN{}(first[i], rest[i]...)); }};
159}
160
162template<vec_type _To, vec_type _From>
163constexpr _To convert_operator(const _From& v) noexcept {
164 using _Tp = typename _To::value_type;
166 return _To::from_builtin(__builtin_convertvector(v.to_builtin(), typename _To::builtin_type));
167 else if constexpr (requires {
168 { simd_convert(v, _XVEC_NAMESPACE::simd::convert_to<_Tp>) } -> std::convertible_to<_To>;
169 })
170 return simd_convert(v, _XVEC_NAMESPACE::simd::convert_to<_Tp>);
171 else
172 return _To([=](auto i) { return static_cast<_Tp>(v[i]); });
173}
174
175} // namespace _XVEC_NAMESPACE::simd::detail
Definition core.hpp:335
Definition core.hpp:61
constexpr auto do_operator(const _Vp &first, const _Vps &... rest) noexcept
Definition operators.hpp:136
constexpr auto builtin_operator(generic_tag, _Vp value, _Op op)
General purpose unary operator. Passes through to the compiler's own builtin.
Definition operators.hpp:28
constexpr auto make_mask_from_op(const _Vp &first, const Args &... args) noexcept
Definition operators.hpp:122
constexpr _To convert_operator(const _From &v) noexcept
Conversion operator for basic_vec. All gate-keeping has been done, just do the convert.
Definition operators.hpp:163
Definition target_config.hpp:161
Definition target_config.hpp:23