25#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26#define _GLIBCXX_EXPERIMENTAL_SIMD_H
28#if __cplusplus >= 201703L
30#include "simd_detail.h"
34#ifdef _GLIBCXX_DEBUG_UB
43#if _GLIBCXX_SIMD_X86INTRIN
45#elif _GLIBCXX_SIMD_HAVE_NEON
46#pragma GCC diagnostic push
49#pragma GCC diagnostic ignored "-Wnarrowing"
51#pragma GCC diagnostic pop
77_GLIBCXX_SIMD_BEGIN_NAMESPACE
79#if !_GLIBCXX_SIMD_X86INTRIN
80using __m128 [[__gnu__::__vector_size__(16)]] = float;
81using __m128d [[__gnu__::__vector_size__(16)]] = double;
82using __m128i [[__gnu__::__vector_size__(16)]] =
long long;
83using __m256 [[__gnu__::__vector_size__(32)]] = float;
84using __m256d [[__gnu__::__vector_size__(32)]] = double;
85using __m256i [[__gnu__::__vector_size__(32)]] =
long long;
86using __m512 [[__gnu__::__vector_size__(64)]] = float;
87using __m512d [[__gnu__::__vector_size__(64)]] = double;
88using __m512i [[__gnu__::__vector_size__(64)]] =
long long;
110template <
int _UsedBytes>
113template <
int _UsedBytes>
114 struct _VecBltnBtmsk;
116template <
typename _Tp,
int _Np>
117 using _VecN = _VecBuiltin<
sizeof(_Tp) * _Np>;
119template <
int _UsedBytes = 16>
120 using _Sse = _VecBuiltin<_UsedBytes>;
122template <
int _UsedBytes = 32>
123 using _Avx = _VecBuiltin<_UsedBytes>;
125template <
int _UsedBytes = 64>
126 using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
128template <
int _UsedBytes = 16>
129 using _Neon = _VecBuiltin<_UsedBytes>;
134using __avx512 = _Avx512<>;
135using __neon = _Neon<>;
136using __neon128 = _Neon<16>;
137using __neon64 = _Neon<8>;
140template <
typename _Tp,
size_t _Np,
typename...>
144 using fixed_size = _Fixed<_Np>;
146using scalar = _Scalar;
151template <
typename _Tp>
154template <
typename _Tp>
157template <
typename _Tp,
typename _Abi>
160template <
typename _Tp,
typename _Abi>
163template <
typename _Tp,
typename _Abi>
168struct element_aligned_tag
170 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
171 static constexpr size_t _S_alignment =
alignof(_Up);
173 template <
typename _Tp,
typename _Up>
174 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
179struct vector_aligned_tag
181 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
182 static constexpr size_t _S_alignment
183 = std::__bit_ceil(
sizeof(_Up) * _Tp::size());
185 template <
typename _Tp,
typename _Up>
186 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
188 {
return static_cast<_Up*
>(__builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>)); }
191template <
size_t _Np>
struct overaligned_tag
193 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
194 static constexpr size_t _S_alignment = _Np;
196 template <
typename _Tp,
typename _Up>
197 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
199 {
return static_cast<_Up*
>(__builtin_assume_aligned(__ptr, _Np)); }
202inline constexpr element_aligned_tag element_aligned = {};
204inline constexpr vector_aligned_tag vector_aligned = {};
207 inline constexpr overaligned_tag<_Np> overaligned = {};
211 using _SizeConstant = integral_constant<size_t, _Xp>;
213constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
214constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
215constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
216constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
217constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
218constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
219constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
220constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
221constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
222constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
223constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
224constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
225constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
226constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
227constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
228constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
229constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
230constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
231constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
232constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
233constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
234constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
235constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
236constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
237constexpr inline bool __have_avx512bitalg = _GLIBCXX_SIMD_HAVE_AVX512BITALG;
238constexpr inline bool __have_avx512vbmi2 = _GLIBCXX_SIMD_HAVE_AVX512VBMI2;
239constexpr inline bool __have_avx512vbmi = _GLIBCXX_SIMD_HAVE_AVX512VBMI;
240constexpr inline bool __have_avx512ifma = _GLIBCXX_SIMD_HAVE_AVX512IFMA;
241constexpr inline bool __have_avx512cd = _GLIBCXX_SIMD_HAVE_AVX512CD;
242constexpr inline bool __have_avx512vnni = _GLIBCXX_SIMD_HAVE_AVX512VNNI;
243constexpr inline bool __have_avx512vpopcntdq = _GLIBCXX_SIMD_HAVE_AVX512VPOPCNTDQ;
244constexpr inline bool __have_avx512vp2intersect = _GLIBCXX_SIMD_HAVE_AVX512VP2INTERSECT;
246constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
247constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
248constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
249constexpr inline bool __support_neon_float =
250#if defined __GCC_IEC_559
252#elif defined __FAST_MATH__
259constexpr inline bool __have_power10vec =
true;
261constexpr inline bool __have_power10vec =
false;
263#ifdef __POWER9_VECTOR__
264constexpr inline bool __have_power9vec =
true;
266constexpr inline bool __have_power9vec =
false;
268#if defined __POWER8_VECTOR__
269constexpr inline bool __have_power8vec =
true;
271constexpr inline bool __have_power8vec = __have_power9vec;
274constexpr inline bool __have_power_vsx =
true;
276constexpr inline bool __have_power_vsx = __have_power8vec;
278#if defined __ALTIVEC__
279constexpr inline bool __have_power_vmx =
true;
281constexpr inline bool __have_power_vmx = __have_power_vsx;
288#ifdef math_errhandling
292 template <
int = math_errhandling>
294 __handle_fpexcept_impl(
int)
295 {
return math_errhandling & MATH_ERREXCEPT; }
301 __handle_fpexcept_impl(
float)
303#if defined __FAST_MATH__
311 static constexpr bool _S_handle_fpexcept = __handle_fpexcept_impl(0);
313 constexpr std::uint_least64_t
314 __floating_point_flags()
316 std::uint_least64_t __flags = 0;
317 if constexpr (_S_handle_fpexcept)
321#elif __FINITE_MATH_ONLY__
323#elif __GCC_IEC_559 < 2
326 __flags |= (__FLT_EVAL_METHOD__ + 1) << 3;
330 constexpr std::uint_least64_t
333 if constexpr (__have_mmx || __have_sse)
338 | (__have_ssse3 << 4)
339 | (__have_sse4_1 << 5)
340 | (__have_sse4_2 << 6)
345 | (__have_bmi2 << 11)
346 | (__have_lzcnt << 12)
347 | (__have_sse4a << 13)
349 | (__have_fma4 << 15)
350 | (__have_f16c << 16)
351 | (__have_popcnt << 17)
352 | (__have_avx512f << 18)
353 | (__have_avx512dq << 19)
354 | (__have_avx512vl << 20)
355 | (__have_avx512bw << 21)
356 | (__have_avx512bitalg << 22)
357 | (__have_avx512vbmi2 << 23)
358 | (__have_avx512vbmi << 24)
359 | (__have_avx512ifma << 25)
360 | (__have_avx512cd << 26)
361 | (__have_avx512vnni << 27)
362 | (__have_avx512vpopcntdq << 28)
363 | (__have_avx512vp2intersect << 29);
364 else if constexpr (__have_neon)
366 | (__have_neon_a32 << 1)
367 | (__have_neon_a64 << 2)
368 | (__have_neon_a64 << 2)
369 | (__support_neon_float << 3);
370 else if constexpr (__have_power_vmx)
371 return __have_power_vmx
372 | (__have_power_vsx << 1)
373 | (__have_power8vec << 2)
374 | (__have_power9vec << 3)
375 | (__have_power10vec << 4);
382 struct _OdrEnforcer {};
385 template <std::uint_least64_t...>
386 struct _MachineFlagsTemplate {};
399 _MachineFlagsTemplate<__machine_flags(), __floating_point_flags()>>;
403 template <
typename _Tp>
404 _GLIBCXX_SIMD_INTRINSIC
constexpr
406 operator()(_Tp __a, _Tp __b)
const
409 return min(__a, __b);
415 template <
typename _Tp>
416 _GLIBCXX_SIMD_INTRINSIC
constexpr
418 operator()(_Tp __a, _Tp __b)
const
421 return max(__a, __b);
428template <
typename _Fp,
size_t... _I>
429 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC
constexpr
431 __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
432 { ((void)__f(_SizeConstant<_I>()), ...); }
434template <
typename _Fp>
435 _GLIBCXX_SIMD_INTRINSIC
constexpr void
436 __execute_on_index_sequence(_Fp&&, index_sequence<>)
439template <
size_t _Np,
typename _Fp>
440 _GLIBCXX_SIMD_INTRINSIC
constexpr void
441 __execute_n_times(_Fp&& __f)
443 __execute_on_index_sequence(
static_cast<_Fp&&
>(__f),
444 make_index_sequence<_Np>{});
449template <
typename _R,
typename _Fp,
size_t... _I>
450 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC
constexpr
452 __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
453 {
return _R{__f(_SizeConstant<_I>())...}; }
455template <
size_t _Np,
typename _R,
typename _Fp>
456 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
457 __generate_from_n_evaluations(_Fp&& __f)
459 return __execute_on_index_sequence_with_return<_R>(
460 static_cast<_Fp&&
>(__f), make_index_sequence<_Np>{});
465template <
size_t... _I,
typename _F0,
typename _FArgs>
466 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC
constexpr
468 __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
469 {
return __f0(__fargs(_SizeConstant<_I>())...); }
471template <
size_t _Np,
typename _F0,
typename _FArgs>
472 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
473 __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
475 return __call_with_n_evaluations(make_index_sequence<_Np>{},
476 static_cast<_F0&&
>(__f0),
477 static_cast<_FArgs&&
>(__fargs));
482template <
size_t _First = 0,
size_t... _It,
typename _Tp,
typename _Fp>
483 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC
constexpr
485 __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
486 {
return __fun(__x[_First + _It]...); }
488template <
size_t _Np,
size_t _First = 0,
typename _Tp,
typename _Fp>
489 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
490 __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
492 return __call_with_subscripts<_First>(
static_cast<_Tp&&
>(__x),
493 make_index_sequence<_Np>(),
494 static_cast<_Fp&&
>(__fun));
501using _UChar =
unsigned char;
502using _SChar =
signed char;
503using _UShort =
unsigned short;
504using _UInt =
unsigned int;
505using _ULong =
unsigned long;
506using _ULLong =
unsigned long long;
507using _LLong =
long long;
511template <
typename _T0,
typename...>
512 struct __first_of_pack
513 {
using type = _T0; };
515template <
typename... _Ts>
516 using __first_of_pack_t =
typename __first_of_pack<_Ts...>::type;
520template <
typename _Tp>
521 typename _Tp::value_type
522 __value_type_or_identity_impl(
int);
524template <
typename _Tp>
526 __value_type_or_identity_impl(
float);
528template <
typename _Tp>
529 using __value_type_or_identity_t
530 =
decltype(__value_type_or_identity_impl<_Tp>(
int()));
534template <
typename _Tp>
535 struct __is_vectorizable :
public is_arithmetic<_Tp> {};
538 struct __is_vectorizable<bool> :
public false_type {};
540template <
typename _Tp>
541 inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
544template <
typename _Tp,
typename = enable_if_t<__is_vectorizable_v<_Tp>>>
545 using _Vectorizable = _Tp;
549template <
typename _Ptr,
typename _ValueType>
550 struct __is_possible_loadstore_conversion
551 : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
554 struct __is_possible_loadstore_conversion<bool, bool> :
true_type {};
557template <
typename _Ptr,
typename _ValueType,
559 __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
560 using _LoadStorePtr = _Ptr;
564template <
typename _Tp,
typename =
void_t<>>
567template <
typename _Tp>
568 inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
571template <
typename _Tp>
572 struct __is_bitmask<_Tp,
573 void_t<decltype(
declval<unsigned&>() = declval<_Tp>() & 1u)>>
578#pragma GCC diagnostic push
579#pragma GCC diagnostic ignored "-Wpedantic"
580template <
size_t _Bytes>
584 static_assert(_Bytes > 0);
585 if constexpr (_Bytes ==
sizeof(int))
587 else if constexpr (_Bytes ==
sizeof(_SChar))
589 else if constexpr (_Bytes ==
sizeof(short))
591 else if constexpr (_Bytes ==
sizeof(long))
593 else if constexpr (_Bytes ==
sizeof(_LLong))
595 #ifdef __SIZEOF_INT128__
596 else if constexpr (_Bytes ==
sizeof(__int128))
599 else if constexpr (_Bytes %
sizeof(int) == 0)
601 constexpr size_t _Np = _Bytes /
sizeof(int);
606 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
609 return __generate_from_n_evaluations<_Np, _Ip>(
610 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
611 return __rhs._M_data[__i] & _M_data[__i];
615 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
618 return __generate_from_n_evaluations<_Np, _Ip>(
619 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
620 return __rhs._M_data[__i] | _M_data[__i];
624 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
627 return __generate_from_n_evaluations<_Np, _Ip>(
628 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
629 return __rhs._M_data[__i] ^ _M_data[__i];
633 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
636 return __generate_from_n_evaluations<_Np, _Ip>(
637 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return ~_M_data[__i]; });
643 static_assert(_Bytes == 0,
"this should be unreachable");
645#pragma GCC diagnostic pop
647template <
typename _Tp>
648 using __int_for_sizeof_t =
decltype(__int_for_sizeof<sizeof(_Tp)>());
651 using __int_with_sizeof_t =
decltype(__int_for_sizeof<_Np>());
655template <
typename _Tp>
659 struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> :
true_type {};
661template <
typename _Tp>
662 inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
666template <
typename _Abi>
669 {
return is_same_v<simd_abi::scalar, _Abi>; }
673template <
template <
int>
class _Abi,
int _Bytes>
675 __abi_bytes_impl(_Abi<_Bytes>*)
678template <
typename _Tp>
680 __abi_bytes_impl(_Tp*)
683template <
typename _Abi>
684 inline constexpr int __abi_bytes_v
685 = __abi_bytes_impl(
static_cast<_Abi*
>(
nullptr));
689template <
typename _Abi>
691 __is_builtin_bitmask_abi()
692 {
return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
696template <
typename _Abi>
700 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
701 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
706template <
typename _Abi>
710 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
711 return _Bytes > 16 && _Bytes <= 32
712 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
717template <
typename _Abi>
721 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
722 return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
727template <
typename _Abi>
731 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
732 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
737template <
typename,
typename _Up>
738 struct __make_dependent
739 {
using type = _Up; };
741template <
typename _Tp,
typename _Up>
742 using __make_dependent_t =
typename __make_dependent<_Tp, _Up>::type;
748template <
typename... _Args>
749 [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE
void
750 __invoke_ub([[maybe_unused]]
const char* __msg, [[maybe_unused]]
const _Args&... __args)
752#ifdef _GLIBCXX_DEBUG_UB
753 __builtin_fprintf(stderr, __msg, __args...);
756 __builtin_unreachable();
762template <
typename _Tp>
763 struct __assert_unreachable
764 {
static_assert(!is_same_v<_Tp, _Tp>,
"this should be unreachable"); };
768template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
770 __size_or_zero_dispatch(
int)
773template <
typename _Tp,
typename _Ap>
775 __size_or_zero_dispatch(
float)
778template <
typename _Tp,
typename _Ap>
779 inline constexpr size_t __size_or_zero_v
780 = __size_or_zero_dispatch<_Tp, _Ap>(0);
784inline constexpr size_t
785__div_roundup(
size_t __a,
size_t __b)
786{
return (__a + __b - 1) / __b; }
795 _GLIBCXX_SIMD_INTRINSIC
constexpr
796 _ExactBool(
bool __b) : _M_data(__b) {}
798 _ExactBool(
int) =
delete;
800 _GLIBCXX_SIMD_INTRINSIC
constexpr
801 operator bool()
const
812template <
typename _Tp>
813 using __may_alias [[__gnu__::__may_alias__]] = _Tp;
818struct _UnsupportedBase
820 _UnsupportedBase() =
delete;
821 _UnsupportedBase(
const _UnsupportedBase&) =
delete;
822 _UnsupportedBase& operator=(
const _UnsupportedBase&) =
delete;
823 ~_UnsupportedBase() =
delete;
839 using _SimdBase = _UnsupportedBase;
840 using _MaskBase = _UnsupportedBase;
842 static constexpr size_t _S_full_size = 0;
843 static constexpr bool _S_is_partial =
false;
845 static constexpr size_t _S_simd_align = 1;
847 struct _SimdMember {};
848 struct _SimdCastType;
850 static constexpr size_t _S_mask_align = 1;
852 struct _MaskMember {};
853 struct _MaskCastType;
858template <
typename _Tp,
typename _Abi,
typename =
void_t<>>
859 struct _SimdTraits : _InvalidTraits {};
867inline constexpr struct _PrivateInit {} __private_init = {};
869inline constexpr struct _BitsetInit {} __bitset_init = {};
873template <
typename _From,
typename _To,
bool = is_arithmetic_v<_From>,
874 bool = is_arithmetic_v<_To>>
875 struct __is_narrowing_conversion;
879template <
typename _From,
typename _To>
880 struct __is_narrowing_conversion<_From, _To, true, true>
881 :
public __bool_constant<(
882 __digits_v<_From> > __digits_v<_To>
883 || __finite_max_v<_From> > __finite_max_v<_To>
884 || __finite_min_v<_From> < __finite_min_v<_To>
885 || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
887template <
typename _Tp>
888 struct __is_narrowing_conversion<_Tp, bool, true, true>
892 struct __is_narrowing_conversion<bool, bool, true, true>
895template <
typename _Tp>
896 struct __is_narrowing_conversion<_Tp, _Tp, true, true>
899template <
typename _From,
typename _To>
900 struct __is_narrowing_conversion<_From, _To, false, true>
901 :
public negation<is_convertible<_From, _To>> {};
905template <
typename _From,
typename _To,
bool = (sizeof(_From) < sizeof(_To))>
906 struct __converts_to_higher_
integer_rank : public true_type {};
909template <
typename _From,
typename _To>
910 struct __converts_to_higher_integer_rank<_From, _To, false>
911 :
public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
915template <
typename _Tp,
typename _Ap>
916 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
917 __data(
const simd<_Tp, _Ap>& __x);
919template <
typename _Tp,
typename _Ap>
920 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
921 __data(simd<_Tp, _Ap>& __x);
923template <
typename _Tp,
typename _Ap>
924 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
925 __data(
const simd_mask<_Tp, _Ap>& __x);
927template <
typename _Tp,
typename _Ap>
928 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
929 __data(simd_mask<_Tp, _Ap>& __x);
933template <
typename _FromT,
typename _FromA,
typename _ToT,
typename _ToA,
935 struct _SimdConverter;
937template <
typename _Tp,
typename _Ap>
938 struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
940 template <
typename _Up>
941 _GLIBCXX_SIMD_INTRINSIC
const _Up&
942 operator()(
const _Up& __x)
948template <
typename _V>
949 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
950 __to_value_type_or_member_type(
const _V& __x) ->
decltype(__data(__x))
951 {
return __data(__x); }
953template <
typename _V>
954 _GLIBCXX_SIMD_INTRINSIC
constexpr const typename _V::value_type&
955 __to_value_type_or_member_type(
const typename _V::value_type& __x)
960template <
size_t _Size>
961 struct __bool_storage_member_type;
963template <
size_t _Size>
964 using __bool_storage_member_type_t =
965 typename __bool_storage_member_type<_Size>::type;
975template <
typename _Tp,
typename... _Abis>
980template <
typename _Tp,
int _Np>
981 struct __fixed_size_storage;
983template <
typename _Tp,
int _Np>
984 using __fixed_size_storage_t =
typename __fixed_size_storage<_Tp, _Np>::type;
988template <
typename _Tp,
size_t _Size,
typename =
void_t<>>
991template <
typename _Tp>
992 using _SimdWrapper8 = _SimdWrapper<_Tp, 8 /
sizeof(_Tp)>;
993template <
typename _Tp>
994 using _SimdWrapper16 = _SimdWrapper<_Tp, 16 /
sizeof(_Tp)>;
995template <
typename _Tp>
996 using _SimdWrapper32 = _SimdWrapper<_Tp, 32 /
sizeof(_Tp)>;
997template <
typename _Tp>
998 using _SimdWrapper64 = _SimdWrapper<_Tp, 64 /
sizeof(_Tp)>;
1002template <
typename _Tp>
1005template <
typename _Tp,
size_t _Np>
1006 struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> :
true_type {};
1008template <
typename _Tp>
1009 inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
1016 template <
typename _Tp,
typename _Fp>
1018 _S_bit_iteration(_Tp __mask, _Fp&& __f)
1020 static_assert(
sizeof(_ULLong) >=
sizeof(_Tp));
1021 conditional_t<
sizeof(_Tp) <=
sizeof(_UInt), _UInt, _ULLong> __k;
1022 if constexpr (is_convertible_v<_Tp,
decltype(__k)>)
1025 __k = __mask.to_ullong();
1028 __f(std::__countr_zero(__k));
1038template <
typename _Tp =
void>
1040 {
constexpr _Tp operator()(_Tp __a)
const {
return ++__a; } };
1043 struct __increment<void>
1045 template <
typename _Tp>
1047 operator()(_Tp __a)
const
1051template <
typename _Tp =
void>
1053 {
constexpr _Tp operator()(_Tp __a)
const {
return --__a; } };
1056 struct __decrement<void>
1058 template <
typename _Tp>
1060 operator()(_Tp __a)
const
1066template <
typename _From,
typename _To,
1068 __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
1069 using _ValuePreserving = _From;
1071template <
typename _From,
typename _To,
1072 typename _DecayedFrom = __remove_cvref_t<_From>,
1074 is_convertible<_From, _To>,
1076 is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
1077 conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
1078 negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
1079 using _ValuePreservingOrInt = _From;
1083template <
typename _Tp,
size_t _Bytes,
typename =
void_t<>>
1084 struct __intrinsic_type;
1086template <
typename _Tp,
size_t _Size>
1087 using __intrinsic_type_t =
1088 typename __intrinsic_type<_Tp, _Size *
sizeof(_Tp)>::type;
1090template <
typename _Tp>
1091 using __intrinsic_type2_t =
typename __intrinsic_type<_Tp, 2>::type;
1092template <
typename _Tp>
1093 using __intrinsic_type4_t =
typename __intrinsic_type<_Tp, 4>::type;
1094template <
typename _Tp>
1095 using __intrinsic_type8_t =
typename __intrinsic_type<_Tp, 8>::type;
1096template <
typename _Tp>
1097 using __intrinsic_type16_t =
typename __intrinsic_type<_Tp, 16>::type;
1098template <
typename _Tp>
1099 using __intrinsic_type32_t =
typename __intrinsic_type<_Tp, 32>::type;
1100template <
typename _Tp>
1101 using __intrinsic_type64_t =
typename __intrinsic_type<_Tp, 64>::type;
1105template <
size_t _Np,
bool _Sanitized = false>
1108template <
size_t _Np,
bool _Sanitized>
1109 struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> :
true_type {};
1111template <
size_t _Np>
1112 using _SanitizedBitMask = _BitMask<_Np, true>;
1114template <
size_t _Np,
bool _Sanitized>
1117 static_assert(_Np > 0);
1119 static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
1123 sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
1125 static constexpr int _S_array_size = __div_roundup(_NBytes,
sizeof(_Tp));
1127 _Tp _M_bits[_S_array_size];
1129 static constexpr int _S_unused_bits
1130 = _Np == 1 ? 0 : _S_array_size *
sizeof(_Tp) * __CHAR_BIT__ - _Np;
1132 static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1134 constexpr _BitMask() noexcept = default;
1136 constexpr _BitMask(
unsigned long long __x) noexcept
1137 : _M_bits{
static_cast<_Tp
>(__x)} {}
1139 _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1141 constexpr _BitMask(
const _BitMask&)
noexcept =
default;
1143 template <
bool _RhsSanitized,
typename =
enable_if_t<_RhsSanitized ==
false
1144 && _Sanitized ==
true>>
1145 constexpr _BitMask(
const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1146 : _BitMask(__rhs._M_sanitized()) {}
1148 constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1150 static_assert(_S_array_size == 1);
1156 _M_to_bits() const noexcept
1158 static_assert(_S_array_size == 1);
1163 constexpr unsigned long long
1164 to_ullong() const noexcept
1166 static_assert(_S_array_size == 1);
1171 constexpr unsigned long
1172 to_ulong() const noexcept
1174 static_assert(_S_array_size == 1);
1178 constexpr bitset<_Np>
1179 _M_to_bitset() const noexcept
1181 static_assert(_S_array_size == 1);
1185 constexpr decltype(
auto)
1186 _M_sanitized()
const noexcept
1188 if constexpr (_Sanitized)
1190 else if constexpr (_Np == 1)
1191 return _SanitizedBitMask<_Np>(_M_bits[0]);
1194 _SanitizedBitMask<_Np> __r = {};
1195 for (
int __i = 0; __i < _S_array_size; ++__i)
1196 __r._M_bits[__i] = _M_bits[__i];
1197 if constexpr (_S_unused_bits > 0)
1198 __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1203 template <
size_t _Mp,
bool _LSanitized>
1204 constexpr _BitMask<_Np + _Mp, _Sanitized>
1205 _M_prepend(_BitMask<_Mp, _LSanitized> __lsb)
const noexcept
1207 constexpr size_t _RN = _Np + _Mp;
1208 using _Rp = _BitMask<_RN, _Sanitized>;
1209 if constexpr (_Rp::_S_array_size == 1)
1211 _Rp __r{{_M_bits[0]}};
1212 __r._M_bits[0] <<= _Mp;
1213 __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1217 __assert_unreachable<_Rp>();
1223 template <
size_t _DropLsb,
size_t _NewSize = _Np - _DropLsb>
1225 _M_extract() const noexcept
1227 static_assert(_Np > _DropLsb);
1228 static_assert(_DropLsb + _NewSize <=
sizeof(_ULLong) * __CHAR_BIT__,
1229 "not implemented for bitmasks larger than one ullong");
1230 if constexpr (_NewSize == 1)
1232 return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1234 return _BitMask<_NewSize,
1235 ((_NewSize + _DropLsb ==
sizeof(_Tp) * __CHAR_BIT__
1236 && _NewSize + _DropLsb <= _Np)
1237 || ((_Sanitized || _Np ==
sizeof(_Tp) * __CHAR_BIT__)
1238 && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1244 all() const noexcept
1246 if constexpr (_Np == 1)
1248 else if constexpr (!_Sanitized)
1249 return _M_sanitized().all();
1252 constexpr _Tp __allbits = ~_Tp();
1253 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1254 if (_M_bits[__i] != __allbits)
1256 return _M_bits[_S_array_size - 1] == _S_bitmask;
1263 any() const noexcept
1265 if constexpr (_Np == 1)
1267 else if constexpr (!_Sanitized)
1268 return _M_sanitized().any();
1271 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1272 if (_M_bits[__i] != 0)
1274 return _M_bits[_S_array_size - 1] != 0;
1280 none() const noexcept
1282 if constexpr (_Np == 1)
1284 else if constexpr (!_Sanitized)
1285 return _M_sanitized().none();
1288 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1289 if (_M_bits[__i] != 0)
1291 return _M_bits[_S_array_size - 1] == 0;
1298 count() const noexcept
1300 if constexpr (_Np == 1)
1302 else if constexpr (!_Sanitized)
1303 return _M_sanitized().none();
1306 int __result = __builtin_popcountll(_M_bits[0]);
1307 for (
int __i = 1; __i < _S_array_size; ++__i)
1308 __result += __builtin_popcountll(_M_bits[__i]);
1315 operator[](
size_t __i)
const noexcept
1317 if constexpr (_Np == 1)
1319 else if constexpr (_S_array_size == 1)
1320 return (_M_bits[0] >> __i) & 1;
1323 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1324 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1325 return (_M_bits[__j] >> __shift) & 1;
1329 template <
size_t __i>
1331 operator[](_SizeConstant<__i>)
const noexcept
1333 static_assert(__i < _Np);
1334 constexpr size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1335 constexpr size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1336 return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1341 set(
size_t __i,
bool __x)
noexcept
1343 if constexpr (_Np == 1)
1345 else if constexpr (_S_array_size == 1)
1347 _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1348 _M_bits[0] |= _Tp(_Tp(__x) << __i);
1352 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1353 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1354 _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1355 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1359 template <
size_t __i>
1361 set(_SizeConstant<__i>,
bool __x)
noexcept
1363 static_assert(__i < _Np);
1364 if constexpr (_Np == 1)
1368 constexpr size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1369 constexpr size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1370 constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1371 _M_bits[__j] &= __mask;
1372 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1378 operator~() const noexcept
1380 if constexpr (_Np == 1)
1384 _BitMask __result{};
1385 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1386 __result._M_bits[__i] = ~_M_bits[__i];
1387 if constexpr (_Sanitized)
1388 __result._M_bits[_S_array_size - 1]
1389 = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1391 __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1397 operator^=(
const _BitMask& __b) &
noexcept
1399 __execute_n_times<_S_array_size>(
1400 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] ^= __b._M_bits[__i]; });
1405 operator|=(
const _BitMask& __b) &
noexcept
1407 __execute_n_times<_S_array_size>(
1408 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] |= __b._M_bits[__i]; });
1413 operator&=(
const _BitMask& __b) &
noexcept
1415 __execute_n_times<_S_array_size>(
1416 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] &= __b._M_bits[__i]; });
1420 friend constexpr _BitMask
1421 operator^(
const _BitMask& __a,
const _BitMask& __b)
noexcept
1428 friend constexpr _BitMask
1429 operator|(
const _BitMask& __a,
const _BitMask& __b)
noexcept
1436 friend constexpr _BitMask
1437 operator&(
const _BitMask& __a,
const _BitMask& __b)
noexcept
1444 _GLIBCXX_SIMD_INTRINSIC
1446 _M_is_constprop()
const
1448 if constexpr (_S_array_size == 0)
1449 return __builtin_constant_p(_M_bits[0]);
1452 for (
int __i = 0; __i < _S_array_size; ++__i)
1453 if (!__builtin_constant_p(_M_bits[__i]))
1464template <
typename _Tp =
void>
1465 static inline constexpr int __min_vector_size = 2 *
sizeof(_Tp);
1467#if _GLIBCXX_SIMD_HAVE_NEON
1469 inline constexpr int __min_vector_size<void> = 8;
1472 inline constexpr int __min_vector_size<void> = 16;
1477template <
typename _Tp,
size_t _Np,
typename =
void>
1478 struct __vector_type_n {};
1481template <
typename _Tp>
1482 struct __vector_type_n<_Tp, 0, void> {};
1485template <
typename _Tp>
1486 struct __vector_type_n<_Tp, 1,
enable_if_t<__is_vectorizable_v<_Tp>>>
1487 {
using type = _Tp; };
1490template <
typename _Tp,
size_t _Np>
1491 struct __vector_type_n<_Tp, _Np,
enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1493 static constexpr size_t _S_Np2 = std::__bit_ceil(_Np *
sizeof(_Tp));
1495 static constexpr size_t _S_Bytes =
1501 _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1504 using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1507template <
typename _Tp,
size_t _Bytes,
size_t = _Bytes % sizeof(_Tp)>
1508 struct __vector_type;
1510template <
typename _Tp,
size_t _Bytes>
1511 struct __vector_type<_Tp, _Bytes, 0>
1512 : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1514template <
typename _Tp,
size_t _Size>
1515 using __vector_type_t =
typename __vector_type_n<_Tp, _Size>::type;
1517template <
typename _Tp>
1518 using __vector_type2_t =
typename __vector_type<_Tp, 2>::type;
1519template <
typename _Tp>
1520 using __vector_type4_t =
typename __vector_type<_Tp, 4>::type;
1521template <
typename _Tp>
1522 using __vector_type8_t =
typename __vector_type<_Tp, 8>::type;
1523template <
typename _Tp>
1524 using __vector_type16_t =
typename __vector_type<_Tp, 16>::type;
1525template <
typename _Tp>
1526 using __vector_type32_t =
typename __vector_type<_Tp, 32>::type;
1527template <
typename _Tp>
1528 using __vector_type64_t =
typename __vector_type<_Tp, 64>::type;
1532template <
typename _Tp,
typename =
void_t<>>
1535template <
typename _Tp>
1536 struct __is_vector_type<
1537 _Tp,
void_t<typename __vector_type<
1539 : is_same<_Tp, typename __vector_type<
1540 remove_reference_t<decltype(declval<_Tp>()[0])>,
1541 sizeof(_Tp)>::type> {};
1543template <
typename _Tp>
1544 inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1548#if _GLIBCXX_SIMD_HAVE_SSE_ABI
1549template <
typename _Tp>
1550 using __is_intrinsic_type = __is_vector_type<_Tp>;
1552template <
typename _Tp,
typename =
void_t<>>
1555template <
typename _Tp>
1556 struct __is_intrinsic_type<
1557 _Tp,
void_t<typename __intrinsic_type<
1559 : is_same<_Tp, typename __intrinsic_type<
1560 remove_reference_t<decltype(declval<_Tp>()[0])>,
1561 sizeof(_Tp)>::type> {};
1564template <
typename _Tp>
1565 inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1569template <
typename _Tp,
typename =
void_t<>>
1570 struct _VectorTraitsImpl;
1572template <
typename _Tp>
1573 struct _VectorTraitsImpl<_Tp,
enable_if_t<__is_vector_type_v<_Tp>
1574 || __is_intrinsic_type_v<_Tp>>>
1577 using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1578 static constexpr int _S_full_size =
sizeof(_Tp) /
sizeof(value_type);
1579 using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1580 template <
typename _Up,
int _W = _S_full_size>
1581 static constexpr bool _S_is
1582 = is_same_v<value_type, _Up> && _W == _S_full_size;
1585template <
typename _Tp,
size_t _Np>
1586 struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1587 void_t<__vector_type_t<_Tp, _Np>>>
1589 using type = __vector_type_t<_Tp, _Np>;
1590 using value_type = _Tp;
1591 static constexpr int _S_full_size =
sizeof(type) /
sizeof(value_type);
1592 using _Wrapper = _SimdWrapper<_Tp, _Np>;
1593 static constexpr bool _S_is_partial = (_Np == _S_full_size);
1594 static constexpr int _S_partial_width = _Np;
1595 template <
typename _Up,
int _W = _S_full_size>
1596 static constexpr bool _S_is
1597 = is_same_v<value_type, _Up>&& _W == _S_full_size;
1600template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1601 using _VectorTraits = _VectorTraitsImpl<_Tp>;
1605template <
typename _V>
1606 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
1609 if constexpr (__is_vector_type_v<_V>)
1611 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1613 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
1615 static_assert(is_simd<_V>::value);
1616 static_assert(_V::abi_type::template __traits<
1617 typename _V::value_type>::_SimdMember::_S_tuple_size == 1);
1618 return __as_vector(__data(__x).first);
1620 else if constexpr (_V::size() > 1)
1621 return __data(__x)._M_data;
1624 static_assert(is_simd<_V>::value);
1625 using _Tp =
typename _V::value_type;
1627 constexpr auto __bytes =
sizeof(_Tp) == 8 ? 16 :
sizeof(_Tp);
1628 using _RV [[__gnu__::__vector_size__(__bytes)]] = _Tp;
1630 using _RV [[__gnu__::__vector_size__(
sizeof(_Tp))]] = _Tp;
1632 return _RV{__data(__x)};
1635 else if constexpr (__is_vectorizable_v<_V>)
1636 return __vector_type_t<_V, 2>{__x};
1643template <
size_t _Np = 0,
typename _V>
1644 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
1645 __as_wrapper(_V __x)
1647 if constexpr (__is_vector_type_v<_V>)
1648 return _SimdWrapper<
typename _VectorTraits<_V>::value_type,
1649 (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1650 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1652 static_assert(_V::size() == _Np);
1657 static_assert(_V::_S_size == _Np);
1664template <
typename _To,
typename _From>
1665 _GLIBCXX_SIMD_INTRINSIC
constexpr _To
1666 __intrin_bitcast(_From __v)
1668 static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1669 && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1670 if constexpr (
sizeof(_To) ==
sizeof(_From))
1671 return reinterpret_cast<_To
>(__v);
1672 else if constexpr (
sizeof(_From) >
sizeof(_To))
1673 if constexpr (
sizeof(_To) >= 16)
1674 return reinterpret_cast<const __may_alias<_To>&
>(__v);
1678 __builtin_memcpy(&__r, &__v,
sizeof(_To));
1681#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1682 else if constexpr (__have_avx &&
sizeof(_From) == 16 &&
sizeof(_To) == 32)
1683 return reinterpret_cast<_To
>(__builtin_ia32_ps256_ps(
1684 reinterpret_cast<__vector_type_t<float, 4>
>(__v)));
1685 else if constexpr (__have_avx512f &&
sizeof(_From) == 16
1686 &&
sizeof(_To) == 64)
1687 return reinterpret_cast<_To
>(__builtin_ia32_ps512_ps(
1688 reinterpret_cast<__vector_type_t<float, 4>
>(__v)));
1689 else if constexpr (__have_avx512f &&
sizeof(_From) == 32
1690 &&
sizeof(_To) == 64)
1691 return reinterpret_cast<_To
>(__builtin_ia32_ps512_256ps(
1692 reinterpret_cast<__vector_type_t<float, 8>
>(__v)));
1694 else if constexpr (
sizeof(__v) <= 8)
1695 return reinterpret_cast<_To
>(
1696 __vector_type_t<__int_for_sizeof_t<_From>,
sizeof(_To) /
sizeof(_From)>{
1697 reinterpret_cast<__int_for_sizeof_t<_From>
>(__v)});
1700 static_assert(
sizeof(_To) >
sizeof(_From));
1702 __builtin_memcpy(&__r, &__v,
sizeof(_From));
1709template <
typename _To,
size_t _NN = 0,
typename _From,
1710 typename _FromVT = _VectorTraits<_From>,
1711 size_t _Np = _NN == 0 ?
sizeof(_From) /
sizeof(_To) : _NN>
1712 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_To, _Np>
1713 __vector_bitcast(_From __x)
1715 using _R = __vector_type_t<_To, _Np>;
1716 return __intrin_bitcast<_R>(__x);
1719template <
typename _To,
size_t _NN = 0,
typename _Tp,
size_t _Nx,
1721 = _NN == 0 ?
sizeof(_SimdWrapper<_Tp, _Nx>) /
sizeof(_To) : _NN>
1722 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_To, _Np>
1723 __vector_bitcast(
const _SimdWrapper<_Tp, _Nx>& __x)
1725 static_assert(_Np > 1);
1726 return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1731#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1732template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1733 _To __convert_x86(_Tp);
1735template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1736 _To __convert_x86(_Tp, _Tp);
1738template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1739 _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1741template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1742 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1744template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1745 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1746 _Tp, _Tp, _Tp, _Tp);
1751template <
typename _To,
typename _From>
1752 _GLIBCXX_SIMD_INTRINSIC
constexpr _To
1753 __bit_cast(
const _From __x)
1755#if __has_builtin(__builtin_bit_cast)
1756 return __builtin_bit_cast(_To, __x);
1758 static_assert(
sizeof(_To) ==
sizeof(_From));
1759 constexpr bool __to_is_vectorizable
1760 = is_arithmetic_v<_To> || is_enum_v<_To>;
1761 constexpr bool __from_is_vectorizable
1762 = is_arithmetic_v<_From> || is_enum_v<_From>;
1763 if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1764 return reinterpret_cast<_To
>(__x);
1765 else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1767 using _FV [[__gnu__::__vector_size__(
sizeof(_From))]] = _From;
1768 return reinterpret_cast<_To
>(_FV{__x});
1770 else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1772 using _TV [[__gnu__::__vector_size__(
sizeof(_To))]] = _To;
1773 using _FV [[__gnu__::__vector_size__(
sizeof(_From))]] = _From;
1774 return reinterpret_cast<_TV
>(_FV{__x})[0];
1776 else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1778 using _TV [[__gnu__::__vector_size__(
sizeof(_To))]] = _To;
1779 return reinterpret_cast<_TV
>(__x)[0];
1784 __builtin_memcpy(
reinterpret_cast<char*
>(&__r),
1785 reinterpret_cast<const char*
>(&__x),
sizeof(_To));
1793template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
1794 typename _R = __
intrinsic_type_t<
typename _TVT::value_type, _TVT::_S_full_size>>
1795 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
1796 __to_intrin(_Tp __x)
1798 static_assert(
sizeof(__x) <=
sizeof(_R),
1799 "__to_intrin may never drop values off the end");
1800 if constexpr (
sizeof(__x) ==
sizeof(_R))
1801 return reinterpret_cast<_R
>(__as_vector(__x));
1804 using _Up = __int_for_sizeof_t<_Tp>;
1805 return reinterpret_cast<_R
>(
1806 __vector_type_t<_Up,
sizeof(_R) /
sizeof(_Up)>{__bit_cast<_Up>(__x)});
1812template <
typename _Tp,
typename... _Args>
1813 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp,
sizeof...(_Args)>
1814 __make_vector(
const _Args&... __args)
1815 {
return __vector_type_t<_Tp,
sizeof...(_Args)>{
static_cast<_Tp
>(__args)...}; }
1819template <
size_t _Np,
typename _Tp,
size_t... _I>
1820 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1821 __vector_broadcast_impl(_Tp __x, index_sequence<_I...>)
1822 {
return __vector_type_t<_Tp, _Np>{((void)_I, __x)...}; }
1824template <
size_t _Np,
typename _Tp>
1825 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1826 __vector_broadcast(_Tp __x)
1827 {
return __vector_broadcast_impl<_Np, _Tp>(__x, make_index_sequence<_Np>()); }
1831 template <
typename _Tp,
size_t _Np,
typename _Gp,
size_t... _I>
1832 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1833 __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1834 {
return __vector_type_t<_Tp, _Np>{
static_cast<_Tp
>(__gen(_SizeConstant<_I>()))...}; }
1836template <
typename _V,
typename _VVT = _VectorTraits<_V>,
typename _Gp>
1837 _GLIBCXX_SIMD_INTRINSIC
constexpr _V
1838 __generate_vector(_Gp&& __gen)
1840 if constexpr (__is_vector_type_v<_V>)
1841 return __generate_vector_impl<
typename _VVT::value_type,
1842 _VVT::_S_full_size>(
1843 static_cast<_Gp&&
>(__gen), make_index_sequence<_VVT::_S_full_size>());
1845 return __generate_vector_impl<
typename _VVT::value_type,
1846 _VVT::_S_partial_width>(
1847 static_cast<_Gp&&
>(__gen),
1848 make_index_sequence<_VVT::_S_partial_width>());
1851template <
typename _Tp,
size_t _Np,
typename _Gp>
1852 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1853 __generate_vector(_Gp&& __gen)
1855 return __generate_vector_impl<_Tp, _Np>(
static_cast<_Gp&&
>(__gen),
1856 make_index_sequence<_Np>());
1861template <
typename _TW>
1862 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1863 __xor(_TW __a, _TW __b)
noexcept
1865 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1867 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1868 _VectorTraitsImpl<_TW>>::value_type;
1869 if constexpr (is_floating_point_v<_Tp>)
1871 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1872 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1873 ^ __vector_bitcast<_Ip>(__b));
1875 else if constexpr (__is_vector_type_v<_TW>)
1878 return __a._M_data ^ __b._M_data;
1886template <
typename _TW>
1887 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1888 __or(_TW __a, _TW __b)
noexcept
1890 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1892 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1893 _VectorTraitsImpl<_TW>>::value_type;
1894 if constexpr (is_floating_point_v<_Tp>)
1896 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1897 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1898 | __vector_bitcast<_Ip>(__b));
1900 else if constexpr (__is_vector_type_v<_TW>)
1903 return __a._M_data | __b._M_data;
1911template <
typename _TW>
1912 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1913 __and(_TW __a, _TW __b)
noexcept
1915 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1917 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1918 _VectorTraitsImpl<_TW>>::value_type;
1919 if constexpr (is_floating_point_v<_Tp>)
1921 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1922 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1923 & __vector_bitcast<_Ip>(__b));
1925 else if constexpr (__is_vector_type_v<_TW>)
1928 return __a._M_data & __b._M_data;
1936#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1937static constexpr struct
1939 _GLIBCXX_SIMD_INTRINSIC __v4sf
1940 operator()(__v4sf __a, __v4sf __b)
const noexcept
1941 {
return __builtin_ia32_andnps(__a, __b); }
1943 _GLIBCXX_SIMD_INTRINSIC __v2df
1944 operator()(__v2df __a, __v2df __b)
const noexcept
1945 {
return __builtin_ia32_andnpd(__a, __b); }
1947 _GLIBCXX_SIMD_INTRINSIC __v2di
1948 operator()(__v2di __a, __v2di __b)
const noexcept
1949 {
return __builtin_ia32_pandn128(__a, __b); }
1951 _GLIBCXX_SIMD_INTRINSIC __v8sf
1952 operator()(__v8sf __a, __v8sf __b)
const noexcept
1953 {
return __builtin_ia32_andnps256(__a, __b); }
1955 _GLIBCXX_SIMD_INTRINSIC __v4df
1956 operator()(__v4df __a, __v4df __b)
const noexcept
1957 {
return __builtin_ia32_andnpd256(__a, __b); }
1959 _GLIBCXX_SIMD_INTRINSIC __v4di
1960 operator()(__v4di __a, __v4di __b)
const noexcept
1962 if constexpr (__have_avx2)
1963 return __builtin_ia32_andnotsi256(__a, __b);
1965 return reinterpret_cast<__v4di
>(
1966 __builtin_ia32_andnpd256(
reinterpret_cast<__v4df
>(__a),
1967 reinterpret_cast<__v4df
>(__b)));
1970 _GLIBCXX_SIMD_INTRINSIC __v16sf
1971 operator()(__v16sf __a, __v16sf __b)
const noexcept
1973 if constexpr (__have_avx512dq)
1974 return _mm512_andnot_ps(__a, __b);
1976 return reinterpret_cast<__v16sf
>(
1977 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
1978 reinterpret_cast<__v8di
>(__b)));
1981 _GLIBCXX_SIMD_INTRINSIC __v8df
1982 operator()(__v8df __a, __v8df __b)
const noexcept
1984 if constexpr (__have_avx512dq)
1985 return _mm512_andnot_pd(__a, __b);
1987 return reinterpret_cast<__v8df
>(
1988 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
1989 reinterpret_cast<__v8di
>(__b)));
1992 _GLIBCXX_SIMD_INTRINSIC __v8di
1993 operator()(__v8di __a, __v8di __b)
const noexcept
1994 {
return _mm512_andnot_si512(__a, __b); }
1998template <
typename _TW>
1999 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
2000 __andnot(_TW __a, _TW __b)
noexcept
2002 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
2004 using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
2005 _VectorTraitsImpl<_TW>>;
2006 using _Tp =
typename _TVT::value_type;
2007#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
2008 if constexpr (
sizeof(_TW) >= 16)
2010 const auto __ai = __to_intrin(__a);
2011 const auto __bi = __to_intrin(__b);
2012 if (!__builtin_is_constant_evaluated()
2013 && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
2015 const auto __r = _S_x86_andnot(__ai, __bi);
2016 if constexpr (is_convertible_v<
decltype(__r), _TW>)
2019 return reinterpret_cast<typename _TVT::type
>(__r);
2023 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
2024 return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
2025 & __vector_bitcast<_Ip>(__b));
2033template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
2034 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
2035 __not(_Tp __a)
noexcept
2037 if constexpr (is_floating_point_v<typename _TVT::value_type>)
2038 return reinterpret_cast<typename _TVT::type
>(
2039 ~__vector_bitcast<unsigned>(__a));
2046template <
typename _T0,
typename _T1,
typename _Fun,
size_t... _Is>
2047 _GLIBCXX_SIMD_INTRINSIC
constexpr
2048 __vector_type_t<remove_reference_t<decltype(declval<_T0>()[0])>,
sizeof...(_Is)>
2049 __vec_shuffle(_T0 __x, _T1 __y, index_sequence<_Is...> __seq, _Fun __idx_perm)
2051 constexpr int _N0 =
sizeof(__x) /
sizeof(__x[0]);
2052 constexpr int _N1 =
sizeof(__y) /
sizeof(__y[0]);
2053 using _Tp = remove_reference_t<decltype(declval<_T0>()[0])>;
2054 using _RV [[maybe_unused]] = __vector_type_t<_Tp,
sizeof...(_Is)>;
2055#if __has_builtin(__builtin_shufflevector)
2058 if constexpr (
sizeof(__x) >
sizeof(__y) and _N1 == 1)
2059 return __vec_shuffle(__x, _T0{__y[0]}, __seq, __idx_perm);
2060 else if constexpr (
sizeof(__x) >
sizeof(__y))
2061 return __vec_shuffle(__x, __intrin_bitcast<_T0>(__y), __seq, __idx_perm);
2062 else if constexpr (
sizeof(__x) <
sizeof(__y) and _N0 == 1)
2063 return __vec_shuffle(_T1{__x[0]}, __y, __seq, [=](
int __i) {
2064 __i = __idx_perm(__i);
2065 return __i < _N0 ? __i : __i - _N0 + _N1;
2067 else if constexpr (
sizeof(__x) <
sizeof(__y))
2068 return __vec_shuffle(__intrin_bitcast<_T1>(__x), __y, __seq, [=](
int __i) {
2069 __i = __idx_perm(__i);
2070 return __i < _N0 ? __i : __i - _N0 + _N1;
2075 const auto __r = __builtin_shufflevector(__x, __y, [=] {
2076 constexpr int __j = __idx_perm(_Is);
2077 static_assert(__j < _N0 + _N1);
2081 if constexpr (
sizeof(__r) ==
sizeof(_RV))
2084 return _RV {__r[_Is]...};
2092 constexpr int __j = __idx_perm(_Is);
2093 static_assert(__j < _N0 + _N1);
2094 if constexpr (__j < 0)
2096 else if constexpr (__j < _N0)
2099 return __y[__j - _N0];
2105template <
typename _T0,
typename _Fun,
typename _Seq>
2106 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2107 __vec_shuffle(_T0 __x, _Seq __seq, _Fun __idx_perm)
2108 {
return __vec_shuffle(__x, _T0(), __seq, __idx_perm); }
2112template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
2113 typename _R = __vector_type_t<
typename _TVT::value_type, _TVT::_S_full_size * 2>>
2115 __concat(_Tp a_, _Tp b_)
2117#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2119 = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
2121 long long,
typename _TVT::value_type>>;
2122 constexpr int input_width =
sizeof(_Tp) /
sizeof(_W);
2123 const auto __a = __vector_bitcast<_W>(a_);
2124 const auto __b = __vector_bitcast<_W>(b_);
2125 using _Up = __vector_type_t<_W,
sizeof(_R) /
sizeof(_W)>;
2127 constexpr int input_width = _TVT::_S_full_size;
2128 const _Tp& __a = a_;
2129 const _Tp& __b = b_;
2132 if constexpr (input_width == 2)
2133 return reinterpret_cast<_R
>(_Up{__a[0], __a[1], __b[0], __b[1]});
2134 else if constexpr (input_width == 4)
2135 return reinterpret_cast<_R
>(
2136 _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
2137 else if constexpr (input_width == 8)
2138 return reinterpret_cast<_R
>(
2139 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
2140 __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
2141 else if constexpr (input_width == 16)
2142 return reinterpret_cast<_R
>(
2143 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2144 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2145 __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
2146 __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
2147 __b[12], __b[13], __b[14], __b[15]});
2148 else if constexpr (input_width == 32)
2149 return reinterpret_cast<_R
>(
2150 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2151 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2152 __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
2153 __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
2154 __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
2155 __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
2156 __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
2157 __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
2158 __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
2164template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
2165 struct _ZeroExtendProxy
2167 using value_type =
typename _TVT::value_type;
2168 static constexpr size_t _Np = _TVT::_S_full_size;
2171 template <
typename _To,
typename _ToVT = _VectorTraits<_To>,
2173 = enable_if_t<is_same_v<
typename _ToVT::value_type, value_type>>>
2174 _GLIBCXX_SIMD_INTRINSIC
operator _To()
const
2176 constexpr size_t _ToN = _ToVT::_S_full_size;
2177 if constexpr (_ToN == _Np)
2179 else if constexpr (_ToN == 2 * _Np)
2181#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2182 if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
2183 return __vector_bitcast<value_type>(
2184 _mm256_insertf128_ps(__m256(), __x, 0));
2185 else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
2186 return __vector_bitcast<value_type>(
2187 _mm256_insertf128_pd(__m256d(), __x, 0));
2188 else if constexpr (__have_avx2 && _Np *
sizeof(value_type) == 16)
2189 return __vector_bitcast<value_type>(
2190 _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
2191 else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
2193 if constexpr (__have_avx512dq)
2194 return __vector_bitcast<value_type>(
2195 _mm512_insertf32x8(__m512(), __x, 0));
2197 return reinterpret_cast<__m512
>(
2198 _mm512_insertf64x4(__m512d(),
2199 reinterpret_cast<__m256d
>(__x), 0));
2201 else if constexpr (__have_avx512f
2202 && _TVT::template _S_is<double, 4>)
2203 return __vector_bitcast<value_type>(
2204 _mm512_insertf64x4(__m512d(), __x, 0));
2205 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 32)
2206 return __vector_bitcast<value_type>(
2207 _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
2209 return __concat(__x, _Tp());
2211 else if constexpr (_ToN == 4 * _Np)
2213#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2214 if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2216 return __vector_bitcast<value_type>(
2217 _mm512_insertf64x2(__m512d(), __x, 0));
2219 else if constexpr (__have_avx512f
2220 && is_floating_point_v<value_type>)
2222 return __vector_bitcast<value_type>(
2223 _mm512_insertf32x4(__m512(),
reinterpret_cast<__m128
>(__x),
2226 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 16)
2228 return __vector_bitcast<value_type>(
2229 _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2232 return __concat(__concat(__x, _Tp()),
2233 __vector_type_t<value_type, _Np * 2>());
2235 else if constexpr (_ToN == 8 * _Np)
2236 return __concat(
operator __vector_type_t<value_type, _Np * 4>(),
2237 __vector_type_t<value_type, _Np * 4>());
2238 else if constexpr (_ToN == 16 * _Np)
2239 return __concat(
operator __vector_type_t<value_type, _Np * 8>(),
2240 __vector_type_t<value_type, _Np * 8>());
2242 __assert_unreachable<_Tp>();
2246template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
2247 _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2248 __zero_extend(_Tp __x)
2253template <
int _Offset,
2256 typename _TVT = _VectorTraits<_Tp>,
2257 typename _R = __vector_type_t<
typename _TVT::value_type, _TVT::_S_full_size / _SplitBy>>
2258 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2261 using value_type =
typename _TVT::value_type;
2262#if _GLIBCXX_SIMD_X86INTRIN
2263 if constexpr (
sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2265 if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2266 return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2267 else if constexpr (is_floating_point_v<value_type>)
2268 return __vector_bitcast<value_type>(
2269 _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2271 return reinterpret_cast<_R
>(
2272 _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2278#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2280 is_floating_point_v<value_type>, double,
2282 static_assert(
sizeof(_R) %
sizeof(_W) == 0);
2283 constexpr int __return_width =
sizeof(_R) /
sizeof(_W);
2284 using _Up = __vector_type_t<_W, __return_width>;
2285 const auto __x = __vector_bitcast<_W>(__in);
2287 constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2289 const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2292 constexpr int _O = _Offset * __return_width;
2293 return __call_with_subscripts<__return_width, _O>(
2294 __x, [](
auto... __entries) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2295 return reinterpret_cast<_R
>(_Up{__entries...});
2302template <
typename _Tp,
2303 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2304 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2308 __builtin_memcpy(&__r, &__x, 8);
2312template <
typename _Tp,
2313 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2314 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2317 static_assert(
sizeof(_Tp) == 16,
"use __hi64z if you meant it");
2319 __builtin_memcpy(&__r,
reinterpret_cast<const char*
>(&__x) + 8, 8);
2323template <
typename _Tp,
2324 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2325 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2326 __hi64z([[maybe_unused]] _Tp __x)
2329 if constexpr (
sizeof(_Tp) == 16)
2330 __builtin_memcpy(&__r,
reinterpret_cast<const char*
>(&__x) + 8, 8);
2336template <
typename _Tp>
2337 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2339 {
return __extract<0,
sizeof(_Tp) / 16>(__x); }
2341template <
typename _Tp>
2342 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2345 static_assert(
sizeof(__x) == 32);
2346 return __extract<1, 2>(__x);
2351template <
typename _Tp>
2352 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2355 static_assert(
sizeof(__x) == 64);
2356 return __extract<0, 2>(__x);
2359template <
typename _Tp>
2360 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2363 static_assert(
sizeof(__x) == 64);
2364 return __extract<1, 2>(__x);
2369template <
typename _Tp>
2372 static_assert(__is_vector_type_v<_Tp>);
2376 template <
typename _Up,
typename _UVT = _VectorTraits<_Up>>
2377 _GLIBCXX_SIMD_INTRINSIC
constexpr operator _Up()
const
2378 {
return __intrin_bitcast<typename _UVT::type>(__x); }
2381template <
typename _Tp>
2382 _GLIBCXX_SIMD_INTRINSIC
constexpr _AutoCast<_Tp>
2383 __auto_bitcast(
const _Tp& __x)
2386template <
typename _Tp,
size_t _Np>
2387 _GLIBCXX_SIMD_INTRINSIC
constexpr
2388 _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2389 __auto_bitcast(
const _SimdWrapper<_Tp, _Np>& __x)
2390 {
return {__x._M_data}; }
2395#if _GLIBCXX_SIMD_HAVE_SSE_ABI
2397#if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2398template <
size_t _Size>
2399 struct __bool_storage_member_type
2401 static_assert((_Size & (_Size - 1)) != 0,
2402 "This trait may only be used for non-power-of-2 sizes. "
2403 "Power-of-2 sizes must be specialized.");
2405 typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2409 struct __bool_storage_member_type<1> {
using type = bool; };
2412 struct __bool_storage_member_type<2> {
using type = __mmask8; };
2415 struct __bool_storage_member_type<4> {
using type = __mmask8; };
2418 struct __bool_storage_member_type<8> {
using type = __mmask8; };
2421 struct __bool_storage_member_type<16> {
using type = __mmask16; };
2424 struct __bool_storage_member_type<32> {
using type = __mmask32; };
2427 struct __bool_storage_member_type<64> {
using type = __mmask64; };
2433#if _GLIBCXX_SIMD_HAVE_SSE
2434template <
typename _Tp,
size_t _Bytes>
2435 struct __intrinsic_type<_Tp, _Bytes,
enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2437 static_assert(!is_same_v<_Tp, long double>,
2438 "no __intrinsic_type support for long double on x86");
2440 static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16 : _Bytes <= 32 ? 32 : 64;
2442 using type [[__gnu__::__vector_size__(_S_VBytes)]]
2443 = conditional_t<is_integral_v<_Tp>,
long long int, _Tp>;
2450#if _GLIBCXX_SIMD_HAVE_NEON
2452 struct __intrinsic_type<float, 8, void>
2453 {
using type = float32x2_t; };
2456 struct __intrinsic_type<float, 16, void>
2457 {
using type = float32x4_t; };
2460 struct __intrinsic_type<double, 8, void>
2462#if _GLIBCXX_SIMD_HAVE_NEON_A64
2463 using type = float64x1_t;
2468 struct __intrinsic_type<double, 16, void>
2470#if _GLIBCXX_SIMD_HAVE_NEON_A64
2471 using type = float64x2_t;
2475#define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2477 struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2478 _Np * _Bits / 8, void> \
2479 { using type = int##_Bits##x##_Np##_t; }; \
2481 struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2482 _Np * _Bits / 8, void> \
2483 { using type = uint##_Bits##x##_Np##_t; }
2484_GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2485_GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2486_GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2487_GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2488_GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2489_GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2490_GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2491_GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2492#undef _GLIBCXX_SIMD_ARM_INTRIN
2494template <
typename _Tp,
size_t _Bytes>
2495 struct __intrinsic_type<_Tp, _Bytes,
enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2497 static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2499 using _Ip = __int_for_sizeof_t<_Tp>;
2502 is_floating_point_v<_Tp>, _Tp,
2503 conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2505 static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2506 "should use explicit specialization above");
2508 using type =
typename __intrinsic_type<_Up, _SVecBytes>::type;
2515template <
typename _Tp>
2516 struct __intrinsic_type_impl;
2518#define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2520 struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2521_GLIBCXX_SIMD_PPC_INTRIN(
float);
2523_GLIBCXX_SIMD_PPC_INTRIN(
double);
2525_GLIBCXX_SIMD_PPC_INTRIN(
signed char);
2526_GLIBCXX_SIMD_PPC_INTRIN(
unsigned char);
2527_GLIBCXX_SIMD_PPC_INTRIN(
signed short);
2528_GLIBCXX_SIMD_PPC_INTRIN(
unsigned short);
2529_GLIBCXX_SIMD_PPC_INTRIN(
signed int);
2530_GLIBCXX_SIMD_PPC_INTRIN(
unsigned int);
2531#if defined __VSX__ || __SIZEOF_LONG__ == 4
2532_GLIBCXX_SIMD_PPC_INTRIN(
signed long);
2533_GLIBCXX_SIMD_PPC_INTRIN(
unsigned long);
2536_GLIBCXX_SIMD_PPC_INTRIN(
signed long long);
2537_GLIBCXX_SIMD_PPC_INTRIN(
unsigned long long);
2539#undef _GLIBCXX_SIMD_PPC_INTRIN
2541template <
typename _Tp,
size_t _Bytes>
2542 struct __intrinsic_type<_Tp, _Bytes,
enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2544 static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2547 static_assert(!(_S_is_ldouble &&
sizeof(
long double) >
sizeof(
double)),
2548 "no __intrinsic_type support for 128-bit floating point on PowerPC");
2551 static_assert(!(is_same_v<_Tp, double>
2552 || (_S_is_ldouble &&
sizeof(
long double) ==
sizeof(
double))),
2553 "no __intrinsic_type support for 64-bit floating point on PowerPC w/o VSX");
2556 static constexpr auto __element_type()
2558 if constexpr (is_floating_point_v<_Tp>)
2560 if constexpr (_S_is_ldouble)
2565 else if constexpr (is_signed_v<_Tp>)
2567 if constexpr (
sizeof(_Tp) ==
sizeof(_SChar))
2569 else if constexpr (
sizeof(_Tp) ==
sizeof(
short))
2571 else if constexpr (
sizeof(_Tp) ==
sizeof(
int))
2573 else if constexpr (
sizeof(_Tp) ==
sizeof(_LLong))
2578 if constexpr (
sizeof(_Tp) ==
sizeof(_UChar))
2580 else if constexpr (
sizeof(_Tp) ==
sizeof(_UShort))
2582 else if constexpr (
sizeof(_Tp) ==
sizeof(_UInt))
2584 else if constexpr (
sizeof(_Tp) ==
sizeof(_ULLong))
2589 using type =
typename __intrinsic_type_impl<
decltype(__element_type())>::type;
2595template <
size_t _W
idth>
2596 struct _SimdWrapper<bool, _Width,
2597 void_t<typename __bool_storage_member_type<_Width>::type>>
2599 using _BuiltinType =
typename __bool_storage_member_type<_Width>::type;
2600 using value_type = bool;
2602 static constexpr size_t _S_full_size =
sizeof(_BuiltinType) * __CHAR_BIT__;
2604 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper<bool, _S_full_size>
2605 __as_full_vector()
const
2608 _GLIBCXX_SIMD_INTRINSIC
constexpr
2609 _SimdWrapper() =
default;
2611 _GLIBCXX_SIMD_INTRINSIC
constexpr
2612 _SimdWrapper(_BuiltinType __k) : _M_data(__k) {};
2614 _GLIBCXX_SIMD_INTRINSIC
2615 operator const _BuiltinType&()
const
2618 _GLIBCXX_SIMD_INTRINSIC
2619 operator _BuiltinType&()
2622 _GLIBCXX_SIMD_INTRINSIC _BuiltinType
2626 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type
2627 operator[](
size_t __i)
const
2628 {
return _M_data & (_BuiltinType(1) << __i); }
2630 template <
size_t __i>
2631 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type
2632 operator[](_SizeConstant<__i>)
const
2633 {
return _M_data & (_BuiltinType(1) << __i); }
2635 _GLIBCXX_SIMD_INTRINSIC
constexpr void
2636 _M_set(
size_t __i, value_type __x)
2639 _M_data |= (_BuiltinType(1) << __i);
2641 _M_data &= ~(_BuiltinType(1) << __i);
2644 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2645 _M_is_constprop()
const
2646 {
return __builtin_constant_p(_M_data); }
2648 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2649 _M_is_constprop_none_of()
const
2651 if (__builtin_constant_p(_M_data))
2653 constexpr int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2654 constexpr _BuiltinType __active_mask
2655 = ~_BuiltinType() >> (__nbits - _Width);
2656 return (_M_data & __active_mask) == 0;
2661 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2662 _M_is_constprop_all_of()
const
2664 if (__builtin_constant_p(_M_data))
2666 constexpr int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2667 constexpr _BuiltinType __active_mask
2668 = ~_BuiltinType() >> (__nbits - _Width);
2669 return (_M_data & __active_mask) == __active_mask;
2674 _BuiltinType _M_data;
2678template <
bool _MustZeroInitPadding,
typename _BuiltinType>
2679 struct _SimdWrapperBase;
2681template <
typename _BuiltinType>
2682 struct _SimdWrapperBase<false, _BuiltinType>
2684 _GLIBCXX_SIMD_INTRINSIC
constexpr
2685 _SimdWrapperBase() =
default;
2687 _GLIBCXX_SIMD_INTRINSIC
constexpr
2688 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2690 _BuiltinType _M_data;
2693template <
typename _BuiltinType>
2694 struct _SimdWrapperBase<true, _BuiltinType>
2697 _GLIBCXX_SIMD_INTRINSIC
constexpr
2698 _SimdWrapperBase() : _M_data() {}
2700 _GLIBCXX_SIMD_INTRINSIC
constexpr
2701 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2703 _BuiltinType _M_data;
2708struct _DisabledSimdWrapper;
2710template <
typename _Tp,
size_t _W
idth>
2711 struct _SimdWrapper<
2713 void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2714 : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2715 && sizeof(_Tp) * _Width
2716 == sizeof(__vector_type_t<_Tp, _Width>),
2717 __vector_type_t<_Tp, _Width>>
2719 static constexpr bool _S_need_default_init
2720 = __has_iec559_behavior<__signaling_NaN, _Tp>::value
2721 and
sizeof(_Tp) * _Width ==
sizeof(__vector_type_t<_Tp, _Width>);
2723 using _BuiltinType = __vector_type_t<_Tp, _Width>;
2725 using _Base = _SimdWrapperBase<_S_need_default_init, _BuiltinType>;
2727 static_assert(__is_vectorizable_v<_Tp>);
2728 static_assert(_Width >= 2);
2730 using value_type = _Tp;
2732 static inline constexpr size_t _S_full_size
2733 =
sizeof(_BuiltinType) /
sizeof(value_type);
2734 static inline constexpr int _S_size = _Width;
2735 static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2737 using _Base::_M_data;
2739 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper<_Tp, _S_full_size>
2740 __as_full_vector()
const
2743 _GLIBCXX_SIMD_INTRINSIC
constexpr
2744 _SimdWrapper(initializer_list<_Tp> __init)
2745 : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2746 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2747 return __init.begin()[__i.value];
2750 _GLIBCXX_SIMD_INTRINSIC
constexpr
2751 _SimdWrapper() =
default;
2753 _GLIBCXX_SIMD_INTRINSIC
constexpr
2754 _SimdWrapper(
const _SimdWrapper&) =
default;
2756 _GLIBCXX_SIMD_INTRINSIC
constexpr
2757 _SimdWrapper(_SimdWrapper&&) =
default;
2759 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper&
2760 operator=(
const _SimdWrapper&) =
default;
2762 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper&
2763 operator=(_SimdWrapper&&) =
default;
2766 using _SimdWrapperBase<_S_need_default_init, _BuiltinType>::_SimdWrapperBase;
2770 _GLIBCXX_SIMD_INTRINSIC
constexpr
2771 _SimdWrapper(
conditional_t<is_same_v<_BuiltinType, __intrinsic_type_t<_Tp, _Width>>,
2772 _DisabledSimdWrapper, __intrinsic_type_t<_Tp, _Width>> __x)
2773 : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2777 template <
typename _V,
typename _TVT = _VectorTraits<_V>,
2778 typename = enable_if_t<sizeof(
typename _TVT::value_type) == sizeof(_Tp)
2779 and sizeof(_V) == sizeof(_BuiltinType)
2780 and is_
integral_v<_Tp>
2781 and is_
integral_v<
typename _TVT::value_type>>>
2782 _GLIBCXX_SIMD_INTRINSIC
constexpr
2783 _SimdWrapper(_V __x)
2784 : _Base(reinterpret_cast<_BuiltinType>(__x)) {}
2786 template <
typename... _As,
2787 typename =
enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2788 &&
sizeof...(_As) <= _Width)>>
2789 _GLIBCXX_SIMD_INTRINSIC
constexpr
2790 operator _SimdTuple<_Tp, _As...>()
const
2792 return __generate_from_n_evaluations<
sizeof...(_As), _SimdTuple<_Tp, _As...>>(
2793 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
2794 {
return _M_data[int(__i)]; });
2797 _GLIBCXX_SIMD_INTRINSIC
constexpr
2798 operator const _BuiltinType&()
const
2801 _GLIBCXX_SIMD_INTRINSIC
constexpr
2802 operator _BuiltinType&()
2805 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
2806 operator[](
size_t __i)
const
2807 {
return _M_data[__i]; }
2809 template <
size_t __i>
2810 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
2811 operator[](_SizeConstant<__i>)
const
2812 {
return _M_data[__i]; }
2814 _GLIBCXX_SIMD_INTRINSIC
constexpr void
2815 _M_set(
size_t __i, _Tp __x)
2817 if (__builtin_is_constant_evaluated())
2818 _M_data = __generate_from_n_evaluations<_Width, _BuiltinType>([&](
auto __j) {
2819 return __j == __i ? __x : _M_data[__j()];
2825 _GLIBCXX_SIMD_INTRINSIC
2827 _M_is_constprop()
const
2828 {
return __builtin_constant_p(_M_data); }
2830 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2831 _M_is_constprop_none_of()
const
2833 if (__builtin_constant_p(_M_data))
2836 if constexpr (is_floating_point_v<_Tp>)
2838 using _Ip = __int_for_sizeof_t<_Tp>;
2839 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2840 __execute_n_times<_Width>(
2841 [&](
auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2844 __execute_n_times<_Width>(
2845 [&](
auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2846 if (__builtin_constant_p(__r))
2852 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2853 _M_is_constprop_all_of()
const
2855 if (__builtin_constant_p(_M_data))
2858 if constexpr (is_floating_point_v<_Tp>)
2860 using _Ip = __int_for_sizeof_t<_Tp>;
2861 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2862 __execute_n_times<_Width>(
2863 [&](
auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2866 __execute_n_times<_Width>(
2867 [&](
auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2868 if (__builtin_constant_p(__r))
2878template <
typename _Tp>
2880 __vectorized_sizeof()
2882 if constexpr (!__is_vectorizable_v<_Tp>)
2885 if constexpr (
sizeof(_Tp) <= 8)
2888 if constexpr (__have_avx512bw)
2890 if constexpr (__have_avx512f &&
sizeof(_Tp) >= 4)
2892 if constexpr (__have_avx2)
2894 if constexpr (__have_avx && is_floating_point_v<_Tp>)
2896 if constexpr (__have_sse2)
2898 if constexpr (__have_sse && is_same_v<_Tp, float>)
2908 if constexpr (__have_power8vec
2909 || (__have_power_vmx && (
sizeof(_Tp) < 8))
2910 || (__have_power_vsx && is_floating_point_v<_Tp>) )
2914 if constexpr (__have_neon_a64)
2916 if constexpr (__have_neon_a32 and (not is_floating_point_v<_Tp>
2917 or is_same_v<_Tp, float>))
2919 if constexpr (__have_neon
2924 && (__support_neon_float || !is_floating_point_v<_Tp>))
2934template <
typename _Tp>
2935 inline constexpr int max_fixed_size
2936 = (__have_avx512bw &&
sizeof(_Tp) == 1) ? 64 : 32;
2939#if defined __x86_64__ || defined __aarch64__
2940template <
typename _Tp>
2941 using compatible =
conditional_t<(
sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2942#elif defined __ARM_NEON
2945template <
typename _Tp>
2948 && (__support_neon_float || !is_floating_point_v<_Tp>)),
2949 _VecBuiltin<16>, scalar>;
2952 using compatible = scalar;
2957template <
typename _Tp>
2959 __determine_native_abi()
2961 constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
2962 if constexpr (__bytes ==
sizeof(_Tp))
2963 return static_cast<scalar*
>(
nullptr);
2964 else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2965 return static_cast<_VecBltnBtmsk<__bytes>*
>(
nullptr);
2967 return static_cast<_VecBuiltin<__bytes>*
>(
nullptr);
2970template <
typename _Tp,
typename = enable_if_t<__is_vectorizable_v<_Tp>>>
2971 using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
2975#if defined _GLIBCXX_SIMD_DEFAULT_ABI
2976template <
typename _Tp>
2977 using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2979template <
typename _Tp>
2980 using __default_abi = compatible<_Tp>;
2987template <
typename _Tp>
2988 struct is_simd_flag_type
2993 struct is_simd_flag_type<element_aligned_tag>
2998 struct is_simd_flag_type<vector_aligned_tag>
3002template <
size_t _Np>
3003 struct is_simd_flag_type<overaligned_tag<_Np>>
3004 : __bool_constant<(_Np > 0) and __has_single_bit(_Np)>
3007template <
typename _Tp>
3008 inline constexpr bool is_simd_flag_type_v = is_simd_flag_type<_Tp>::value;
3010template <
typename _Tp,
typename = enable_if_t<is_simd_flag_type_v<_Tp>>>
3011 using _IsSimdFlagType = _Tp;
3014template <
typename _Tp,
typename =
void_t<>>
3017template <
typename _Tp>
3018 struct is_abi_tag<_Tp,
void_t<typename _Tp::_IsValidAbiTag>>
3019 :
public _Tp::_IsValidAbiTag {};
3021template <
typename _Tp>
3022 inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
3025template <
typename _Tp>
3028template <
typename _Tp>
3029 inline constexpr bool is_simd_v = is_simd<_Tp>::value;
3031template <
typename _Tp>
3034template <
typename _Tp>
3035inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
3038template <
typename _Tp,
typename _Abi,
typename =
void>
3039 struct __simd_size_impl {};
3041template <
typename _Tp,
typename _Abi>
3042 struct __simd_size_impl<
3044 enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
3045 : _SizeConstant<_Abi::template _S_size<_Tp>> {};
3047template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
3048 struct simd_size : __simd_size_impl<_Tp, _Abi> {};
3050template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
3051 inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
3054template <
typename _Tp,
size_t _Np,
typename =
void>
3055 struct __deduce_impl;
3066template <
typename _Tp,
size_t _Np,
typename...>
3069template <
typename _Tp,
size_t _Np,
typename... _Abis>
3070 using deduce_t =
typename deduce<_Tp, _Np, _Abis...>::type;
3075template <
typename _Tp,
typename _V,
typename =
void>
3078template <
typename _Tp,
typename _Up,
typename _Abi>
3079 struct rebind_simd<_Tp, simd<_Up, _Abi>,
3080 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
3081 {
using type = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>; };
3083template <
typename _Tp,
typename _Up,
typename _Abi>
3084 struct rebind_simd<_Tp, simd_mask<_Up, _Abi>,
3085 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
3086 {
using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>; };
3088template <
typename _Tp,
typename _V>
3089 using rebind_simd_t =
typename rebind_simd<_Tp, _V>::type;
3092template <
int _Np,
typename _V,
typename =
void>
3095template <
int _Np,
typename _Tp,
typename _Abi>
3096 struct resize_simd<_Np, simd<_Tp, _Abi>,
void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
3097 {
using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
3099template <
int _Np,
typename _Tp,
typename _Abi>
3100 struct resize_simd<_Np, simd_mask<_Tp, _Abi>,
void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
3101 {
using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
3103template <
int _Np,
typename _V>
3104 using resize_simd_t =
typename resize_simd<_Np, _V>::type;
3108template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
3109 struct memory_alignment
3110 :
public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
3112template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
3113 inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
3116template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
3119template <
typename _Tp,
typename _Abi>
3120 struct is_simd<simd<_Tp, _Abi>> :
public true_type {};
3122template <
typename _Tp>
3123 using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
3125template <
typename _Tp,
int _Np>
3126 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
3128template <
typename _Tp,
size_t _Np>
3129 using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
3132template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
3135template <
typename _Tp,
typename _Abi>
3136 struct is_simd_mask<simd_mask<_Tp, _Abi>> :
public true_type {};
3138template <
typename _Tp>
3139 using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
3141template <
typename _Tp,
int _Np>
3142 using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
3144template <
typename _Tp,
size_t _Np>
3145 using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
3149template <
typename _Tp,
typename _Up,
typename _Ap,
bool = is_simd_v<_Tp>,
typename =
void>
3150 struct __static_simd_cast_return_type;
3152template <
typename _Tp,
typename _A0,
typename _Up,
typename _Ap>
3153 struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false, void>
3154 : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
3156template <
typename _Tp,
typename _Up,
typename _Ap>
3157 struct __static_simd_cast_return_type<
3158 _Tp, _Up, _Ap, true,
enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
3159 {
using type = _Tp; };
3161template <
typename _Tp,
typename _Ap>
3162 struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
3163#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
3169 {
using type = simd<_Tp, _Ap>; };
3171template <
typename _Tp,
typename =
void>
3172 struct __safe_make_signed {
using type = _Tp;};
3174template <
typename _Tp>
3175 struct __safe_make_signed<_Tp,
enable_if_t<is_integral_v<_Tp>>>
3178 using type = make_signed_t<make_unsigned_t<_Tp>>;
3181template <
typename _Tp>
3182 using safe_make_signed_t =
typename __safe_make_signed<_Tp>::type;
3184template <
typename _Tp,
typename _Up,
typename _Ap>
3185 struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
3186#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
3194 (is_integral_v<_Up> && is_integral_v<_Tp> &&
3195#ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
3196 is_signed_v<_Up> != is_signed_v<_Tp> &&
3198 is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
3199 simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
3202template <
typename _Tp,
typename _Up,
typename _Ap,
3204 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3205 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
3206 static_simd_cast(
const simd<_Up, _Ap>& __x)
3208 if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
3212 _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
3214 return _R(__private_init, __c(__data(__x)));
3218namespace __proposed {
3219template <
typename _Tp,
typename _Up,
typename _Ap,
3221 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3222 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
typename _R::mask_type
3223 static_simd_cast(
const simd_mask<_Up, _Ap>& __x)
3225 using _RM =
typename _R::mask_type;
3226 return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
3227 typename _RM::simd_type::value_type>(__x)};
3230template <
typename _To,
typename _Up,
typename _Abi>
3231 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3233 simd_bit_cast(
const simd<_Up, _Abi>& __x)
3235 using _Tp =
typename _To::value_type;
3236 using _ToMember =
typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3237 using _From = simd<_Up, _Abi>;
3238 using _FromMember =
typename _SimdTraits<_Up, _Abi>::_SimdMember;
3240 static_assert(
sizeof(_To) ==
sizeof(_From));
3241 static_assert(is_trivially_copyable_v<_Tp> && is_trivially_copyable_v<_Up>);
3242 static_assert(is_trivially_copyable_v<_ToMember> && is_trivially_copyable_v<_FromMember>);
3243#if __has_builtin(__builtin_bit_cast)
3244 return {__private_init, __builtin_bit_cast(_ToMember, __data(__x))};
3246 return {__private_init, __bit_cast<_ToMember>(__data(__x))};
3250template <
typename _To,
typename _Up,
typename _Abi>
3251 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3253 simd_bit_cast(
const simd_mask<_Up, _Abi>& __x)
3255 using _From = simd_mask<_Up, _Abi>;
3256 static_assert(
sizeof(_To) ==
sizeof(_From));
3257 static_assert(is_trivially_copyable_v<_From>);
3260 if constexpr (is_simd_v<_To>)
3262 using _Tp =
typename _To::value_type;
3263 using _ToMember =
typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3264 static_assert(is_trivially_copyable_v<_ToMember>);
3265#if __has_builtin(__builtin_bit_cast)
3266 return {__private_init, __builtin_bit_cast(_ToMember, __x)};
3268 return {__private_init, __bit_cast<_ToMember>(__x)};
3273 static_assert(is_trivially_copyable_v<_To>);
3274#if __has_builtin(__builtin_bit_cast)
3275 return __builtin_bit_cast(_To, __x);
3277 return __bit_cast<_To>(__x);
3284template <
typename _Tp,
typename _Up,
typename _Ap,
3285 typename _To = __value_type_or_identity_t<_Tp>>
3286 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
3287 simd_cast(
const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
3288 ->
decltype(static_simd_cast<_Tp>(__x))
3289 {
return static_simd_cast<_Tp>(__x); }
3291namespace __proposed {
3292template <
typename _Tp,
typename _Up,
typename _Ap,
3293 typename _To = __value_type_or_identity_t<_Tp>>
3294 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
3295 simd_cast(
const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
3296 ->
decltype(static_simd_cast<_Tp>(__x))
3297 {
return static_simd_cast<_Tp>(__x); }
3302namespace __proposed {
3329template <
typename _Tp,
typename _Up,
typename _Ap>
3330 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
enable_if_t<
3331 conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
3332 resizing_simd_cast(
const simd<_Up, _Ap>& __x)
3334 if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
3336 else if (__builtin_is_constant_evaluated())
3337 return _Tp([&](
auto __i)
constexpr {
3338 return __i < simd_size_v<_Up, _Ap> ? __x[__i] : _Up();
3340 else if constexpr (simd_size_v<_Up, _Ap> == 1)
3346 else if constexpr (_Tp::size() == 1)
3348 else if constexpr (
sizeof(_Tp) ==
sizeof(__x)
3349 && !__is_fixed_size_abi_v<_Ap>)
3350 return {__private_init,
3351 __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
3352 _Ap::_S_masked(__data(__x))._M_data)};
3356 __builtin_memcpy(&__data(__r), &__data(__x),
3358 *
std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
3363template <
typename _Tp,
typename _Up,
typename _Ap>
3364 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3365 enable_if_t<is_simd_mask_v<_Tp>, _Tp>
3366 resizing_simd_cast(
const simd_mask<_Up, _Ap>& __x)
3368 return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3369 typename _Tp::simd_type::value_type>(__x)};
3375template <
typename _Tp,
int _Np>
3376 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3377 to_fixed_size(
const fixed_size_simd<_Tp, _Np>& __x)
3380template <
typename _Tp,
int _Np>
3381 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3382 to_fixed_size(
const fixed_size_simd_mask<_Tp, _Np>& __x)
3385template <
typename _Tp,
typename _Ap>
3386 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>
3387 to_fixed_size(
const simd<_Tp, _Ap>& __x)
3389 using _Rp = fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>;
3390 return _Rp([&__x](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; });
3393template <
typename _Tp,
typename _Ap>
3394 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, simd_size_v<_Tp, _Ap>>
3395 to_fixed_size(
const simd_mask<_Tp, _Ap>& __x)
3397 return {__private_init,
3398 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; }};
3402template <
typename _Tp,
int _Np>
3403 _GLIBCXX_SIMD_INTRINSIC
3404 enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3405 to_native(
const fixed_size_simd<_Tp, _Np>& __x)
3407 alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3408 __x.copy_to(__mem, vector_aligned);
3409 return {__mem, vector_aligned};
3412template <
typename _Tp,
int _Np>
3413 _GLIBCXX_SIMD_INTRINSIC
3414 enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3415 to_native(
const fixed_size_simd_mask<_Tp, _Np>& __x)
3417 return native_simd_mask<_Tp>(
3419 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; });
3423template <
typename _Tp,
int _Np>
3424 _GLIBCXX_SIMD_INTRINSIC
enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3425 to_compatible(
const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3427 alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3428 __x.copy_to(__mem, vector_aligned);
3429 return {__mem, vector_aligned};
3432template <
typename _Tp,
int _Np>
3433 _GLIBCXX_SIMD_INTRINSIC
3434 enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3435 to_compatible(
const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3437 return simd_mask<_Tp>(
3439 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; });
3446template <
typename _M,
typename _Tp>
3447 class const_where_expression
3450 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3452 struct _Wrapper {
using value_type = _V; };
3455 using _Impl =
typename _V::_Impl;
3458 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3460 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
3461 __get_mask(
const const_where_expression& __x)
3462 {
return __x._M_k; }
3464 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
3465 __get_lvalue(
const const_where_expression& __x)
3466 {
return __x._M_value; }
3472 const_where_expression(
const const_where_expression&) =
delete;
3474 const_where_expression& operator=(
const const_where_expression&) =
delete;
3476 _GLIBCXX_SIMD_INTRINSIC
constexpr
3477 const_where_expression(
const _M& __kk,
const _Tp& dd)
3478 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3480 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3483 return {__private_init,
3484 _Impl::template _S_masked_unary<negate>(__data(_M_k),
3488 template <
typename _Up,
typename _Flags>
3489 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3490 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>)
const&&
3492 return {__private_init,
3493 _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3494 _Flags::template _S_apply<_V>(__mem))};
3497 template <
typename _Up,
typename _Flags>
3498 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3499 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>)
const&&
3501 _Impl::_S_masked_store(__data(_M_value),
3502 _Flags::template _S_apply<_V>(__mem),
3508template <
typename _Tp>
3509 class const_where_expression<bool, _Tp>
3514 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3516 struct _Wrapper {
using value_type = _V; };
3520 =
typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3522 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
3523 __get_mask(
const const_where_expression& __x)
3524 {
return __x._M_k; }
3526 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
3527 __get_lvalue(
const const_where_expression& __x)
3528 {
return __x._M_value; }
3534 const_where_expression(
const const_where_expression&) =
delete;
3535 const_where_expression& operator=(
const const_where_expression&) =
delete;
3537 _GLIBCXX_SIMD_INTRINSIC
constexpr
3538 const_where_expression(
const bool __kk,
const _Tp& dd)
3539 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3541 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3543 {
return _M_k ? -_M_value : _M_value; }
3545 template <
typename _Up,
typename _Flags>
3546 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3547 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>)
const&&
3548 {
return _M_k ?
static_cast<_V
>(__mem[0]) : _M_value; }
3550 template <
typename _Up,
typename _Flags>
3551 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3552 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>)
const&&
3555 __mem[0] = _M_value;
3560template <
typename _M,
typename _Tp>
3561 class where_expression :
public const_where_expression<_M, _Tp>
3563 using _Impl =
typename const_where_expression<_M, _Tp>::_Impl;
3565 static_assert(!is_const<_Tp>::value,
3566 "where_expression may only be instantiated with __a non-const "
3569 using typename const_where_expression<_M, _Tp>::value_type;
3570 using const_where_expression<_M, _Tp>::_M_k;
3571 using const_where_expression<_M, _Tp>::_M_value;
3574 is_same<typename _M::abi_type, typename _Tp::abi_type>::value,
"");
3575 static_assert(_M::size() == _Tp::size(),
"");
3577 _GLIBCXX_SIMD_INTRINSIC
friend constexpr _Tp&
3578 __get_lvalue(where_expression& __x)
3579 {
return __x._M_value; }
3582 where_expression(
const where_expression&) =
delete;
3583 where_expression& operator=(
const where_expression&) =
delete;
3585 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3586 where_expression(
const _M& __kk, _Tp& dd)
3587 : const_where_expression<_M, _Tp>(__kk, dd) {}
3589 template <
typename _Up>
3590 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3591 operator=(_Up&& __x) &&
3593 _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3594 __to_value_type_or_member_type<_Tp>(
3595 static_cast<_Up&&
>(__x)));
3598#define _GLIBCXX_SIMD_OP_(__op, __name) \
3599 template <typename _Up> \
3600 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3601 operator __op##=(_Up&& __x)&& \
3603 _Impl::template _S_masked_cassign( \
3604 __data(_M_k), __data(_M_value), \
3605 __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3606 [](auto __impl, auto __lhs, auto __rhs) \
3607 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
3608 { return __impl.__name(__lhs, __rhs); }); \
3611 _GLIBCXX_SIMD_OP_(+, _S_plus);
3612 _GLIBCXX_SIMD_OP_(-, _S_minus);
3613 _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3614 _GLIBCXX_SIMD_OP_(/, _S_divides);
3615 _GLIBCXX_SIMD_OP_(%, _S_modulus);
3616 _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3617 _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3618 _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3619 _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3620 _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3621#undef _GLIBCXX_SIMD_OP_
3623 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3627 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3630 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3634 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3637 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3641 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3644 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3648 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3652 template <
typename _Up,
typename _Flags>
3653 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3654 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3656 __data(_M_value) = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3657 _Flags::template _S_apply<_Tp>(__mem));
3662template <
typename _Tp>
3663 class where_expression<bool, _Tp>
3664 :
public const_where_expression<bool, _Tp>
3667 using typename const_where_expression<_M, _Tp>::value_type;
3668 using const_where_expression<_M, _Tp>::_M_k;
3669 using const_where_expression<_M, _Tp>::_M_value;
3672 where_expression(
const where_expression&) =
delete;
3673 where_expression& operator=(
const where_expression&) =
delete;
3675 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3676 where_expression(
const _M& __kk, _Tp& dd)
3677 : const_where_expression<_M, _Tp>(__kk, dd) {}
3679#define _GLIBCXX_SIMD_OP_(__op) \
3680 template <typename _Up> \
3681 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3682 operator __op(_Up&& __x)&& \
3683 { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3685 _GLIBCXX_SIMD_OP_(=)
3686 _GLIBCXX_SIMD_OP_(+=)
3687 _GLIBCXX_SIMD_OP_(-=)
3688 _GLIBCXX_SIMD_OP_(*=)
3689 _GLIBCXX_SIMD_OP_(/=)
3690 _GLIBCXX_SIMD_OP_(%=)
3691 _GLIBCXX_SIMD_OP_(&=)
3692 _GLIBCXX_SIMD_OP_(|=)
3693 _GLIBCXX_SIMD_OP_(^=)
3694 _GLIBCXX_SIMD_OP_(<<=)
3695 _GLIBCXX_SIMD_OP_(>>=)
3696 #undef _GLIBCXX_SIMD_OP_
3698 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3700 {
if (_M_k) ++_M_value; }
3702 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3704 {
if (_M_k) ++_M_value; }
3706 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3708 {
if (_M_k) --_M_value; }
3710 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3712 {
if (_M_k) --_M_value; }
3715 template <
typename _Up,
typename _Flags>
3716 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3717 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3718 {
if (_M_k) _M_value = __mem[0]; }
3722template <
typename _Tp,
typename _Ap>
3723 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3724 where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3725 where(
const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3726 {
return {__k, __value}; }
3728template <
typename _Tp,
typename _Ap>
3729 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3730 const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3731 where(
const typename simd<_Tp, _Ap>::mask_type& __k,
const simd<_Tp, _Ap>& __value)
3732 {
return {__k, __value}; }
3734template <
typename _Tp,
typename _Ap>
3735 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3736 where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3737 where(
const remove_const_t<simd_mask<_Tp, _Ap>>& __k, simd_mask<_Tp, _Ap>& __value)
3738 {
return {__k, __value}; }
3740template <
typename _Tp,
typename _Ap>
3741 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3742 const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3743 where(
const remove_const_t<simd_mask<_Tp, _Ap>>& __k,
const simd_mask<_Tp, _Ap>& __value)
3744 {
return {__k, __value}; }
3746template <
typename _Tp>
3747 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR where_expression<bool, _Tp>
3748 where(_ExactBool __k, _Tp& __value)
3749 {
return {__k, __value}; }
3751template <
typename _Tp>
3752 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR const_where_expression<bool, _Tp>
3753 where(_ExactBool __k,
const _Tp& __value)
3754 {
return {__k, __value}; }
3756template <
typename _Tp,
typename _Ap>
3757 _GLIBCXX_SIMD_CONSTEXPR
void
3758 where(
bool __k, simd<_Tp, _Ap>& __value) =
delete;
3760template <
typename _Tp,
typename _Ap>
3761 _GLIBCXX_SIMD_CONSTEXPR
void
3762 where(
bool __k,
const simd<_Tp, _Ap>& __value) =
delete;
3765namespace __proposed {
3766template <
size_t _Np>
3769 const bitset<_Np> __bits;
3772 where_range(bitset<_Np> __b) : __bits(__b) {}
3779 _GLIBCXX_SIMD_INTRINSIC
void
3781 { __bit = __builtin_ctzl(__mask); }
3783 _GLIBCXX_SIMD_INTRINSIC
void
3787 __mask &= (__mask - 1);
3792 iterator(
decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3793 iterator(
const iterator&) =
default;
3794 iterator(iterator&&) =
default;
3796 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t
3800 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t
3804 _GLIBCXX_SIMD_ALWAYS_INLINE iterator&
3812 _GLIBCXX_SIMD_ALWAYS_INLINE iterator
3815 iterator __tmp = *
this;
3821 _GLIBCXX_SIMD_ALWAYS_INLINE
bool
3822 operator==(
const iterator& __rhs)
const
3823 {
return __mask == __rhs.__mask; }
3825 _GLIBCXX_SIMD_ALWAYS_INLINE
bool
3826 operator!=(
const iterator& __rhs)
const
3827 {
return __mask != __rhs.__mask; }
3832 {
return __bits.to_ullong(); }
3839template <
typename _Tp,
typename _Ap>
3840 where_range<simd_size_v<_Tp, _Ap>>
3841 where(
const simd_mask<_Tp, _Ap>& __k)
3842 {
return __k.__to_bitset(); }
3848template <
typename _Tp,
typename _Abi,
typename _BinaryOperation = plus<>>
3849 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3850 reduce(
const simd<_Tp, _Abi>& __v, _BinaryOperation __binary_op = _BinaryOperation())
3851 {
return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3853template <
typename _M,
typename _V,
typename _BinaryOperation = plus<>>
3854 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3855 reduce(
const const_where_expression<_M, _V>& __x,
3856 typename _V::value_type __identity_element, _BinaryOperation __binary_op)
3858 if (__builtin_expect(none_of(__get_mask(__x)),
false))
3859 return __identity_element;
3861 _V __tmp = __identity_element;
3862 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3863 __data(__get_lvalue(__x)));
3864 return reduce(__tmp, __binary_op);
3867template <
typename _M,
typename _V>
3868 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3869 reduce(
const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3870 {
return reduce(__x, 0, __binary_op); }
3872template <
typename _M,
typename _V>
3873 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3874 reduce(
const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3875 {
return reduce(__x, 1, __binary_op); }
3877template <
typename _M,
typename _V>
3878 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3879 reduce(
const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3880 {
return reduce(__x, ~
typename _V::value_type(), __binary_op); }
3882template <
typename _M,
typename _V>
3883 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3884 reduce(
const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3885 {
return reduce(__x, 0, __binary_op); }
3887template <
typename _M,
typename _V>
3888 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3889 reduce(
const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3890 {
return reduce(__x, 0, __binary_op); }
3892template <
typename _Tp,
typename _Abi>
3893 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3894 hmin(
const simd<_Tp, _Abi>& __v)
noexcept
3895 {
return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum()); }
3897template <
typename _Tp,
typename _Abi>
3898 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3899 hmax(
const simd<_Tp, _Abi>& __v)
noexcept
3900 {
return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum()); }
3902template <
typename _M,
typename _V>
3903 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3904 typename _V::value_type
3905 hmin(
const const_where_expression<_M, _V>& __x)
noexcept
3907 using _Tp =
typename _V::value_type;
3908 constexpr _Tp __id_elem =
3909#ifdef __FINITE_MATH_ONLY__
3910 __finite_max_v<_Tp>;
3912 __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3914 _V __tmp = __id_elem;
3915 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3916 __data(__get_lvalue(__x)));
3917 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3920template <
typename _M,
typename _V>
3921 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3922 typename _V::value_type
3923 hmax(
const const_where_expression<_M, _V>& __x)
noexcept
3925 using _Tp =
typename _V::value_type;
3926 constexpr _Tp __id_elem =
3927#ifdef __FINITE_MATH_ONLY__
3928 __finite_min_v<_Tp>;
3931 if constexpr (__value_exists_v<__infinity, _Tp>)
3932 return -__infinity_v<_Tp>;
3934 return __finite_min_v<_Tp>;
3937 _V __tmp = __id_elem;
3938 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3939 __data(__get_lvalue(__x)));
3940 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3945template <
typename _Tp,
typename _Ap>
3946 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3947 min(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3948 {
return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3950template <
typename _Tp,
typename _Ap>
3951 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3952 max(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3953 {
return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3955template <
typename _Tp,
typename _Ap>
3956 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3957 pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3958 minmax(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3960 const auto pair_of_members
3961 = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3962 return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3963 simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3966template <
typename _Tp,
typename _Ap>
3967 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3968 clamp(
const simd<_Tp, _Ap>& __v,
const simd<_Tp, _Ap>& __lo,
const simd<_Tp, _Ap>& __hi)
3970 using _Impl =
typename _Ap::_SimdImpl;
3971 return {__private_init,
3972 _Impl::_S_min(__data(__hi),
3973 _Impl::_S_max(__data(__lo), __data(__v)))};
3978template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
3979 typename =
enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
3980 inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3981 split(
const simd<_Tp, _Ap>&);
3984template <
int _Index,
int _Total,
int _Combine = 1,
typename _Tp,
size_t _Np>
3985 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST
constexpr
3986 conditional_t<_Np == _Total and _Combine == 1, _Tp, _SimdWrapper<_Tp, _Np / _Total * _Combine>>
3987 __extract_part(
const _SimdWrapper<_Tp, _Np> __x);
3989template <
int _Index,
int _Parts,
int _Combine = 1,
typename _Tp,
typename _A0,
typename... _As>
3990 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
3991 __extract_part(
const _SimdTuple<_Tp, _A0, _As...>& __x);
3995template <
size_t _V0,
size_t... _Values>
3998 template <
size_t _I>
3999 static constexpr size_t
4000 _S_at(_SizeConstant<_I> = {})
4002 if constexpr (_I == 0)
4005 return _SizeList<_Values...>::template _S_at<_I - 1>();
4008 template <
size_t _I>
4009 static constexpr auto
4010 _S_before(_SizeConstant<_I> = {})
4012 if constexpr (_I == 0)
4013 return _SizeConstant<0>();
4015 return _SizeConstant<
4016 _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
4019 template <
size_t _Np>
4020 static constexpr auto
4021 _S_pop_front(_SizeConstant<_Np> = {})
4023 if constexpr (_Np == 0)
4026 return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
4032template <
typename _Tp,
size_t _Np>
4033 _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
4034 __extract_center(_SimdWrapper<_Tp, _Np> __x)
4036 static_assert(_Np >= 4);
4037 static_assert(_Np % 4 == 0);
4038#if _GLIBCXX_SIMD_X86INTRIN
4039 if constexpr (__have_avx512f &&
sizeof(_Tp) * _Np == 64)
4041 const auto __intrin = __to_intrin(__x);
4042 if constexpr (is_integral_v<_Tp>)
4043 return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
4044 _mm512_shuffle_i32x4(__intrin, __intrin,
4045 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
4046 else if constexpr (
sizeof(_Tp) == 4)
4047 return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
4048 _mm512_shuffle_f32x4(__intrin, __intrin,
4049 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
4050 else if constexpr (
sizeof(_Tp) == 8)
4051 return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
4052 _mm512_shuffle_f64x2(__intrin, __intrin,
4053 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
4055 __assert_unreachable<_Tp>();
4057 else if constexpr (
sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
4058 return __vector_bitcast<_Tp>(
4059 _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
4060 __hi128(__vector_bitcast<double>(__x)), 1));
4061 else if constexpr (
sizeof(__x) == 32 &&
sizeof(_Tp) * _Np <= 32)
4062 return __vector_bitcast<_Tp>(
4063 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
4064 __lo128(__vector_bitcast<_LLong>(__x)),
4065 sizeof(_Tp) * _Np / 4));
4069 __vector_type_t<_Tp, _Np / 2> __r;
4070 __builtin_memcpy(&__r,
4071 reinterpret_cast<const char*
>(&__x)
4072 +
sizeof(_Tp) * _Np / 4,
4073 sizeof(_Tp) * _Np / 2);
4078template <
typename _Tp,
typename _A0,
typename... _As>
4079 _GLIBCXX_SIMD_INTRINSIC
4080 _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
4081 __extract_center(
const _SimdTuple<_Tp, _A0, _As...>& __x)
4083 if constexpr (
sizeof...(_As) == 0)
4084 return __extract_center(__x.first);
4086 return __extract_part<1, 4, 2>(__x);
4091template <
size_t... _Sizes,
typename _Tp,
typename... _As>
4093 __split_wrapper(_SizeList<_Sizes...>,
const _SimdTuple<_Tp, _As...>& __x)
4095 return split<_Sizes...>(
4096 fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
4103template <
typename _V,
typename _Ap,
4104 size_t _Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
4105 enable_if_t<simd_size_v<typename _V::value_type, _Ap> == _Parts * _V::size()
4106 && is_simd_v<_V>, array<_V, _Parts>>
4107 split(
const simd<typename _V::value_type, _Ap>& __x)
4109 using _Tp =
typename _V::value_type;
4110 if constexpr (_Parts == 1)
4112 return {simd_cast<_V>(__x)};
4114 else if (__x._M_is_constprop())
4116 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4117 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4118 return _V([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4119 {
return __x[__i * _V::size() + __j]; });
4123 __is_fixed_size_abi_v<_Ap>
4124 && (is_same_v<typename _V::abi_type, simd_abi::scalar>
4125 || (__is_fixed_size_abi_v<typename _V::abi_type>
4126 &&
sizeof(_V) ==
sizeof(_Tp) * _V::size()
4130#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4131 const __may_alias<_Tp>*
const __element_ptr
4132 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__data(__x));
4133 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4134 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4135 {
return _V(__element_ptr + __i * _V::size(), vector_aligned); });
4137 const auto& __xx = __data(__x);
4138 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4139 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4140 [[maybe_unused]]
constexpr size_t __offset
4141 =
decltype(__i)::value * _V::size();
4142 return _V([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4143 constexpr _SizeConstant<__j + __offset> __k;
4149 else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
4152 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4153 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; });
4157 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4158 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4159 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
4160 return _V([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4161 return __x[__i * _V::size() + __j];
4164 return _V(__private_init,
4165 __extract_part<
decltype(__i)::value, _Parts>(__data(__x)));
4172template <
typename _V,
typename _Ap,
4173 size_t _Parts = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
4174 enable_if_t<is_simd_mask_v<_V> && simd_size_v<
typename
4175 _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
4176 split(
const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
4178 if constexpr (is_same_v<_Ap, typename _V::abi_type>)
4180 else if constexpr (_Parts == 1)
4181 return {__proposed::static_simd_cast<_V>(__x)};
4182 else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
4183 && __is_avx_abi<_Ap>())
4184 return {_V(__private_init, __lo128(__data(__x))),
4185 _V(__private_init, __hi128(__data(__x)))};
4186 else if constexpr (_V::size() <= __CHAR_BIT__ *
sizeof(_ULLong))
4188 const bitset __bits = __x.__to_bitset();
4189 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4190 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4191 constexpr size_t __offset = __i * _V::size();
4192 return _V(__bitset_init, (__bits >> __offset).to_ullong());
4197 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4198 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4199 constexpr size_t __offset = __i * _V::size();
4200 return _V(__private_init,
4201 [&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4202 return __x[__j + __offset];
4210template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
typename>
4211 _GLIBCXX_SIMD_ALWAYS_INLINE
4212 tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
4213 split(
const simd<_Tp, _Ap>& __x)
4215 using _SL = _SizeList<_Sizes...>;
4216 using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
4217 constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
4218 constexpr size_t _N0 = _SL::template _S_at<0>();
4219 using _V = __deduced_simd<_Tp, _N0>;
4221 if (__x._M_is_constprop())
4222 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>(
4223 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4224 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4225 constexpr size_t __offset = _SL::_S_before(__i);
4226 return _Vi([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4227 return __x[__offset + __j];
4230 else if constexpr (_Np == _N0)
4232 static_assert(
sizeof...(_Sizes) == 1);
4233 return {simd_cast<_V>(__x)};
4236 (__is_fixed_size_abi_v<
4237 _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
4240 !__is_fixed_size_abi_v<typename _V::abi_type>,
4241 "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
4247 __split_wrapper(_SL::template _S_pop_front<1>(),
4248 __data(__x).second));
4250 else if constexpr ((!__is_fixed_size_abi_v<simd_abi::deduce_t<_Tp, _Sizes>> && ...))
4252 constexpr array<size_t,
sizeof...(_Sizes)> __size = {_Sizes...};
4253 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>(
4254 [&](
auto __i)
constexpr {
4255 constexpr size_t __offset = [&]() {
4257 for (
unsigned __j = 0; __j < __i; ++__j)
4261 return __deduced_simd<_Tp, __size[__i]>(
4263 __extract_part<__offset, _Np, __size[__i]>(__data(__x)));
4266#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4267 const __may_alias<_Tp>*
const __element_ptr
4268 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__x);
4269 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>(
4270 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4271 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4272 constexpr size_t __offset = _SL::_S_before(__i);
4273 constexpr size_t __base_align =
alignof(simd<_Tp, _Ap>);
4274 constexpr size_t __a
4275 = __base_align - ((__offset *
sizeof(_Tp)) % __base_align);
4276 constexpr size_t __b = ((__a - 1) & __a) ^ __a;
4277 constexpr size_t __alignment = __b == 0 ? __a : __b;
4278 return _Vi(__element_ptr + __offset, overaligned<__alignment>);
4281 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>(
4282 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4283 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4284 const auto& __xx = __data(__x);
4285 using _Offset =
decltype(_SL::_S_before(__i));
4286 return _Vi([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4287 constexpr _SizeConstant<_Offset::value + __j> __k;
4297template <
size_t _I,
typename _Tp,
typename _Ap,
typename... _As>
4298 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
4299 __subscript_in_pack(
const simd<_Tp, _Ap>& __x,
const simd<_Tp, _As>&... __xs)
4301 if constexpr (_I < simd_size_v<_Tp, _Ap>)
4304 return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
4309template <
typename _Tp,
typename _A0,
typename... _As>
4310 _GLIBCXX_SIMD_INTRINSIC
void
4311 __store_pack_of_simd(
char* __mem,
const simd<_Tp, _A0>& __x0,
const simd<_Tp, _As>&... __xs)
4313 constexpr size_t __n_bytes =
sizeof(_Tp) * simd_size_v<_Tp, _A0>;
4314 __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
4315 if constexpr (
sizeof...(__xs) > 0)
4316 __store_pack_of_simd(__mem + __n_bytes, __xs...);
4321template <
typename _Tp,
typename... _As,
typename = __detail::__odr_helper>
4322 inline _GLIBCXX_SIMD_CONSTEXPR
4323 simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
4324 concat(
const simd<_Tp, _As>&... __xs)
4326 constexpr int _Np = (simd_size_v<_Tp, _As> + ...);
4327 using _Abi = simd_abi::deduce_t<_Tp, _Np>;
4328 using _Rp = simd<_Tp, _Abi>;
4329 using _RW =
typename _SimdTraits<_Tp, _Abi>::_SimdMember;
4330 if constexpr (
sizeof...(__xs) == 1)
4331 return simd_cast<_Rp>(__xs...);
4332 else if ((... && __xs._M_is_constprop()))
4333 return _Rp([&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4334 {
return __subscript_in_pack<__i>(__xs...); });
4335 else if constexpr (__is_simd_wrapper_v<_RW> and
sizeof...(__xs) == 2)
4337 return {__private_init,
4340 constexpr int __sizes[2] = {int(simd_size_v<_Tp, _As>)...};
4341 constexpr int __vsizes[2]
4342 = {int(
sizeof(__as_vector(__xs)) /
sizeof(_Tp))...};
4343 constexpr int __padding0 = __vsizes[0] - __sizes[0];
4344 return __i >= _Np ? -1 : __i < __sizes[0] ? __i : __i + __padding0;
4347 else if constexpr (__is_simd_wrapper_v<_RW> and
sizeof...(__xs) == 3)
4348 return [](
const auto& __x0,
const auto& __x1,
const auto& __x2)
4349 _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4350 return concat(concat(__x0, __x1), __x2);
4352 else if constexpr (__is_simd_wrapper_v<_RW> and
sizeof...(__xs) > 3)
4353 return [](
const auto& __x0,
const auto& __x1,
const auto&... __rest)
4354 _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4355 return concat(concat(__x0, __x1), concat(__rest...));
4360 __store_pack_of_simd(
reinterpret_cast<char*
>(&__data(__r)), __xs...);
4367template <
typename _Tp,
typename _Abi,
size_t _Np>
4368 _GLIBCXX_SIMD_ALWAYS_INLINE
4369 _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
4370 concat(
const array<simd<_Tp, _Abi>, _Np>& __x)
4372 return __call_with_subscripts<_Np>(
4373 __x, [](
const auto&... __xs) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4374 return concat(__xs...);
4382template <
typename _Up,
typename _Accessor = _Up,
4383 typename _ValueType =
typename _Up::value_type>
4384 class _SmartReference
4390 _GLIBCXX_SIMD_INTRINSIC
constexpr _ValueType
4391 _M_read() const noexcept
4393 if constexpr (is_arithmetic_v<_Up>)
4396 return _M_obj[_M_index];
4399 template <
typename _Tp>
4400 _GLIBCXX_SIMD_INTRINSIC
constexpr void
4401 _M_write(_Tp&& __x)
const
4402 { _Accessor::_S_set(_M_obj, _M_index,
static_cast<_Tp&&
>(__x)); }
4405 _GLIBCXX_SIMD_INTRINSIC
constexpr
4406 _SmartReference(_Up& __o,
int __i) noexcept
4407 : _M_index(__i), _M_obj(__o) {}
4409 using value_type = _ValueType;
4411 _GLIBCXX_SIMD_INTRINSIC
4412 _SmartReference(
const _SmartReference&) =
delete;
4414 _GLIBCXX_SIMD_INTRINSIC
constexpr
4415 operator value_type() const noexcept
4416 {
return _M_read(); }
4418 template <
typename _Tp,
typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4419 _GLIBCXX_SIMD_INTRINSIC
constexpr _SmartReference
4420 operator=(_Tp&& __x) &&
4422 _M_write(
static_cast<_Tp&&
>(__x));
4423 return {_M_obj, _M_index};
4426#define _GLIBCXX_SIMD_OP_(__op) \
4427 template <typename _Tp, \
4428 typename _TT = decltype(declval<value_type>() __op declval<_Tp>()), \
4429 typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4430 typename = _ValuePreservingOrInt<_TT, value_type>> \
4431 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4432 operator __op##=(_Tp&& __x) && \
4434 const value_type& __lhs = _M_read(); \
4435 _M_write(__lhs __op __x); \
4436 return {_M_obj, _M_index}; \
4438 _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4439 _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4440 _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4441#undef _GLIBCXX_SIMD_OP_
4443 template <
typename _Tp = void,
4444 typename =
decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4445 _GLIBCXX_SIMD_INTRINSIC
constexpr _SmartReference
4448 value_type __x = _M_read();
4450 return {_M_obj, _M_index};
4453 template <
typename _Tp = void,
4454 typename =
decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4455 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type
4458 const value_type __r = _M_read();
4459 value_type __x = __r;
4464 template <
typename _Tp = void,
4465 typename =
decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4466 _GLIBCXX_SIMD_INTRINSIC
constexpr _SmartReference
4469 value_type __x = _M_read();
4471 return {_M_obj, _M_index};
4474 template <
typename _Tp = void,
4475 typename =
decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4476 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type
4479 const value_type __r = _M_read();
4480 value_type __x = __r;
4485 _GLIBCXX_SIMD_INTRINSIC
friend void
4486 swap(_SmartReference&& __a, _SmartReference&& __b)
noexcept(
4488 is_nothrow_constructible<value_type, _SmartReference&&>,
4489 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4491 value_type __tmp =
static_cast<_SmartReference&&
>(__a);
4492 static_cast<_SmartReference&&
>(__a) =
static_cast<value_type
>(__b);
4493 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
4496 _GLIBCXX_SIMD_INTRINSIC
friend void
4497 swap(value_type& __a, _SmartReference&& __b)
noexcept(
4499 is_nothrow_constructible<value_type, value_type&&>,
4500 is_nothrow_assignable<value_type&, value_type&&>,
4501 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4504 __a =
static_cast<value_type
>(__b);
4505 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
4508 _GLIBCXX_SIMD_INTRINSIC
friend void
4509 swap(_SmartReference&& __a, value_type& __b)
noexcept(
4511 is_nothrow_constructible<value_type, _SmartReference&&>,
4512 is_nothrow_assignable<value_type&, value_type&&>,
4513 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4515 value_type __tmp(__a);
4516 static_cast<_SmartReference&&
>(__a) =
std::move(__b);
4523template <
int _Bytes>
4524 struct __scalar_abi_wrapper
4526 template <
typename _Tp>
static constexpr size_t _S_full_size = 1;
4527 template <
typename _Tp>
static constexpr size_t _S_size = 1;
4528 template <
typename _Tp>
static constexpr size_t _S_is_partial =
false;
4530 template <
typename _Tp,
typename _Abi = simd_abi::scalar>
4531 static constexpr bool _S_is_valid_v
4532 = _Abi::template _IsValid<_Tp>::value &&
sizeof(_Tp) == _Bytes;
4537template <
typename _Tp>
4538 struct __decay_abi {
using type = _Tp; };
4540template <
int _Bytes>
4541 struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4542 {
using type = simd_abi::scalar; };
4550template <
template <
int>
class _Abi,
int _Bytes,
typename _Tp>
4551 struct __find_next_valid_abi
4553 static constexpr auto
4556 constexpr int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4557 using _NextAbi = _Abi<_NextBytes>;
4558 if constexpr (_NextBytes <
sizeof(_Tp) * 2)
4559 return _Abi<_Bytes>();
4560 else if constexpr (_NextAbi::template _S_is_partial<_Tp> ==
false
4561 && _NextAbi::template _S_is_valid_v<_Tp>)
4564 return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4567 using type =
decltype(_S_choose());
4570template <
int _Bytes,
typename _Tp>
4571 struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4572 {
using type = simd_abi::scalar; };
4575template <
template <
int>
class...>
4578 template <
typename,
int>
static constexpr bool _S_has_valid_abi =
false;
4579 template <
typename,
int>
using _FirstValidAbi = void;
4580 template <
typename,
int>
using _BestAbi = void;
4583template <
template <
int>
class _A0,
template <
int>
class... _Rest>
4584 struct _AbiList<_A0, _Rest...>
4586 template <
typename _Tp,
int _Np>
4587 static constexpr bool _S_has_valid_abi
4588 = _A0<
sizeof(_Tp) * _Np>::template _S_is_valid_v<
4589 _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4591 template <
typename _Tp,
int _Np>
4593 _A0<
sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4594 typename __decay_abi<_A0<
sizeof(_Tp) * _Np>>::type,
4595 typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4597 template <
typename _Tp,
int _Np>
4598 static constexpr auto
4599 _S_determine_best_abi()
4601 static_assert(_Np >= 1);
4602 constexpr int _Bytes =
sizeof(_Tp) * _Np;
4603 if constexpr (_Np == 1)
4604 return __make_dependent_t<_Tp, simd_abi::scalar>{};
4607 constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4612 if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4613 _Tp> && __fullsize / 2 < _Np)
4614 return typename __decay_abi<_A0<_Bytes>>::type{};
4618 typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4619 if constexpr (_Bp::template _S_is_valid_v<
4620 _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4624 typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4629 template <
typename _Tp,
int _Np>
4630 using _BestAbi =
decltype(_S_determine_best_abi<_Tp, _Np>());
4638using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4639 __scalar_abi_wrapper>;
4642template <
typename _Tp,
typename _Abi>
4643 struct _SimdTraits<_Tp, _Abi,
void_t<typename _Abi::template _IsValid<_Tp>>>
4644 : _Abi::template __traits<_Tp> {};
4648template <
typename _Tp,
size_t _Np>
4649 struct __deduce_impl<
4650 _Tp, _Np,
enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4651 {
using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4654template <
typename _Tp,
size_t _Np,
typename =
void>
4655 struct __deduce_fixed_size_fallback {};
4657template <
typename _Tp,
size_t _Np>
4658 struct __deduce_fixed_size_fallback<_Tp, _Np,
4659 enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4660 {
using type = simd_abi::fixed_size<_Np>; };
4662template <
typename _Tp,
size_t _Np,
typename>
4663 struct __deduce_impl :
public __deduce_fixed_size_fallback<_Tp, _Np> {};
4669template <
typename _Tp,
typename _Abi>
4670 class simd_mask :
public _SimdTraits<_Tp, _Abi>::_MaskBase
4673 using _Traits = _SimdTraits<_Tp, _Abi>;
4674 using _MemberType =
typename _Traits::_MaskMember;
4680 using _Ip = __int_for_sizeof_t<_Tp>;
4681 static constexpr _Ip* _S_type_tag =
nullptr;
4683 friend typename _Traits::_MaskBase;
4684 friend class simd<_Tp, _Abi>;
4685 friend typename _Traits::_SimdImpl;
4688 using _Impl =
typename _Traits::_MaskImpl;
4693 using value_type = bool;
4694 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4695 using simd_type = simd<_Tp, _Abi>;
4696 using abi_type = _Abi;
4699 static constexpr size_t size()
4700 {
return __size_or_zero_v<_Tp, _Abi>; }
4704 simd_mask() =
default;
4705 simd_mask(
const simd_mask&) =
default;
4706 simd_mask(simd_mask&&) =
default;
4707 simd_mask& operator=(
const simd_mask&) =
default;
4708 simd_mask& operator=(simd_mask&&) =
default;
4712 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
explicit
4713 simd_mask(
typename _Traits::_MaskCastType __init)
4714 : _M_data{__init} {}
4723 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
static simd_mask
4724 __from_bitset(bitset<size()> bs)
4725 {
return {__bitset_init, bs}; }
4727 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bitset<size()>
4729 {
return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4733 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
4734 simd_mask(value_type __x)
4735 : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4739 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4741 template <
typename _Up,
typename _A2,
4742 typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4743 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit(
sizeof(_MemberType)
4744 !=
sizeof(
typename _SimdTraits<_Up, _A2>::_MaskMember))
4745 simd_mask(
const simd_mask<_Up, _A2>& __x)
4746 : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4749 template <
typename _Up,
typename =
enable_if_t<conjunction<
4750 is_same<abi_type, simd_abi::fixed_size<size()>>,
4751 is_same<_Up, _Up>>::value>>
4752 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4753 simd_mask(
const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4754 : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4759 template <
typename _Flags>
4760 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4761 simd_mask(
const value_type* __mem, _IsSimdFlagType<_Flags>)
4762 : _M_data(_Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem))) {}
4764 template <
typename _Flags>
4765 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4766 simd_mask(
const value_type* __mem, simd_mask __k, _IsSimdFlagType<_Flags>)
4769 _M_data = _Impl::_S_masked_load(_M_data, __k._M_data,
4770 _Flags::template _S_apply<simd_mask>(__mem));
4775 template <
typename _Flags>
4776 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
void
4777 copy_from(
const value_type* __mem, _IsSimdFlagType<_Flags>)
4778 { _M_data = _Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem)); }
4782 template <
typename _Flags>
4783 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
void
4784 copy_to(value_type* __mem, _IsSimdFlagType<_Flags>)
const
4785 { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4789 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
4790 operator[](
size_t __i)
4793 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i, size() - 1);
4794 return {_M_data, int(__i)};
4797 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
4798 operator[](
size_t __i)
const
4801 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i, size() - 1);
4802 if constexpr (__is_scalar_abi<_Abi>())
4805 return static_cast<bool>(_M_data[__i]);
4810 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd_mask
4812 {
return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4816 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4818 template <
typename _Up,
typename _A2,
4819 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4820 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4821 operator&&(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4823 return {__private_init,
4824 _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4827 template <
typename _Up,
typename _A2,
4828 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4829 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4830 operator||(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4832 return {__private_init,
4833 _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4837 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4838 operator&&(
const simd_mask& __x,
const simd_mask& __y)
4839 {
return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)}; }
4841 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4842 operator||(
const simd_mask& __x,
const simd_mask& __y)
4843 {
return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)}; }
4845 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4846 operator&(
const simd_mask& __x,
const simd_mask& __y)
4847 {
return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4849 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4850 operator|(
const simd_mask& __x,
const simd_mask& __y)
4851 {
return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4853 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4854 operator^(
const simd_mask& __x,
const simd_mask& __y)
4855 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4857 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask&
4858 operator&=(simd_mask& __x,
const simd_mask& __y)
4860 __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4864 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask&
4865 operator|=(simd_mask& __x,
const simd_mask& __y)
4867 __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4871 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask&
4872 operator^=(simd_mask& __x,
const simd_mask& __y)
4874 __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4880 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4881 operator==(
const simd_mask& __x,
const simd_mask& __y)
4882 {
return !operator!=(__x, __y); }
4884 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4885 operator!=(
const simd_mask& __x,
const simd_mask& __y)
4886 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4890 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4891 simd_mask(_PrivateInit,
typename _Traits::_MaskMember __init)
4892 : _M_data(__init) {}
4896 template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4897 _GLIBCXX_SIMD_INTRINSIC
constexpr
4898 simd_mask(_PrivateInit, _Fp&& __gen)
4901 __execute_n_times<size()>([&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4902 _Impl::_S_set(_M_data, __i, __gen(__i));
4908 _GLIBCXX_SIMD_INTRINSIC
constexpr
4909 simd_mask(_BitsetInit, bitset<size()> __init)
4910 : _M_data(_Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
4921 template <
typename _Up,
typename _A2,
4922 typename = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4923 _GLIBCXX_SIMD_ALWAYS_INLINE
4924 operator simd_mask<_Up, _A2>() &&
4926 using namespace std::experimental::__proposed;
4927 return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4930 const simd_mask<_Tp, _Abi>& _M_data;
4933 _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4939 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4940 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4941 operator?:(
const simd_mask& __k,
const simd_mask& __where_true,
4942 const simd_mask& __where_false)
4944 auto __ret = __where_false;
4945 _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4949 template <
typename _U1,
typename _U2,
4950 typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4952 is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4953 is_convertible<simd_mask, typename _Rp::mask_type>>>>
4954 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend _Rp
4955 operator?:(
const simd_mask& __k,
const _U1& __where_true,
4956 const _U2& __where_false)
4958 _Rp __ret = __where_false;
4959 _Rp::_Impl::_S_masked_assign(
4960 __data(
static_cast<typename _Rp::mask_type
>(__k)), __data(__ret),
4961 __data(
static_cast<_Rp
>(__where_true)));
4965 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4966 template <
typename _Kp,
typename _Ak,
typename _Up,
typename _Au,
4968 conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4969 is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4970 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4971 operator?:(
const simd_mask<_Kp, _Ak>& __k,
const simd_mask& __where_true,
4972 const simd_mask<_Up, _Au>& __where_false)
4974 simd_mask __ret = __where_false;
4975 _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4976 __where_true._M_data);
4984 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
4985 _M_is_constprop()
const
4987 if constexpr (__is_scalar_abi<_Abi>())
4988 return __builtin_constant_p(_M_data);
4990 return _M_data._M_is_constprop();
4996 friend const auto& __data<_Tp, abi_type>(
const simd_mask&);
4997 friend auto& __data<_Tp, abi_type>(simd_mask&);
4998 alignas(_Traits::_S_mask_align) _MemberType _M_data;
5005template <
typename _Tp,
typename _Ap>
5006 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
5007 __data(
const simd_mask<_Tp, _Ap>& __x)
5008 {
return __x._M_data; }
5010template <
typename _Tp,
typename _Ap>
5011 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
5012 __data(simd_mask<_Tp, _Ap>& __x)
5013 {
return __x._M_data; }
5019template <
typename _Tp,
typename _Abi>
5020 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5021 all_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
5023 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5025 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
5031 return _Abi::_MaskImpl::_S_all_of(__k);
5034template <
typename _Tp,
typename _Abi>
5035 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5036 any_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
5038 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5040 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
5046 return _Abi::_MaskImpl::_S_any_of(__k);
5049template <
typename _Tp,
typename _Abi>
5050 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5051 none_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
5053 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5055 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
5061 return _Abi::_MaskImpl::_S_none_of(__k);
5064template <
typename _Tp,
typename _Abi>
5065 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5066 some_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
5068 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5070 for (
size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
5071 if (__k[__i] != __k[__i - 1])
5076 return _Abi::_MaskImpl::_S_some_of(__k);
5079template <
typename _Tp,
typename _Abi>
5080 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
5081 popcount(
const simd_mask<_Tp, _Abi>& __k)
noexcept
5083 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5085 const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
5086 __k, [](
auto... __elements) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5087 return ((__elements != 0) + ...);
5089 if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
5092 return _Abi::_MaskImpl::_S_popcount(__k);
5095template <
typename _Tp,
typename _Abi>
5096 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
5097 find_first_set(
const simd_mask<_Tp, _Abi>& __k)
5099 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5101 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
5102 const size_t _Idx = __call_with_n_evaluations<_Np>(
5103 [](
auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5105 }, [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5106 return __k[__i] ? +__i : _Np;
5109 __invoke_ub(
"find_first_set(empty mask) is UB");
5110 if (__builtin_constant_p(_Idx))
5113 return _Abi::_MaskImpl::_S_find_first_set(__k);
5116template <
typename _Tp,
typename _Abi>
5117 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
5118 find_last_set(
const simd_mask<_Tp, _Abi>& __k)
5120 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5122 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
5123 const int _Idx = __call_with_n_evaluations<_Np>(
5124 [](
auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5126 }, [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5127 return __k[__i] ? int(__i) : -1;
5130 __invoke_ub(
"find_first_set(empty mask) is UB");
5131 if (__builtin_constant_p(_Idx))
5134 return _Abi::_MaskImpl::_S_find_last_set(__k);
5137_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5138all_of(_ExactBool __x)
noexcept
5141_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5142any_of(_ExactBool __x)
noexcept
5145_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5146none_of(_ExactBool __x)
noexcept
5149_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5150some_of(_ExactBool)
noexcept
5153_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
5157_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
5158find_first_set(_ExactBool)
5161_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
5162find_last_set(_ExactBool)
5169template <
typename _V,
typename _Tp,
typename _Abi,
bool>
5170 class _SimdIntOperators {};
5172template <
typename _V,
typename _Tp,
typename _Abi>
5173 class _SimdIntOperators<_V, _Tp, _Abi, true>
5175 using _Impl =
typename _SimdTraits<_Tp, _Abi>::_SimdImpl;
5177 _GLIBCXX_SIMD_INTRINSIC
constexpr const _V&
5179 {
return *
static_cast<const _V*
>(
this); }
5181 template <
typename _Up>
5182 _GLIBCXX_SIMD_INTRINSIC
static _GLIBCXX_SIMD_CONSTEXPR _V
5183 _S_make_derived(_Up&& __d)
5184 {
return {__private_init,
static_cast<_Up&&
>(__d)}; }
5187 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5189 operator%=(_V& __lhs,
const _V& __x)
5190 {
return __lhs = __lhs % __x; }
5192 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5194 operator&=(_V& __lhs,
const _V& __x)
5195 {
return __lhs = __lhs & __x; }
5197 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5199 operator|=(_V& __lhs,
const _V& __x)
5200 {
return __lhs = __lhs | __x; }
5202 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5204 operator^=(_V& __lhs,
const _V& __x)
5205 {
return __lhs = __lhs ^ __x; }
5207 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5209 operator<<=(_V& __lhs,
const _V& __x)
5210 {
return __lhs = __lhs << __x; }
5212 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5214 operator>>=(_V& __lhs,
const _V& __x)
5215 {
return __lhs = __lhs >> __x; }
5217 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5219 operator<<=(_V& __lhs,
int __x)
5220 {
return __lhs = __lhs << __x; }
5222 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5224 operator>>=(_V& __lhs,
int __x)
5225 {
return __lhs = __lhs >> __x; }
5227 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5231 return _SimdIntOperators::_S_make_derived(
5232 _Impl::_S_modulus(__data(__x), __data(__y)));
5235 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5239 return _SimdIntOperators::_S_make_derived(
5240 _Impl::_S_bit_and(__data(__x), __data(__y)));
5243 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5247 return _SimdIntOperators::_S_make_derived(
5248 _Impl::_S_bit_or(__data(__x), __data(__y)));
5251 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5255 return _SimdIntOperators::_S_make_derived(
5256 _Impl::_S_bit_xor(__data(__x), __data(__y)));
5259 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5263 return _SimdIntOperators::_S_make_derived(
5264 _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
5267 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5271 return _SimdIntOperators::_S_make_derived(
5272 _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
5275 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5280 __invoke_ub(
"The behavior is undefined if the right operand of a "
5281 "shift operation is negative. [expr.shift]\nA shift by "
5284 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5286 "The behavior is undefined if the right operand of a "
5287 "shift operation is greater than or equal to the width of the "
5288 "promoted left operand. [expr.shift]\nA shift by %d was requested",
5290 return _SimdIntOperators::_S_make_derived(
5291 _Impl::_S_bit_shift_left(__data(__x), __y));
5294 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5300 "The behavior is undefined if the right operand of a shift "
5301 "operation is negative. [expr.shift]\nA shift by %d was requested",
5303 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5305 "The behavior is undefined if the right operand of a shift "
5306 "operation is greater than or equal to the width of the promoted "
5307 "left operand. [expr.shift]\nA shift by %d was requested",
5309 return _SimdIntOperators::_S_make_derived(
5310 _Impl::_S_bit_shift_right(__data(__x), __y));
5314 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5317 {
return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
5324template <
typename _Tp,
typename _Abi>
5325 class simd :
public _SimdIntOperators<
5326 simd<_Tp, _Abi>, _Tp, _Abi,
5327 conjunction<is_integral<_Tp>,
5328 typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
5329 public _SimdTraits<_Tp, _Abi>::_SimdBase
5331 using _Traits = _SimdTraits<_Tp, _Abi>;
5332 using _MemberType =
typename _Traits::_SimdMember;
5333 using _CastType =
typename _Traits::_SimdCastType;
5334 static constexpr _Tp* _S_type_tag =
nullptr;
5335 friend typename _Traits::_SimdBase;
5338 using _Impl =
typename _Traits::_SimdImpl;
5340 friend _SimdIntOperators<simd, _Tp, _Abi, true>;
5342 using value_type = _Tp;
5343 using reference = _SmartReference<_MemberType, _Impl, value_type>;
5344 using mask_type = simd_mask<_Tp, _Abi>;
5345 using abi_type = _Abi;
5347 static constexpr size_t size()
5348 {
return __size_or_zero_v<_Tp, _Abi>; }
5350 _GLIBCXX_SIMD_CONSTEXPR simd() =
default;
5351 _GLIBCXX_SIMD_CONSTEXPR simd(
const simd&) =
default;
5352 _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
5353 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
5354 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
5357 template <typename _Up,
5358 typename =
enable_if_t<!is_same_v<__remove_cvref_t<_Up>,
bool>>>
5359 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5360 simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
5362 _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
5367 template <
typename _Up>
5368 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5369 simd(
const simd<_Up, simd_abi::fixed_size<size()>>& __x,
5372 is_same<simd_abi::fixed_size<size()>, abi_type>,
5373 negation<__is_narrowing_conversion<_Up, value_type>>,
5374 __converts_to_higher_integer_rank<_Up, value_type>>::value,
5376 : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
5379#ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5380 template <
typename _Up,
typename _A2,
5381 typename =
decltype(static_simd_cast<simd>(
5382 declval<
const simd<_Up, _A2>&>()))>
5383 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5384 simd(
const simd<_Up, _A2>& __x)
5385 : simd(static_simd_cast<simd>(__x)) {}
5389 template <
typename _Fp>
5390 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5391 simd(_Fp&& __gen, _ValuePreservingOrInt<
decltype(declval<_Fp>()(
5392 declval<_SizeConstant<0>&>())),
5393 value_type>* =
nullptr)
5394 : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
5397 template <
typename _Up,
typename _Flags>
5398 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5399 simd(
const _Up* __mem, _IsSimdFlagType<_Flags>)
5401 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
5405 template <
typename _Up,
typename _Flags>
5406 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
void
5407 copy_from(
const _Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>)
5409 _M_data =
static_cast<decltype(_M_data)
>(
5410 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
5414 template <
typename _Up,
typename _Flags>
5415 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
void
5416 copy_to(_Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>)
const
5418 _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5423 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5424 operator[](
size_t __i)
5425 {
return {_M_data, int(__i)}; }
5427 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5428 operator[]([[maybe_unused]]
size_t __i)
const
5430 if constexpr (__is_scalar_abi<_Abi>())
5432 _GLIBCXX_DEBUG_ASSERT(__i == 0);
5436 return _M_data[__i];
5440 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5443 _Impl::_S_increment(_M_data);
5447 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5451 _Impl::_S_increment(_M_data);
5455 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5458 _Impl::_S_decrement(_M_data);
5462 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5466 _Impl::_S_decrement(_M_data);
5471 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5473 {
return {__private_init, _Impl::_S_negate(_M_data)}; }
5475 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5479 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5481 {
return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5484 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5485 simd(_CastType __init) : _M_data(__init) {}
5488 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5489 operator+=(simd& __lhs,
const simd& __x)
5490 {
return __lhs = __lhs + __x; }
5492 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5493 operator-=(simd& __lhs,
const simd& __x)
5494 {
return __lhs = __lhs - __x; }
5496 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5497 operator*=(simd& __lhs,
const simd& __x)
5498 {
return __lhs = __lhs * __x; }
5500 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5501 operator/=(simd& __lhs,
const simd& __x)
5502 {
return __lhs = __lhs / __x; }
5505 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5506 operator+(
const simd& __x,
const simd& __y)
5507 {
return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5509 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5510 operator-(
const simd& __x,
const simd& __y)
5511 {
return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5513 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5514 operator*(
const simd& __x,
const simd& __y)
5515 {
return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5517 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5518 operator/(
const simd& __x,
const simd& __y)
5519 {
return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5522 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5523 operator==(
const simd& __x,
const simd& __y)
5524 {
return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5526 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5527 operator!=(
const simd& __x,
const simd& __y)
5529 return simd::_S_make_mask(
5530 _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5533 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5534 operator<(
const simd& __x,
const simd& __y)
5535 {
return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5537 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5538 operator<=(
const simd& __x,
const simd& __y)
5540 return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5543 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5544 operator>(
const simd& __x,
const simd& __y)
5545 {
return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5547 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5548 operator>=(
const simd& __x,
const simd& __y)
5550 return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5554#ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5555 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5556 operator?:(
const mask_type& __k,
const simd& __where_true,
5557 const simd& __where_false)
5559 auto __ret = __where_false;
5560 _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5568 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5569 simd(_PrivateInit,
const _MemberType& __init)
5570 : _M_data(__init) {}
5573 _GLIBCXX_SIMD_INTRINSIC
5574 simd(_BitsetInit, bitset<size()> __init) : _M_data()
5575 { where(mask_type(__bitset_init, __init), *
this) = ~*
this; }
5577 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
5578 _M_is_constprop()
const
5580 if constexpr (__is_scalar_abi<_Abi>())
5581 return __builtin_constant_p(_M_data);
5583 return _M_data._M_is_constprop();
5587 _GLIBCXX_SIMD_INTRINSIC
static constexpr mask_type
5588 _S_make_mask(
typename mask_type::_MemberType __k)
5589 {
return {__private_init, __k}; }
5591 friend const auto& __data<value_type, abi_type>(
const simd&);
5592 friend auto& __data<value_type, abi_type>(simd&);
5593 alignas(_Traits::_S_simd_align) _MemberType _M_data;
5599template <
typename _Tp,
typename _Ap>
5600 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
5601 __data(
const simd<_Tp, _Ap>& __x)
5602 {
return __x._M_data; }
5604template <
typename _Tp,
typename _Ap>
5605 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
5606 __data(simd<_Tp, _Ap>& __x)
5607 {
return __x._M_data; }
5610namespace __float_bitwise_operators {
5611template <
typename _Tp,
typename _Ap>
5612 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5613 operator^(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5614 {
return {__private_init, _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))}; }
5616template <
typename _Tp,
typename _Ap>
5617 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5618 operator|(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5619 {
return {__private_init, _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))}; }
5621template <
typename _Tp,
typename _Ap>
5622 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5623 operator&(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5624 {
return {__private_init, _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))}; }
5626template <
typename _Tp,
typename _Ap>
5627 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5628 enable_if_t<is_floating_point_v<_Tp>, simd<_Tp, _Ap>>
5629 operator~(
const simd<_Tp, _Ap>& __a)
5630 {
return {__private_init, _Ap::_SimdImpl::_S_complement(__data(__a))}; }
5635_GLIBCXX_SIMD_END_NAMESPACE
constexpr _If_is_unsigned_integer< _Tp, int > popcount(_Tp __x) noexcept
The number of bits set in x.
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator%(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
constexpr complex< _Tp > operator*(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x times y.
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
constexpr complex< _Tp > operator/(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x divided by y.
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
void void_t
A metafunction that always yields void, used for detecting valid types.
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
typename remove_const< _Tp >::type remove_const_t
Alias template for remove_const.
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
Create a tuple containing all elements from multiple tuple-like objects.
auto declval() noexcept -> decltype(__declval< _Tp >(0))
constexpr tuple< typename __decay_and_strip< _Elements >::__type... > make_tuple(_Elements &&... __args)
Create a tuple containing copies of the arguments.
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
make_integer_sequence< size_t, _Num > make_index_sequence
Alias template make_index_sequence.
constexpr bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
constexpr bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
constexpr bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.