core/stdarch/crates/core_arch/src/s390x/
vector.rs

1//! s390x vector intrinsics.
2//!
3//! For more info see the [Reference Summary] or the online [IBM docs].
4//!
5//! [Reference Summary]: https://www.ibm.com/support/pages/sites/default/files/2021-05/SA22-7871-10.pdf
6//! [IBM docs]: https://www.ibm.com/docs/en/zos/2.4.0?topic=support-vector-built-in-functions
7
8#![allow(non_camel_case_types)]
9
10use crate::{core_arch::simd::*, intrinsics::simd::*, mem::MaybeUninit, mem::transmute};
11
12#[cfg(test)]
13use stdarch_test::assert_instr;
14
15use super::macros::*;
16
17types! {
18    #![unstable(feature = "stdarch_s390x", issue = "135681")]
19
20    /// s390x-specific 128-bit wide vector of sixteen packed `i8`
21    pub struct vector_signed_char(16 x i8);
22    /// s390x-specific 128-bit wide vector of sixteen packed `u8`
23    pub struct vector_unsigned_char(16 x u8);
24    /// s390x-specific 128-bit wide vector mask of sixteen packed elements
25    pub struct vector_bool_char(16 x i8);
26
27    /// s390x-specific 128-bit wide vector of eight packed `i16`
28    pub struct vector_signed_short(8 x i16);
29    /// s390x-specific 128-bit wide vector of eight packed `u16`
30    pub struct vector_unsigned_short(8 x u16);
31    /// s390x-specific 128-bit wide vector mask of eight packed elements
32    pub struct vector_bool_short(8 x i16);
33
34    /// s390x-specific 128-bit wide vector of four packed `i32`
35    pub struct vector_signed_int(4 x i32);
36    /// s390x-specific 128-bit wide vector of four packed `u32`
37    pub struct vector_unsigned_int(4 x u32);
38    /// s390x-specific 128-bit wide vector mask of four packed elements
39    pub struct vector_bool_int(4 x i32);
40
41    /// s390x-specific 128-bit wide vector of two packed `i64`
42    pub struct vector_signed_long_long(2 x i64);
43    /// s390x-specific 128-bit wide vector of two packed `u64`
44    pub struct vector_unsigned_long_long(2 x u64);
45    /// s390x-specific 128-bit wide vector mask of two packed elements
46    pub struct vector_bool_long_long(2 x i64);
47
48    /// s390x-specific 128-bit wide vector of four packed `f32`
49    pub struct vector_float(4 x f32);
50    /// s390x-specific 128-bit wide vector of two packed `f64`
51    pub struct vector_double(2 x f64);
52}
53
54#[repr(C, packed)]
55struct PackedTuple<T, U> {
56    x: T,
57    y: U,
58}
59
60#[allow(improper_ctypes)]
61#[rustfmt::skip]
62unsafe extern "unadjusted" {
63    #[link_name = "llvm.smax.v16i8"] fn vmxb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
64    #[link_name = "llvm.smax.v8i16"] fn vmxh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
65    #[link_name = "llvm.smax.v4i32"] fn vmxf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
66    #[link_name = "llvm.smax.v2i64"] fn vmxg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
67
68    #[link_name = "llvm.umax.v16i8"] fn vmxlb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
69    #[link_name = "llvm.umax.v8i16"] fn vmxlh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
70    #[link_name = "llvm.umax.v4i32"] fn vmxlf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
71    #[link_name = "llvm.umax.v2i64"] fn vmxlg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
72
73    #[link_name = "llvm.smin.v16i8"] fn vmnb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
74    #[link_name = "llvm.smin.v8i16"] fn vmnh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
75    #[link_name = "llvm.smin.v4i32"] fn vmnf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
76    #[link_name = "llvm.smin.v2i64"] fn vmng(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
77
78    #[link_name = "llvm.umin.v16i8"] fn vmnlb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
79    #[link_name = "llvm.umin.v8i16"] fn vmnlh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
80    #[link_name = "llvm.umin.v4i32"] fn vmnlf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
81    #[link_name = "llvm.umin.v2i64"] fn vmnlg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
82
83    #[link_name = "llvm.nearbyint.v4f32"] fn nearbyint_v4f32(a: vector_float) -> vector_float;
84    #[link_name = "llvm.nearbyint.v2f64"] fn nearbyint_v2f64(a: vector_double) -> vector_double;
85
86    #[link_name = "llvm.roundeven.v4f32"] fn roundeven_v4f32(a: vector_float) -> vector_float;
87    #[link_name = "llvm.roundeven.v2f64"] fn roundeven_v2f64(a: vector_double) -> vector_double;
88
89    #[link_name = "llvm.s390.vsra"] fn vsra(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
90    #[link_name = "llvm.s390.vsrl"] fn vsrl(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
91    #[link_name = "llvm.s390.vsl"] fn vsl(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
92
93    #[link_name = "llvm.s390.vsrab"] fn vsrab(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
94    #[link_name = "llvm.s390.vsrlb"] fn vsrlb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
95    #[link_name = "llvm.s390.vslb"] fn vslb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
96
97    #[link_name = "llvm.s390.vsrd"] fn vsrd(a: i8x16, b: i8x16, c: u32) -> i8x16;
98
99    #[link_name = "llvm.s390.verimb"] fn verimb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char, d: i32) -> vector_signed_char;
100    #[link_name = "llvm.s390.verimh"] fn verimh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short, d: i32) -> vector_signed_short;
101    #[link_name = "llvm.s390.verimf"] fn verimf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int, d: i32) -> vector_signed_int;
102    #[link_name = "llvm.s390.verimg"] fn verimg(a: vector_signed_long_long, b: vector_signed_long_long, c: vector_signed_long_long, d: i32) -> vector_signed_long_long;
103
104    #[link_name = "llvm.s390.vperm"] fn vperm(a: vector_signed_char, b: vector_signed_char, c: vector_unsigned_char) -> vector_signed_char;
105
106    #[link_name = "llvm.s390.vsumb"] fn vsumb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_int;
107    #[link_name = "llvm.s390.vsumh"] fn vsumh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
108
109    #[link_name = "llvm.s390.vsumgh"] fn vsumgh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_long_long;
110    #[link_name = "llvm.s390.vsumgf"] fn vsumgf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
111
112    #[link_name = "llvm.s390.vsumqf"] fn vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128;
113    #[link_name = "llvm.s390.vsumqg"] fn vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128;
114
115    #[link_name = "llvm.s390.vaccq"] fn vaccq(a: u128, b: u128) -> u128;
116    #[link_name = "llvm.s390.vacccq"] fn vacccq(a: u128, b: u128, c: u128) -> u128;
117
118    #[link_name = "llvm.s390.vscbiq"] fn vscbiq(a: u128, b: u128) -> u128;
119    #[link_name = "llvm.s390.vsbiq"] fn vsbiq(a: u128, b: u128, c: u128) -> u128;
120    #[link_name = "llvm.s390.vsbcbiq"] fn vsbcbiq(a: u128, b: u128, c: u128) -> u128;
121
122    #[link_name = "llvm.s390.vacq"] fn vacq(a: u128, b: u128, c: u128) -> u128;
123
124    #[link_name = "llvm.s390.vscbib"] fn vscbib(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
125    #[link_name = "llvm.s390.vscbih"] fn vscbih(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
126    #[link_name = "llvm.s390.vscbif"] fn vscbif(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
127    #[link_name = "llvm.s390.vscbig"] fn vscbig(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
128
129    #[link_name = "llvm.s390.vfaeb"] fn vfaeb(a: vector_signed_char, b: vector_signed_char, c: i32) -> vector_signed_char;
130    #[link_name = "llvm.s390.vfaeh"] fn vfaeh(a: vector_signed_short, b: vector_signed_short, c: i32) -> vector_signed_short;
131    #[link_name = "llvm.s390.vfaef"] fn vfaef(a: vector_signed_int, b: vector_signed_int, c: i32) -> vector_signed_int;
132
133    #[link_name = "llvm.s390.vfaezb"] fn vfaezb(a: vector_signed_char, b: vector_signed_char, c: i32) -> vector_signed_char;
134    #[link_name = "llvm.s390.vfaezh"] fn vfaezh(a: vector_signed_short, b: vector_signed_short, c: i32) -> vector_signed_short;
135    #[link_name = "llvm.s390.vfaezf"] fn vfaezf(a: vector_signed_int, b: vector_signed_int, c: i32) -> vector_signed_int;
136
137    #[link_name = "llvm.s390.vfaebs"] fn vfaebs(a: vector_signed_char, b: vector_signed_char, c: i32) -> PackedTuple<vector_signed_char, i32>;
138    #[link_name = "llvm.s390.vfaehs"] fn vfaehs(a: vector_signed_short, b: vector_signed_short, c: i32) -> PackedTuple<vector_signed_short, i32>;
139    #[link_name = "llvm.s390.vfaefs"] fn vfaefs(a: vector_signed_int, b: vector_signed_int, c: i32) -> PackedTuple<vector_signed_int, i32>;
140
141    #[link_name = "llvm.s390.vfaezbs"] fn vfaezbs(a: vector_signed_char, b: vector_signed_char, c: i32) -> PackedTuple<vector_signed_char, i32>;
142    #[link_name = "llvm.s390.vfaezhs"] fn vfaezhs(a: vector_signed_short, b: vector_signed_short, c: i32) -> PackedTuple<vector_signed_short, i32>;
143    #[link_name = "llvm.s390.vfaezfs"] fn vfaezfs(a: vector_signed_int, b: vector_signed_int, c: i32) -> PackedTuple<vector_signed_int, i32>;
144
145    #[link_name = "llvm.s390.vll"] fn vll(a: u32, b: *const u8) -> vector_signed_char;
146    #[link_name = "llvm.s390.vstl"] fn vstl(a: vector_signed_char, b: u32, c: *mut u8);
147
148    #[link_name = "llvm.s390.vlrl"] fn vlrl(a: u32, b: *const u8) -> vector_unsigned_char;
149    #[link_name = "llvm.s390.vstrl"] fn vstrl(a: vector_unsigned_char, b: u32, c: *mut u8);
150
151    #[link_name = "llvm.s390.lcbb"] fn lcbb(a: *const u8, b: u32) -> u32;
152    #[link_name = "llvm.s390.vlbb"] fn vlbb(a: *const u8, b: u32) -> MaybeUninit<vector_signed_char>;
153
154    #[link_name = "llvm.s390.vpksh"] fn vpksh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_char;
155    #[link_name = "llvm.s390.vpksf"] fn vpksf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_short;
156    #[link_name = "llvm.s390.vpksg"] fn vpksg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_int;
157
158    #[link_name = "llvm.s390.vpklsh"] fn vpklsh(a: vector_signed_short, b: vector_signed_short) -> vector_unsigned_char;
159    #[link_name = "llvm.s390.vpklsf"] fn vpklsf(a: vector_signed_int, b: vector_signed_int) -> vector_unsigned_short;
160    #[link_name = "llvm.s390.vpklsg"] fn vpklsg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_unsigned_int;
161
162    #[link_name = "llvm.s390.vpkshs"] fn vpkshs(a: vector_signed_short, b: vector_signed_short) -> PackedTuple<vector_signed_char, i32>;
163    #[link_name = "llvm.s390.vpksfs"] fn vpksfs(a: vector_signed_int, b: vector_signed_int) -> PackedTuple<vector_signed_short, i32>;
164    #[link_name = "llvm.s390.vpksgs"] fn vpksgs(a: vector_signed_long_long, b: vector_signed_long_long) -> PackedTuple<vector_signed_int, i32>;
165
166    #[link_name = "llvm.s390.vpklshs"] fn vpklshs(a: vector_unsigned_short, b: vector_unsigned_short) -> PackedTuple<vector_unsigned_char, i32>;
167    #[link_name = "llvm.s390.vpklsfs"] fn vpklsfs(a: vector_unsigned_int, b: vector_unsigned_int) -> PackedTuple<vector_unsigned_short, i32>;
168    #[link_name = "llvm.s390.vpklsgs"] fn vpklsgs(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> PackedTuple<vector_unsigned_int, i32>;
169
170    #[link_name = "llvm.s390.vavgb"] fn vavgb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
171    #[link_name = "llvm.s390.vavgh"] fn vavgh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
172    #[link_name = "llvm.s390.vavgf"] fn vavgf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
173    #[link_name = "llvm.s390.vavgg"] fn vavgg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
174
175    #[link_name = "llvm.s390.vavglb"] fn vavglb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
176    #[link_name = "llvm.s390.vavglh"] fn vavglh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
177    #[link_name = "llvm.s390.vavglf"] fn vavglf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
178    #[link_name = "llvm.s390.vavglg"] fn vavglg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
179
180    #[link_name = "llvm.s390.vcksm"] fn vcksm(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
181
182    #[link_name = "llvm.s390.vmhb"] fn vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
183    #[link_name = "llvm.s390.vmhh"] fn vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
184    #[link_name = "llvm.s390.vmhf"] fn vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
185
186    #[link_name = "llvm.s390.vmlhb"] fn vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
187    #[link_name = "llvm.s390.vmlhh"] fn vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
188    #[link_name = "llvm.s390.vmlhf"] fn vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
189
190    #[link_name = "llvm.s390.vmaeb"] fn vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short;
191    #[link_name = "llvm.s390.vmaeh"] fn vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int;
192    #[link_name = "llvm.s390.vmaef"] fn vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long;
193
194    #[link_name = "llvm.s390.vmaleb"] fn vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
195    #[link_name = "llvm.s390.vmaleh"] fn vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
196    #[link_name = "llvm.s390.vmalef"] fn vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
197
198    #[link_name = "llvm.s390.vmaob"] fn vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short;
199    #[link_name = "llvm.s390.vmaoh"] fn vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int;
200    #[link_name = "llvm.s390.vmaof"] fn vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long;
201
202    #[link_name = "llvm.s390.vmalob"] fn vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
203    #[link_name = "llvm.s390.vmaloh"] fn vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
204    #[link_name = "llvm.s390.vmalof"] fn vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
205
206    #[link_name = "llvm.s390.vmahb"] fn vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char;
207    #[link_name = "llvm.s390.vmahh"] fn vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short;
208    #[link_name = "llvm.s390.vmahf"] fn vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int;
209
210    #[link_name = "llvm.s390.vmalhb"] fn vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char;
211    #[link_name = "llvm.s390.vmalhh"] fn vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short;
212    #[link_name = "llvm.s390.vmalhf"] fn vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int;
213
214    #[link_name = "llvm.s390.vmalb"] fn vmalb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char;
215    #[link_name = "llvm.s390.vmalh"] fn vmalh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short;
216    #[link_name = "llvm.s390.vmalf"] fn vmalf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int;
217
218    #[link_name = "llvm.s390.vmallb"] fn vmallb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char;
219    #[link_name = "llvm.s390.vmallh"] fn vmallh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short;
220    #[link_name = "llvm.s390.vmallf"] fn vmallf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int;
221
222    #[link_name = "llvm.s390.vgfmb"] fn vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
223    #[link_name = "llvm.s390.vgfmh"] fn vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
224    #[link_name = "llvm.s390.vgfmf"] fn vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
225    #[link_name = "llvm.s390.vgfmg"] fn vgfmg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128;
226
227    #[link_name = "llvm.s390.vgfmab"] fn vgfmab(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
228    #[link_name = "llvm.s390.vgfmah"] fn vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
229    #[link_name = "llvm.s390.vgfmaf"] fn vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
230    #[link_name = "llvm.s390.vgfmag"] fn vgfmag(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128) -> u128;
231
232    #[link_name = "llvm.s390.vbperm"] fn vbperm(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_long_long;
233
234    #[link_name = "llvm.s390.vftcisb"] fn vftcisb(a: vector_float, b: u32) -> PackedTuple<vector_bool_int, i32>;
235    #[link_name = "llvm.s390.vftcidb"] fn vftcidb(a: vector_double, b: u32) -> PackedTuple<vector_bool_long_long, i32>;
236
237    #[link_name = "llvm.s390.vtm"] fn vtm(a: i8x16, b: i8x16) -> i32;
238
239    #[link_name = "llvm.s390.vstrsb"] fn vstrsb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
240    #[link_name = "llvm.s390.vstrsh"] fn vstrsh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
241    #[link_name = "llvm.s390.vstrsf"] fn vstrsf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
242
243    #[link_name = "llvm.s390.vstrszb"] fn vstrszb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
244    #[link_name = "llvm.s390.vstrszh"] fn vstrszh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
245    #[link_name = "llvm.s390.vstrszf"] fn vstrszf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
246
247    #[link_name = "llvm.s390.vistrb"] fn vistrb(a: vector_unsigned_char) -> vector_unsigned_char;
248    #[link_name = "llvm.s390.vistrh"] fn vistrh(a: vector_unsigned_short) -> vector_unsigned_short;
249    #[link_name = "llvm.s390.vistrf"] fn vistrf(a: vector_unsigned_int) -> vector_unsigned_int;
250
251    #[link_name = "llvm.s390.vistrbs"] fn vistrbs(a: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
252    #[link_name = "llvm.s390.vistrhs"] fn vistrhs(a: vector_unsigned_short) -> PackedTuple<vector_unsigned_short, i32>;
253    #[link_name = "llvm.s390.vistrfs"] fn vistrfs(a: vector_unsigned_int) -> PackedTuple<vector_unsigned_int, i32>;
254
255    #[link_name = "llvm.s390.vmslg"] fn vmslg(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128, d: u32) -> u128;
256
257    #[link_name = "llvm.s390.vstrcb"] fn vstrcb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char;
258    #[link_name = "llvm.s390.vstrch"] fn vstrch(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short;
259    #[link_name = "llvm.s390.vstrcf"] fn vstrcf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int;
260
261    #[link_name = "llvm.s390.vstrcbs"] fn vstrcbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple<vector_bool_char, i32>;
262    #[link_name = "llvm.s390.vstrchs"] fn vstrchs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple<vector_bool_short, i32>;
263    #[link_name = "llvm.s390.vstrcfs"] fn vstrcfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple<vector_bool_int, i32>;
264
265    #[link_name = "llvm.s390.vstrczb"] fn vstrczb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char;
266    #[link_name = "llvm.s390.vstrczh"] fn vstrczh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short;
267    #[link_name = "llvm.s390.vstrczf"] fn vstrczf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int;
268
269    #[link_name = "llvm.s390.vstrczbs"] fn vstrczbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple<vector_bool_char, i32>;
270    #[link_name = "llvm.s390.vstrczhs"] fn vstrczhs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple<vector_bool_short, i32>;
271    #[link_name = "llvm.s390.vstrczfs"] fn vstrczfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple<vector_bool_int, i32>;
272
273    #[link_name = "llvm.s390.vfeeb"] fn vfeeb(a: i8x16, b: i8x16) -> i8x16;
274    #[link_name = "llvm.s390.vfeeh"] fn vfeeh(a: i16x8, b: i16x8) -> i16x8;
275    #[link_name = "llvm.s390.vfeef"] fn vfeef(a: i32x4, b: i32x4) -> i32x4;
276
277    #[link_name = "llvm.s390.vfeezb"] fn vfeezb(a: i8x16, b: i8x16) -> i8x16;
278    #[link_name = "llvm.s390.vfeezh"] fn vfeezh(a: i16x8, b: i16x8) -> i16x8;
279    #[link_name = "llvm.s390.vfeezf"] fn vfeezf(a: i32x4, b: i32x4) -> i32x4;
280
281    #[link_name = "llvm.s390.vfeebs"] fn vfeebs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
282    #[link_name = "llvm.s390.vfeehs"] fn vfeehs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
283    #[link_name = "llvm.s390.vfeefs"] fn vfeefs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
284
285    #[link_name = "llvm.s390.vfeezbs"] fn vfeezbs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
286    #[link_name = "llvm.s390.vfeezhs"] fn vfeezhs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
287    #[link_name = "llvm.s390.vfeezfs"] fn vfeezfs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
288
289    #[link_name = "llvm.s390.vfeneb"] fn vfeneb(a: i8x16, b: i8x16) -> i8x16;
290    #[link_name = "llvm.s390.vfeneh"] fn vfeneh(a: i16x8, b: i16x8) -> i16x8;
291    #[link_name = "llvm.s390.vfenef"] fn vfenef(a: i32x4, b: i32x4) -> i32x4;
292
293    #[link_name = "llvm.s390.vfenezb"] fn vfenezb(a: i8x16, b: i8x16) -> i8x16;
294    #[link_name = "llvm.s390.vfenezh"] fn vfenezh(a: i16x8, b: i16x8) -> i16x8;
295    #[link_name = "llvm.s390.vfenezf"] fn vfenezf(a: i32x4, b: i32x4) -> i32x4;
296
297    #[link_name = "llvm.s390.vfenebs"] fn vfenebs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
298    #[link_name = "llvm.s390.vfenehs"] fn vfenehs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
299    #[link_name = "llvm.s390.vfenefs"] fn vfenefs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
300
301    #[link_name = "llvm.s390.vfenezbs"] fn vfenezbs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
302    #[link_name = "llvm.s390.vfenezhs"] fn vfenezhs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
303    #[link_name = "llvm.s390.vfenezfs"] fn vfenezfs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
304}
305
306impl_from! { i8x16, u8x16,  i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 }
307
308impl_neg! { i8x16 : 0 }
309impl_neg! { i16x8 : 0 }
310impl_neg! { i32x4 : 0 }
311impl_neg! { i64x2 : 0 }
312impl_neg! { f32x4 : 0f32 }
313impl_neg! { f64x2 : 0f64 }
314
315#[repr(simd)]
316struct ShuffleMask<const N: usize>([u32; N]);
317
318impl<const N: usize> ShuffleMask<N> {
319    const fn reverse() -> Self {
320        let mut index = [0; N];
321        let mut i = 0;
322        while i < N {
323            index[i] = (N - i - 1) as u32;
324            i += 1;
325        }
326        ShuffleMask(index)
327    }
328
329    const fn merge_low() -> Self {
330        let mut mask = [0; N];
331        let mut i = N / 2;
332        let mut index = 0;
333        while index < N {
334            mask[index] = i as u32;
335            mask[index + 1] = (i + N) as u32;
336
337            i += 1;
338            index += 2;
339        }
340        ShuffleMask(mask)
341    }
342
343    const fn merge_high() -> Self {
344        let mut mask = [0; N];
345        let mut i = 0;
346        let mut index = 0;
347        while index < N {
348            mask[index] = i as u32;
349            mask[index + 1] = (i + N) as u32;
350
351            i += 1;
352            index += 2;
353        }
354        ShuffleMask(mask)
355    }
356
357    const fn even() -> Self {
358        let mut mask = [0; N];
359        let mut i = 0;
360        let mut index = 0;
361        while index < N {
362            mask[index] = i as u32;
363
364            i += 2;
365            index += 1;
366        }
367        ShuffleMask(mask)
368    }
369
370    const fn odd() -> Self {
371        let mut mask = [0; N];
372        let mut i = 1;
373        let mut index = 0;
374        while index < N {
375            mask[index] = i as u32;
376
377            i += 2;
378            index += 1;
379        }
380        ShuffleMask(mask)
381    }
382
383    const fn pack() -> Self {
384        Self::odd()
385    }
386
387    const fn unpack_low() -> Self {
388        let mut mask = [0; N];
389        let mut i = 0;
390        while i < N {
391            mask[i] = (N + i) as u32;
392            i += 1;
393        }
394        ShuffleMask(mask)
395    }
396
397    const fn unpack_high() -> Self {
398        let mut mask = [0; N];
399        let mut i = 0;
400        while i < N {
401            mask[i] = i as u32;
402            i += 1;
403        }
404        ShuffleMask(mask)
405    }
406}
407
408const fn genmask<const MASK: u16>() -> [u8; 16] {
409    let mut bits = MASK;
410    let mut elements = [0u8; 16];
411
412    let mut i = 0;
413    while i < 16 {
414        elements[i] = match bits & (1u16 << 15) {
415            0 => 0,
416            _ => 0xFF,
417        };
418
419        bits <<= 1;
420        i += 1;
421    }
422
423    elements
424}
425
426const fn genmasks(bit_width: u32, a: u8, b: u8) -> u64 {
427    let bit_width = bit_width as u8;
428    let a = a % bit_width;
429    let mut b = b % bit_width;
430    if a > b {
431        b = bit_width - 1;
432    }
433
434    // of course these indices start from the left
435    let a = (bit_width - 1) - a;
436    let b = (bit_width - 1) - b;
437
438    ((1u64.wrapping_shl(a as u32 + 1)) - 1) & !((1u64.wrapping_shl(b as u32)) - 1)
439}
440
441const fn validate_block_boundary(block_boundary: u16) -> u32 {
442    assert!(
443        block_boundary.is_power_of_two() && block_boundary >= 64 && block_boundary <= 4096,
444        "block boundary must be a constant power of 2 from 64 to 4096",
445    );
446
447    // so that 64 is encoded as 0, 128 as 1, ect.
448    block_boundary as u32 >> 7
449}
450
451enum FindImm {
452    Eq = 4,
453    Ne = 12,
454    EqIdx = 0,
455    NeIdx = 8,
456}
457
458#[macro_use]
459mod sealed {
460    use super::*;
461
462    #[unstable(feature = "stdarch_s390x", issue = "135681")]
463    pub trait VectorAdd<Other> {
464        type Result;
465        unsafe fn vec_add(self, other: Other) -> Self::Result;
466    }
467
468    macro_rules! impl_add {
469        ($name:ident, $a:ty, $instr:ident) => {
470            impl_add!($name, $a, $a, $a, $instr);
471        };
472        ($name:ident, $a:ty, $b:ty, $c:ty, $instr:ident) => {
473            #[inline]
474            #[target_feature(enable = "vector")]
475            #[cfg_attr(test, assert_instr($instr))]
476            pub unsafe fn $name(a: $a, b: $b) -> $c {
477                transmute(simd_add(transmute(a), b))
478            }
479
480            #[unstable(feature = "stdarch_s390x", issue = "135681")]
481            impl VectorAdd<$b> for $a {
482                type Result = $c;
483
484                #[inline]
485                #[target_feature(enable = "vector")]
486                unsafe fn vec_add(self, other: $b) -> Self::Result {
487                    $name(self, other)
488                }
489            }
490        };
491    }
492
493    #[rustfmt::skip]
494    mod impl_add {
495        use super::*;
496
497        impl_add!(va_sc, vector_signed_char, vab);
498        impl_add!(va_uc, vector_unsigned_char, vab);
499        impl_add!(va_sh, vector_signed_short, vah);
500        impl_add!(va_uh, vector_unsigned_short, vah);
501        impl_add!(va_sf, vector_signed_int, vaf);
502        impl_add!(va_uf, vector_unsigned_int, vaf);
503        impl_add!(va_sg, vector_signed_long_long, vag);
504        impl_add!(va_ug, vector_unsigned_long_long, vag);
505
506        impl_add!(va_sc_bc, vector_signed_char, vector_bool_char, vector_signed_char, vab);
507        impl_add!(va_uc_bc, vector_unsigned_char, vector_bool_char, vector_unsigned_char, vab);
508        impl_add!(va_sh_bh, vector_signed_short, vector_bool_short, vector_signed_short, vah);
509        impl_add!(va_uh_bh, vector_unsigned_short, vector_bool_short, vector_unsigned_short, vah);
510        impl_add!(va_sf_bf, vector_signed_int, vector_bool_int, vector_signed_int, vaf);
511        impl_add!(va_uf_bf, vector_unsigned_int, vector_bool_int, vector_unsigned_int, vaf);
512        impl_add!(va_sg_bg, vector_signed_long_long, vector_bool_long_long, vector_signed_long_long, vag);
513        impl_add!(va_ug_bg, vector_unsigned_long_long, vector_bool_long_long, vector_unsigned_long_long, vag);
514
515        impl_add!(va_bc_sc, vector_bool_char, vector_signed_char, vector_signed_char, vab);
516        impl_add!(va_bc_uc, vector_bool_char, vector_unsigned_char, vector_unsigned_char, vab);
517        impl_add!(va_bh_sh, vector_bool_short, vector_signed_short, vector_signed_short, vah);
518        impl_add!(va_bh_uh, vector_bool_short, vector_unsigned_short, vector_unsigned_short, vah);
519        impl_add!(va_bf_sf, vector_bool_int, vector_signed_int, vector_signed_int, vaf);
520        impl_add!(va_bf_uf, vector_bool_int, vector_unsigned_int, vector_unsigned_int, vaf);
521        impl_add!(va_bg_sg, vector_bool_long_long, vector_signed_long_long, vector_signed_long_long, vag);
522        impl_add!(va_bg_ug, vector_bool_long_long, vector_unsigned_long_long, vector_unsigned_long_long, vag);
523
524        impl_add!(va_double, vector_double, vfadb);
525
526        #[inline]
527        #[target_feature(enable = "vector")]
528        #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfasb))]
529        pub unsafe fn va_float(a: vector_float, b: vector_float) -> vector_float {
530            transmute(simd_add(a, b))
531        }
532
533        #[unstable(feature = "stdarch_s390x", issue = "135681")]
534        impl VectorAdd<Self> for vector_float {
535            type Result = Self;
536
537            #[inline]
538            #[target_feature(enable = "vector")]
539            unsafe fn vec_add(self, other: Self) -> Self::Result {
540                va_float(self, other)
541            }
542        }
543    }
544
545    #[unstable(feature = "stdarch_s390x", issue = "135681")]
546    pub trait VectorSub<Other> {
547        type Result;
548        unsafe fn vec_sub(self, other: Other) -> Self::Result;
549    }
550
551    macro_rules! impl_sub {
552        ($name:ident, $a:ty, $instr:ident) => {
553            impl_sub!($name, $a, $a, $a, $instr);
554        };
555        ($name:ident, $a:ty, $b:ty, $c:ty, $instr:ident) => {
556            #[inline]
557            #[target_feature(enable = "vector")]
558            #[cfg_attr(test, assert_instr($instr))]
559            pub unsafe fn $name(a: $a, b: $b) -> $c {
560                transmute(simd_sub(transmute(a), b))
561            }
562
563            #[unstable(feature = "stdarch_s390x", issue = "135681")]
564            impl VectorSub<$b> for $a {
565                type Result = $c;
566
567                #[inline]
568                #[target_feature(enable = "vector")]
569                unsafe fn vec_sub(self, other: $b) -> Self::Result {
570                    $name(self, other)
571                }
572            }
573        };
574    }
575
576    #[rustfmt::skip]
577    mod impl_sub {
578        use super::*;
579
580        impl_sub!(vs_sc, vector_signed_char, vsb);
581        impl_sub!(vs_uc, vector_unsigned_char, vsb);
582        impl_sub!(vs_sh, vector_signed_short, vsh);
583        impl_sub!(vs_uh, vector_unsigned_short, vsh);
584        impl_sub!(vs_sf, vector_signed_int, vsf);
585        impl_sub!(vs_uf, vector_unsigned_int, vsf);
586        impl_sub!(vs_sg, vector_signed_long_long, vsg);
587        impl_sub!(vs_ug, vector_unsigned_long_long, vsg);
588
589        impl_sub!(vs_sc_bc, vector_signed_char, vector_bool_char, vector_signed_char, vsb);
590        impl_sub!(vs_uc_bc, vector_unsigned_char, vector_bool_char, vector_unsigned_char, vsb);
591        impl_sub!(vs_sh_bh, vector_signed_short, vector_bool_short, vector_signed_short, vsh);
592        impl_sub!(vs_uh_bh, vector_unsigned_short, vector_bool_short, vector_unsigned_short, vsh);
593        impl_sub!(vs_sf_bf, vector_signed_int, vector_bool_int, vector_signed_int, vsf);
594        impl_sub!(vs_uf_bf, vector_unsigned_int, vector_bool_int, vector_unsigned_int, vsf);
595        impl_sub!(vs_sg_bg, vector_signed_long_long, vector_bool_long_long, vector_signed_long_long, vsg);
596        impl_sub!(vs_ug_bg, vector_unsigned_long_long, vector_bool_long_long, vector_unsigned_long_long, vsg);
597
598        impl_sub!(vs_bc_sc, vector_bool_char, vector_signed_char, vector_signed_char, vsb);
599        impl_sub!(vs_bc_uc, vector_bool_char, vector_unsigned_char, vector_unsigned_char, vsb);
600        impl_sub!(vs_bh_sh, vector_bool_short, vector_signed_short, vector_signed_short, vsh);
601        impl_sub!(vs_bh_uh, vector_bool_short, vector_unsigned_short, vector_unsigned_short, vsh);
602        impl_sub!(vs_bf_sf, vector_bool_int, vector_signed_int, vector_signed_int, vsf);
603        impl_sub!(vs_bf_uf, vector_bool_int, vector_unsigned_int, vector_unsigned_int, vsf);
604        impl_sub!(vs_bg_sg, vector_bool_long_long, vector_signed_long_long, vector_signed_long_long, vsg);
605        impl_sub!(vs_bg_ug, vector_bool_long_long, vector_unsigned_long_long, vector_unsigned_long_long, vsg);
606
607        impl_sub!(vs_double, vector_double, vfsdb);
608
609        #[inline]
610        #[target_feature(enable = "vector")]
611        #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfssb))]
612        pub unsafe fn vs_float(a: vector_float, b: vector_float) -> vector_float {
613            transmute(simd_sub(a, b))
614        }
615
616        #[unstable(feature = "stdarch_s390x", issue = "135681")]
617        impl VectorSub<Self> for vector_float {
618            type Result = Self;
619
620            #[inline]
621            #[target_feature(enable = "vector")]
622            unsafe fn vec_sub(self, other: Self) -> Self::Result {
623                vs_float(self, other)
624            }
625        }
626    }
627
628    #[unstable(feature = "stdarch_s390x", issue = "135681")]
629    pub trait VectorMul {
630        unsafe fn vec_mul(self, b: Self) -> Self;
631    }
632
633    macro_rules! impl_mul {
634        ($name:ident, $a:ty, std_simd) => {
635            #[unstable(feature = "stdarch_s390x", issue = "135681")]
636            impl VectorMul for $a {
637                #[inline]
638                #[target_feature(enable = "vector")]
639                unsafe fn vec_mul(self, other: Self) -> Self {
640                    transmute(simd_mul(transmute(self), other))
641                }
642            }
643        };
644        ($name:ident, $a:ty, $instr:ident) => {
645            #[inline]
646            #[target_feature(enable = "vector")]
647            #[cfg_attr(test, assert_instr($instr))]
648            pub unsafe fn $name(a: $a, b: $a) -> $a {
649                transmute(simd_mul(transmute(a), b))
650            }
651
652            #[unstable(feature = "stdarch_s390x", issue = "135681")]
653            impl VectorMul for $a {
654                #[inline]
655                #[target_feature(enable = "vector")]
656                unsafe fn vec_mul(self, other: Self) -> Self {
657                    $name(self, other)
658                }
659            }
660        };
661    }
662
663    #[rustfmt::skip]
664    mod impl_mul {
665        use super::*;
666
667        impl_mul!(vml_sc, vector_signed_char, vmlb);
668        impl_mul!(vml_uc, vector_unsigned_char, vmlb);
669        impl_mul!(vml_sh, vector_signed_short, vmlhw);
670        impl_mul!(vml_uh, vector_unsigned_short, vmlhw);
671        impl_mul!(vml_sf, vector_signed_int, vmlf);
672        impl_mul!(vml_uf, vector_unsigned_int, vmlf);
673        impl_mul!(vml_sg, vector_signed_long_long, std_simd);
674        impl_mul!(vml_ug, vector_unsigned_long_long, std_simd);
675
676        impl_mul!(vml_float, vector_float, std_simd);
677        impl_mul!(vml_double, vector_double, vfmdb);
678    }
679
680    #[unstable(feature = "stdarch_s390x", issue = "135681")]
681    pub trait VectorMax<Other> {
682        type Result;
683        unsafe fn vec_max(self, b: Other) -> Self::Result;
684    }
685
686    test_impl! { vec_vmxsb (a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [vmxb, vmxb] }
687    test_impl! { vec_vmxsh (a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [vmxh, vmxh] }
688    test_impl! { vec_vmxsf (a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [vmxf, vmxf] }
689    test_impl! { vec_vmxsg (a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [vmxg, vmxg] }
690
691    test_impl! { vec_vmxslb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vmxlb, vmxlb] }
692    test_impl! { vec_vmxslh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vmxlh, vmxlh] }
693    test_impl! { vec_vmxslf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vmxlf, vmxlf] }
694    test_impl! { vec_vmxslg (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vmxlg, vmxlg] }
695
696    impl_vec_trait! { [VectorMax vec_max] ~(vmxlb, vmxb, vmxlh, vmxh, vmxlf, vmxf, vmxlg, vmxg) }
697
698    test_impl! { vec_vfmaxsb (a: vector_float, b: vector_float) -> vector_float [simd_fmax, "vector-enhancements-1" vfmaxsb ] }
699    test_impl! { vec_vfmaxdb (a: vector_double, b: vector_double) -> vector_double [simd_fmax, "vector-enhancements-1" vfmaxdb] }
700
701    impl_vec_trait!([VectorMax vec_max] vec_vfmaxsb (vector_float, vector_float) -> vector_float);
702    impl_vec_trait!([VectorMax vec_max] vec_vfmaxdb (vector_double, vector_double) -> vector_double);
703
704    #[unstable(feature = "stdarch_s390x", issue = "135681")]
705    pub trait VectorMin<Other> {
706        type Result;
707        unsafe fn vec_min(self, b: Other) -> Self::Result;
708    }
709
710    test_impl! { vec_vmnsb (a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [vmnb, vmnb] }
711    test_impl! { vec_vmnsh (a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [vmnh, vmnh] }
712    test_impl! { vec_vmnsf (a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [vmnf, vmnf] }
713    test_impl! { vec_vmnsg (a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [vmng, vmng] }
714
715    test_impl! { vec_vmnslb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vmnlb, vmnlb] }
716    test_impl! { vec_vmnslh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vmnlh, vmnlh] }
717    test_impl! { vec_vmnslf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vmnlf, vmnlf] }
718    test_impl! { vec_vmnslg (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vmnlg, vmnlg] }
719
720    impl_vec_trait! { [VectorMin vec_min] ~(vmxlb, vmxb, vmxlh, vmxh, vmxlf, vmxf, vmxlg, vmxg) }
721
722    test_impl! { vec_vfminsb (a: vector_float, b: vector_float) -> vector_float [simd_fmin, "vector-enhancements-1" vfminsb]  }
723    test_impl! { vec_vfmindb (a: vector_double, b: vector_double) -> vector_double [simd_fmin, "vector-enhancements-1" vfmindb]  }
724
725    impl_vec_trait!([VectorMin vec_min] vec_vfminsb (vector_float, vector_float) -> vector_float);
726    impl_vec_trait!([VectorMin vec_min] vec_vfmindb (vector_double, vector_double) -> vector_double);
727
728    #[unstable(feature = "stdarch_s390x", issue = "135681")]
729    pub trait VectorAbs {
730        unsafe fn vec_abs(self) -> Self;
731    }
732
733    macro_rules! impl_abs {
734        ($name:ident, $ty:ident) => {
735            #[inline]
736            #[target_feature(enable = "vector")]
737            unsafe fn $name(v: s_t_l!($ty)) -> s_t_l!($ty) {
738                v.vec_max(-v)
739            }
740
741            impl_vec_trait! { [VectorAbs vec_abs] $name (s_t_l!($ty)) }
742        };
743    }
744
745    impl_abs! { vec_abs_i8, i8x16 }
746    impl_abs! { vec_abs_i16, i16x8 }
747    impl_abs! { vec_abs_i32, i32x4 }
748    impl_abs! { vec_abs_i64, i64x2 }
749
750    test_impl! { vec_abs_f32 (v: vector_float) -> vector_float [ simd_fabs, "vector-enhancements-1" vflpsb ] }
751    test_impl! { vec_abs_f64 (v: vector_double) -> vector_double [ simd_fabs, vflpdb ] }
752
753    impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f32 (vector_float) }
754    impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f64 (vector_double) }
755
756    #[unstable(feature = "stdarch_s390x", issue = "135681")]
757    pub trait VectorNabs {
758        unsafe fn vec_nabs(self) -> Self;
759    }
760
761    #[inline]
762    #[target_feature(enable = "vector")]
763    #[cfg_attr(
764        all(test, target_feature = "vector-enhancements-1"),
765        assert_instr(vflnsb)
766    )]
767    unsafe fn vec_nabs_f32(a: vector_float) -> vector_float {
768        simd_neg(simd_fabs(a))
769    }
770
771    #[inline]
772    #[target_feature(enable = "vector")]
773    #[cfg_attr(test, assert_instr(vflndb))]
774    unsafe fn vec_nabs_f64(a: vector_double) -> vector_double {
775        simd_neg(simd_fabs(a))
776    }
777
778    impl_vec_trait! { [VectorNabs vec_nabs] vec_nabs_f32 (vector_float) }
779    impl_vec_trait! { [VectorNabs vec_nabs] vec_nabs_f64 (vector_double) }
780
781    #[unstable(feature = "stdarch_s390x", issue = "135681")]
782    pub trait VectorNmsub {
783        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self;
784    }
785
786    #[inline]
787    #[target_feature(enable = "vector")]
788    #[cfg_attr(
789        all(test, target_feature = "vector-enhancements-2"),
790        assert_instr(vfnmssb)
791    )]
792    unsafe fn vec_nmsub_f32(a: vector_float, b: vector_float, c: vector_float) -> vector_float {
793        simd_neg(simd_fma(a, b, simd_neg(c)))
794    }
795
796    #[unstable(feature = "stdarch_s390x", issue = "135681")]
797    impl VectorNmsub for vector_float {
798        #[target_feature(enable = "vector")]
799        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self {
800            vec_nmsub_f32(self, b, c)
801        }
802    }
803
804    #[inline]
805    #[target_feature(enable = "vector")]
806    #[cfg_attr(
807        all(test, target_feature = "vector-enhancements-2"),
808        assert_instr(vfnmsdb)
809    )]
810    unsafe fn vec_nmsub_f64(a: vector_double, b: vector_double, c: vector_double) -> vector_double {
811        simd_neg(simd_fma(a, b, simd_neg(c)))
812    }
813
814    #[unstable(feature = "stdarch_s390x", issue = "135681")]
815    impl VectorNmsub for vector_double {
816        #[target_feature(enable = "vector")]
817        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self {
818            vec_nmsub_f64(self, b, c)
819        }
820    }
821
822    #[unstable(feature = "stdarch_s390x", issue = "135681")]
823    pub trait VectorNmadd {
824        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self;
825    }
826
827    #[inline]
828    #[target_feature(enable = "vector")]
829    #[cfg_attr(
830        all(test, target_feature = "vector-enhancements-2"),
831        assert_instr(vfnmasb)
832    )]
833    unsafe fn vec_nmadd_f32(a: vector_float, b: vector_float, c: vector_float) -> vector_float {
834        simd_neg(simd_fma(a, b, c))
835    }
836
837    #[unstable(feature = "stdarch_s390x", issue = "135681")]
838    impl VectorNmadd for vector_float {
839        #[target_feature(enable = "vector")]
840        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self {
841            vec_nmadd_f32(self, b, c)
842        }
843    }
844
845    #[inline]
846    #[target_feature(enable = "vector")]
847    #[cfg_attr(
848        all(test, target_feature = "vector-enhancements-2"),
849        assert_instr(vfnmadb)
850    )]
851    unsafe fn vec_nmadd_f64(a: vector_double, b: vector_double, c: vector_double) -> vector_double {
852        simd_neg(simd_fma(a, b, c))
853    }
854
855    #[unstable(feature = "stdarch_s390x", issue = "135681")]
856    impl VectorNmadd for vector_double {
857        #[target_feature(enable = "vector")]
858        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self {
859            vec_nmadd_f64(self, b, c)
860        }
861    }
862
863    #[unstable(feature = "stdarch_s390x", issue = "135681")]
864    pub trait VectorSplat {
865        unsafe fn vec_splat<const IMM: u32>(self) -> Self;
866    }
867
868    #[inline]
869    #[target_feature(enable = "vector")]
870    #[cfg_attr(test, assert_instr(vrepb, IMM2 = 1))]
871    unsafe fn vrepb<const IMM2: u32>(a: vector_signed_char) -> vector_signed_char {
872        static_assert_uimm_bits!(IMM2, 4);
873        simd_shuffle(a, a, const { u32x16::from_array([IMM2; 16]) })
874    }
875
876    #[inline]
877    #[target_feature(enable = "vector")]
878    #[cfg_attr(test, assert_instr(vreph, IMM2 = 1))]
879    unsafe fn vreph<const IMM2: u32>(a: vector_signed_short) -> vector_signed_short {
880        static_assert_uimm_bits!(IMM2, 3);
881        simd_shuffle(a, a, const { u32x8::from_array([IMM2; 8]) })
882    }
883
884    #[inline]
885    #[target_feature(enable = "vector")]
886    #[cfg_attr(test, assert_instr(vrepf, IMM2 = 1))]
887    unsafe fn vrepf<const IMM2: u32>(a: vector_signed_int) -> vector_signed_int {
888        static_assert_uimm_bits!(IMM2, 2);
889        simd_shuffle(a, a, const { u32x4::from_array([IMM2; 4]) })
890    }
891
892    #[inline]
893    #[target_feature(enable = "vector")]
894    #[cfg_attr(test, assert_instr(vrepg, IMM2 = 1))]
895    unsafe fn vrepg<const IMM2: u32>(a: vector_signed_long_long) -> vector_signed_long_long {
896        static_assert_uimm_bits!(IMM2, 1);
897        simd_shuffle(a, a, const { u32x2::from_array([IMM2; 2]) })
898    }
899
900    macro_rules! impl_vec_splat {
901        ($ty:ty, $fun:ident) => {
902            #[unstable(feature = "stdarch_s390x", issue = "135681")]
903            impl VectorSplat for $ty {
904                #[inline]
905                #[target_feature(enable = "vector")]
906                unsafe fn vec_splat<const IMM: u32>(self) -> Self {
907                    transmute($fun::<IMM>(transmute(self)))
908                }
909            }
910        };
911    }
912
913    impl_vec_splat! { vector_signed_char, vrepb }
914    impl_vec_splat! { vector_unsigned_char, vrepb }
915    impl_vec_splat! { vector_bool_char, vrepb }
916    impl_vec_splat! { vector_signed_short, vreph }
917    impl_vec_splat! { vector_unsigned_short, vreph }
918    impl_vec_splat! { vector_bool_short, vreph }
919    impl_vec_splat! { vector_signed_int, vrepf }
920    impl_vec_splat! { vector_unsigned_int, vrepf }
921    impl_vec_splat! { vector_bool_int, vrepf }
922    impl_vec_splat! { vector_signed_long_long, vrepg }
923    impl_vec_splat! { vector_unsigned_long_long, vrepg }
924    impl_vec_splat! { vector_bool_long_long, vrepg }
925
926    impl_vec_splat! { vector_float, vrepf }
927    impl_vec_splat! { vector_double, vrepg }
928
929    #[unstable(feature = "stdarch_s390x", issue = "135681")]
930    pub trait VectorSplats<Output> {
931        unsafe fn vec_splats(self) -> Output;
932    }
933
934    macro_rules! impl_vec_splats {
935        ($(($fn:ident ($ty:ty, $shortty:tt) $instr:ident)),*) => {
936            $(
937                #[inline]
938                #[target_feature(enable = "vector")]
939                #[cfg_attr(test, assert_instr($instr))]
940                pub unsafe fn $fn(v: $ty) -> s_t_l!($shortty) {
941                    transmute($shortty::splat(v))
942                }
943
944                #[unstable(feature = "stdarch_s390x", issue = "135681")]
945                impl VectorSplats<s_t_l!($shortty)> for $ty {
946                    #[inline]
947                    #[target_feature(enable = "vector")]
948                    unsafe fn vec_splats(self) -> s_t_l!($shortty) {
949                        $fn (self)
950                    }
951                }
952            )*
953        }
954    }
955
956    impl_vec_splats! {
957        (vec_splats_u8 (u8, u8x16) vrepb),
958        (vec_splats_i8 (i8, i8x16) vrepb),
959        (vec_splats_u16 (u16, u16x8) vreph),
960        (vec_splats_i16 (i16, i16x8) vreph),
961        (vec_splats_u32 (u32, u32x4) vrepf),
962        (vec_splats_i32 (i32, i32x4) vrepf),
963        (vec_splats_u64 (u64, u64x2) vlvgp),
964        (vec_splats_i64 (i64, i64x2) vlvgp),
965        (vec_splats_f32 (f32, f32x4) vrepf),
966        (vec_splats_f64 (f64, f64x2) vrepg)
967    }
968
969    macro_rules! impl_bool_vec_splats {
970        ($(($ty:ty, $shortty:tt, $boolty:ty)),*) => {
971            $(
972                #[unstable(feature = "stdarch_s390x", issue = "135681")]
973                impl VectorSplats<$boolty> for $ty {
974                    #[inline]
975                    #[target_feature(enable = "vector")]
976                    unsafe fn vec_splats(self) -> $boolty {
977                        transmute($shortty::splat(self))
978                    }
979                }
980            )*
981        }
982    }
983
984    impl_bool_vec_splats! {
985        (u8, u8x16, vector_bool_char),
986        (i8, i8x16, vector_bool_char),
987        (u16, u16x8, vector_bool_short),
988        (i16, i16x8, vector_bool_short),
989        (u32, u32x4, vector_bool_int),
990        (i32, i32x4, vector_bool_int),
991        (u64, u64x2, vector_bool_long_long),
992        (i64, i64x2, vector_bool_long_long)
993    }
994
995    #[unstable(feature = "stdarch_s390x", issue = "135681")]
996    pub trait CountBits {
997        type Result;
998
999        unsafe fn vec_cntlz(self) -> Self::Result;
1000        unsafe fn vec_cnttz(self) -> Self::Result;
1001        unsafe fn vec_popcnt(self) -> Self::Result;
1002    }
1003
1004    macro_rules! impl_count_bits {
1005        ($ty:tt) => {
1006            #[unstable(feature = "stdarch_s390x", issue = "135681")]
1007            impl CountBits for $ty {
1008                type Result = t_u!($ty);
1009
1010                #[inline]
1011                #[target_feature(enable = "vector")]
1012                unsafe fn vec_cntlz(self) -> Self::Result {
1013                    transmute(simd_ctlz(self))
1014                }
1015
1016                #[inline]
1017                #[target_feature(enable = "vector")]
1018                unsafe fn vec_cnttz(self) -> Self::Result {
1019                    transmute(simd_cttz(self))
1020                }
1021
1022                #[inline]
1023                #[target_feature(enable = "vector")]
1024                unsafe fn vec_popcnt(self) -> Self::Result {
1025                    transmute(simd_ctpop(self))
1026                }
1027            }
1028        };
1029    }
1030
1031    impl_count_bits!(vector_signed_char);
1032    impl_count_bits!(vector_unsigned_char);
1033    impl_count_bits!(vector_signed_short);
1034    impl_count_bits!(vector_unsigned_short);
1035    impl_count_bits!(vector_signed_int);
1036    impl_count_bits!(vector_unsigned_int);
1037    impl_count_bits!(vector_signed_long_long);
1038    impl_count_bits!(vector_unsigned_long_long);
1039
1040    test_impl! { vec_clzb_signed +(a: vector_signed_char) -> vector_unsigned_char [simd_ctlz, vclzb] }
1041    test_impl! { vec_clzh_signed +(a: vector_signed_short) -> vector_unsigned_short [simd_ctlz, vclzh] }
1042    test_impl! { vec_clzf_signed +(a: vector_signed_int) -> vector_unsigned_int [simd_ctlz, vclzf] }
1043    test_impl! { vec_clzg_signed +(a: vector_signed_long_long) -> vector_unsigned_long_long [simd_ctlz, vclzg] }
1044
1045    test_impl! { vec_clzb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctlz, vclzb] }
1046    test_impl! { vec_clzh_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_ctlz, vclzh] }
1047    test_impl! { vec_clzf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_ctlz, vclzf] }
1048    test_impl! { vec_clzg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_ctlz, vclzg] }
1049
1050    test_impl! { vec_ctzb_signed +(a: vector_signed_char) -> vector_unsigned_char [simd_cttz, vctzb] }
1051    test_impl! { vec_ctzh_signed +(a: vector_signed_short) -> vector_unsigned_short [simd_cttz, vctzh] }
1052    test_impl! { vec_ctzf_signed +(a: vector_signed_int) -> vector_unsigned_int [simd_cttz, vctzf] }
1053    test_impl! { vec_ctzg_signed +(a: vector_signed_long_long) -> vector_unsigned_long_long [simd_cttz, vctzg] }
1054
1055    test_impl! { vec_ctzb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_cttz, vctzb] }
1056    test_impl! { vec_ctzh_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_cttz, vctzh] }
1057    test_impl! { vec_ctzf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_cttz, vctzf] }
1058    test_impl! { vec_ctzg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_cttz, vctzg] }
1059
1060    test_impl! { vec_vpopctb_signed +(a: vector_signed_char) -> vector_signed_char [simd_ctpop, vpopctb] }
1061    test_impl! { vec_vpopcth_signed +(a: vector_signed_short) -> vector_signed_short [simd_ctpop, "vector-enhancements-1" vpopcth] }
1062    test_impl! { vec_vpopctf_signed +(a: vector_signed_int) -> vector_signed_int [simd_ctpop, "vector-enhancements-1" vpopctf] }
1063    test_impl! { vec_vpopctg_signed +(a: vector_signed_long_long) -> vector_signed_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] }
1064
1065    test_impl! { vec_vpopctb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctpop, vpopctb] }
1066    test_impl! { vec_vpopcth_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_ctpop, "vector-enhancements-1" vpopcth] }
1067    test_impl! { vec_vpopctf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_ctpop, "vector-enhancements-1" vpopctf] }
1068    test_impl! { vec_vpopctg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] }
1069
1070    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1071    pub trait VectorAnd<Other> {
1072        type Result;
1073        unsafe fn vec_and(self, b: Other) -> Self::Result;
1074    }
1075
1076    impl_vec_trait! { [VectorAnd vec_and] ~(simd_and) }
1077
1078    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1079    pub trait VectorOr<Other> {
1080        type Result;
1081        unsafe fn vec_or(self, b: Other) -> Self::Result;
1082    }
1083
1084    impl_vec_trait! { [VectorOr vec_or] ~(simd_or) }
1085
1086    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1087    pub trait VectorXor<Other> {
1088        type Result;
1089        unsafe fn vec_xor(self, b: Other) -> Self::Result;
1090    }
1091
1092    impl_vec_trait! { [VectorXor vec_xor] ~(simd_xor) }
1093
1094    #[inline]
1095    #[target_feature(enable = "vector")]
1096    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vno))]
1097    unsafe fn nor(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1098        let a: u8x16 = transmute(a);
1099        let b: u8x16 = transmute(b);
1100        transmute(simd_xor(simd_or(a, b), u8x16::splat(0xff)))
1101    }
1102
1103    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1104    pub trait VectorNor<Other> {
1105        type Result;
1106        unsafe fn vec_nor(self, b: Other) -> Self::Result;
1107    }
1108
1109    impl_vec_trait! { [VectorNor vec_nor]+ 2c (nor) }
1110
1111    #[inline]
1112    #[target_feature(enable = "vector")]
1113    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnn))]
1114    unsafe fn nand(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1115        let a: u8x16 = transmute(a);
1116        let b: u8x16 = transmute(b);
1117        transmute(simd_xor(simd_and(a, b), u8x16::splat(0xff)))
1118    }
1119
1120    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1121    pub trait VectorNand<Other> {
1122        type Result;
1123        unsafe fn vec_nand(self, b: Other) -> Self::Result;
1124    }
1125
1126    impl_vec_trait! { [VectorNand vec_nand]+ 2c (nand) }
1127
1128    #[inline]
1129    #[target_feature(enable = "vector")]
1130    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnx))]
1131    unsafe fn eqv(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1132        let a: u8x16 = transmute(a);
1133        let b: u8x16 = transmute(b);
1134        transmute(simd_xor(simd_xor(a, b), u8x16::splat(0xff)))
1135    }
1136
1137    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1138    pub trait VectorEqv<Other> {
1139        type Result;
1140        unsafe fn vec_eqv(self, b: Other) -> Self::Result;
1141    }
1142
1143    impl_vec_trait! { [VectorEqv vec_eqv]+ 2c (eqv) }
1144
1145    #[inline]
1146    #[target_feature(enable = "vector")]
1147    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnc))]
1148    unsafe fn andc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1149        let a = transmute(a);
1150        let b = transmute(b);
1151        transmute(simd_and(simd_xor(u8x16::splat(0xff), b), a))
1152    }
1153
1154    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1155    pub trait VectorAndc<Other> {
1156        type Result;
1157        unsafe fn vec_andc(self, b: Other) -> Self::Result;
1158    }
1159
1160    impl_vec_trait! { [VectorAndc vec_andc]+ 2c (andc) }
1161
1162    #[inline]
1163    #[target_feature(enable = "vector")]
1164    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(voc))]
1165    unsafe fn orc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1166        let a = transmute(a);
1167        let b = transmute(b);
1168        transmute(simd_or(simd_xor(u8x16::splat(0xff), b), a))
1169    }
1170
1171    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1172    pub trait VectorOrc<Other> {
1173        type Result;
1174        unsafe fn vec_orc(self, b: Other) -> Self::Result;
1175    }
1176
1177    impl_vec_trait! { [VectorOrc vec_orc]+ 2c (orc) }
1178
1179    // Z vector intrinsic      C23 math.h  LLVM IR         ISO/IEC 60559 operation        inexact  vfidb parameters
1180    //
1181    // vec_rint                rint        llvm.rint       roundToIntegralExact           yes      0, 0
1182    // vec_roundc              nearbyint   llvm.nearbyint  n/a                            no       4, 0
1183    // vec_floor / vec_roundm  floor       llvm.floor      roundToIntegralTowardNegative  no       4, 7
1184    // vec_ceil / vec_roundp   ceil        llvm.ceil       roundToIntegralTowardPositive  no       4, 6
1185    // vec_trunc / vec_roundz  trunc       llvm.trunc      roundToIntegralTowardZero      no       4, 5
1186    // vec_round               roundeven   llvm.roundeven  roundToIntegralTiesToEven      no       4, 4
1187    // n/a                     round       llvm.round      roundToIntegralTiesAway        no       4, 1
1188
1189    // `simd_round_ties_even` is implemented as `llvm.rint`.
1190    test_impl! { vec_rint_f32 (a: vector_float) -> vector_float [simd_round_ties_even, "vector-enhancements-1" vfisb] }
1191    test_impl! { vec_rint_f64 (a: vector_double) -> vector_double [simd_round_ties_even, vfidb] }
1192
1193    test_impl! { vec_roundc_f32 (a: vector_float) -> vector_float [nearbyint_v4f32,  "vector-enhancements-1" vfisb] }
1194    test_impl! { vec_roundc_f64 (a: vector_double) -> vector_double [nearbyint_v2f64, vfidb] }
1195
1196    test_impl! { vec_round_f32 (a: vector_float) -> vector_float [roundeven_v4f32, "vector-enhancements-1" vfisb] }
1197    test_impl! { vec_round_f64 (a: vector_double) -> vector_double [roundeven_v2f64, vfidb] }
1198
1199    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1200    pub trait VectorRoundc {
1201        unsafe fn vec_roundc(self) -> Self;
1202    }
1203
1204    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1205    pub trait VectorRound {
1206        unsafe fn vec_round(self) -> Self;
1207    }
1208
1209    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1210    pub trait VectorRint {
1211        unsafe fn vec_rint(self) -> Self;
1212    }
1213
1214    impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f32 (vector_float) }
1215    impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f64 (vector_double) }
1216
1217    impl_vec_trait! { [VectorRound vec_round] vec_round_f32 (vector_float) }
1218    impl_vec_trait! { [VectorRound vec_round] vec_round_f64 (vector_double) }
1219
1220    impl_vec_trait! { [VectorRint vec_rint] simd_round_ties_even (vector_float) }
1221    impl_vec_trait! { [VectorRint vec_rint] simd_round_ties_even (vector_double) }
1222
1223    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1224    pub trait VectorTrunc {
1225        // same as vec_roundz
1226        unsafe fn vec_trunc(self) -> Self;
1227    }
1228
1229    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1230    pub trait VectorCeil {
1231        // same as vec_roundp
1232        unsafe fn vec_ceil(self) -> Self;
1233    }
1234
1235    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1236    pub trait VectorFloor {
1237        // same as vec_roundm
1238        unsafe fn vec_floor(self) -> Self;
1239    }
1240
1241    impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_float) }
1242    impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_double) }
1243
1244    impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_float) }
1245    impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_double) }
1246
1247    impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_float) }
1248    impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_double) }
1249
1250    macro_rules! impl_vec_shift {
1251        ([$Trait:ident $m:ident] ($b:ident, $h:ident, $w:ident, $g:ident)) => {
1252            impl_vec_trait!{ [$Trait $m]+ $b (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1253            impl_vec_trait!{ [$Trait $m]+ $b (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1254            impl_vec_trait!{ [$Trait $m]+ $h (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1255            impl_vec_trait!{ [$Trait $m]+ $h (vector_signed_short, vector_unsigned_short) -> vector_signed_short }
1256            impl_vec_trait!{ [$Trait $m]+ $w (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1257            impl_vec_trait!{ [$Trait $m]+ $w (vector_signed_int, vector_unsigned_int) -> vector_signed_int }
1258            impl_vec_trait!{ [$Trait $m]+ $g (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1259            impl_vec_trait!{ [$Trait $m]+ $g (vector_signed_long_long, vector_unsigned_long_long) -> vector_signed_long_long }
1260        };
1261    }
1262
1263    macro_rules! impl_shift {
1264        ($fun:ident $intr:ident $ty:ident) => {
1265            #[inline]
1266            #[target_feature(enable = "vector")]
1267            #[cfg_attr(test, assert_instr($fun))]
1268            unsafe fn $fun(a: t_t_l!($ty), b: t_t_l!($ty)) -> t_t_l!($ty) {
1269                let a = transmute(a);
1270                // use the remainder of b by the width of a's elements to prevent UB
1271                let b = simd_rem(transmute(b), <t_t_s!($ty)>::splat($ty::BITS as $ty));
1272
1273                transmute($intr(a, b))
1274            }
1275        };
1276    }
1277
1278    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1279    pub trait VectorSl<Other> {
1280        type Result;
1281        unsafe fn vec_sl(self, b: Other) -> Self::Result;
1282    }
1283
1284    impl_shift! { veslvb simd_shl u8 }
1285    impl_shift! { veslvh simd_shl u16 }
1286    impl_shift! { veslvf simd_shl u32 }
1287    impl_shift! { veslvg simd_shl u64 }
1288
1289    impl_vec_shift! { [VectorSl vec_sl] (veslvb, veslvh, veslvf, veslvg) }
1290
1291    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1292    pub trait VectorSr<Other> {
1293        type Result;
1294        unsafe fn vec_sr(self, b: Other) -> Self::Result;
1295    }
1296
1297    impl_shift! { vesrlvb simd_shr u8 }
1298    impl_shift! { vesrlvh simd_shr u16 }
1299    impl_shift! { vesrlvf simd_shr u32 }
1300    impl_shift! { vesrlvg simd_shr u64 }
1301
1302    impl_vec_shift! { [VectorSr vec_sr] (vesrlvb, vesrlvh, vesrlvf, vesrlvg) }
1303
1304    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1305    pub trait VectorSra<Other> {
1306        type Result;
1307        unsafe fn vec_sra(self, b: Other) -> Self::Result;
1308    }
1309
1310    impl_shift! { vesravb simd_shr i8 }
1311    impl_shift! { vesravh simd_shr i16 }
1312    impl_shift! { vesravf simd_shr i32 }
1313    impl_shift! { vesravg simd_shr i64 }
1314
1315    impl_vec_shift! { [VectorSra vec_sra] (vesravb, vesravh, vesravf, vesravg) }
1316
1317    macro_rules! impl_vec_shift_byte {
1318        ([$trait:ident $m:ident] ($f:ident)) => {
1319            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_signed_char) -> vector_unsigned_char }
1320            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1321            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_signed_char) -> vector_signed_char }
1322            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1323            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_signed_short) -> vector_unsigned_short }
1324            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1325            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_signed_short) -> vector_signed_short }
1326            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_unsigned_short) -> vector_signed_short }
1327            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_signed_int) -> vector_unsigned_int }
1328            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1329            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_signed_int) -> vector_signed_int }
1330            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_unsigned_int) -> vector_signed_int }
1331            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_signed_long_long) -> vector_unsigned_long_long }
1332            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1333            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_signed_long_long) -> vector_signed_long_long }
1334            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_unsigned_long_long) -> vector_signed_long_long }
1335            impl_vec_trait!{ [$trait $m]+ $f (vector_float, vector_signed_int) -> vector_float }
1336            impl_vec_trait!{ [$trait $m]+ $f (vector_float, vector_unsigned_int) -> vector_float }
1337            impl_vec_trait!{ [$trait $m]+ $f (vector_double, vector_signed_long_long) -> vector_double }
1338            impl_vec_trait!{ [$trait $m]+ $f (vector_double, vector_unsigned_long_long) -> vector_double }
1339        };
1340    }
1341
1342    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1343    pub trait VectorSlb<Other> {
1344        type Result;
1345        unsafe fn vec_slb(self, b: Other) -> Self::Result;
1346    }
1347
1348    impl_vec_shift_byte! { [VectorSlb vec_slb] (vslb) }
1349
1350    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1351    pub trait VectorSrab<Other> {
1352        type Result;
1353        unsafe fn vec_srab(self, b: Other) -> Self::Result;
1354    }
1355
1356    impl_vec_shift_byte! { [VectorSrab vec_srab] (vsrab) }
1357
1358    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1359    pub trait VectorSrb<Other> {
1360        type Result;
1361        unsafe fn vec_srb(self, b: Other) -> Self::Result;
1362    }
1363
1364    impl_vec_shift_byte! { [VectorSrb vec_srb] (vsrlb) }
1365
1366    macro_rules! impl_vec_shift_long {
1367        ([$trait:ident $m:ident] ($f:ident)) => {
1368            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1369            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1370            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_unsigned_char) -> vector_unsigned_short }
1371            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_unsigned_char) -> vector_signed_short }
1372            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_unsigned_char) -> vector_unsigned_int }
1373            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_unsigned_char) -> vector_signed_int }
1374            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_unsigned_char) -> vector_unsigned_long_long }
1375            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_unsigned_char) -> vector_signed_long_long }
1376        };
1377    }
1378
1379    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1380    pub trait VectorSrl<Other> {
1381        type Result;
1382        unsafe fn vec_srl(self, b: Other) -> Self::Result;
1383    }
1384
1385    impl_vec_shift_long! { [VectorSrl vec_srl] (vsrl) }
1386
1387    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1388    pub trait VectorSral<Other> {
1389        type Result;
1390        unsafe fn vec_sral(self, b: Other) -> Self::Result;
1391    }
1392
1393    impl_vec_shift_long! { [VectorSral vec_sral] (vsra) }
1394
1395    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1396    pub trait VectorSll<Other> {
1397        type Result;
1398        unsafe fn vec_sll(self, b: Other) -> Self::Result;
1399    }
1400
1401    impl_vec_shift_long! { [VectorSll vec_sll] (vsl) }
1402
1403    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1404    pub trait VectorRl<Other> {
1405        type Result;
1406        unsafe fn vec_rl(self, b: Other) -> Self::Result;
1407    }
1408
1409    macro_rules! impl_rot {
1410        ($fun:ident $ty:ident) => {
1411            #[inline]
1412            #[target_feature(enable = "vector")]
1413            #[cfg_attr(test, assert_instr($fun))]
1414            unsafe fn $fun(a: t_t_l!($ty), b: t_t_l!($ty)) -> t_t_l!($ty) {
1415                simd_funnel_shl(a, a, b)
1416            }
1417        };
1418    }
1419
1420    impl_rot! { verllvb u8 }
1421    impl_rot! { verllvh u16 }
1422    impl_rot! { verllvf u32 }
1423    impl_rot! { verllvg u64 }
1424
1425    impl_vec_shift! { [VectorRl vec_rl] (verllvb, verllvh, verllvf, verllvg) }
1426
1427    macro_rules! test_rot_imm {
1428        ($fun:ident $instr:ident $ty:ident) => {
1429            #[inline]
1430            #[target_feature(enable = "vector")]
1431            #[cfg_attr(test, assert_instr($instr))]
1432            unsafe fn $fun(a: t_t_l!($ty), bits: core::ffi::c_ulong) -> t_t_l!($ty) {
1433                // mod by the number of bits in a's element type to prevent UB
1434                let bits = (bits % $ty::BITS as core::ffi::c_ulong) as $ty;
1435                let b = <t_t_s!($ty)>::splat(bits);
1436
1437                simd_funnel_shl(a, a, transmute(b))
1438            }
1439        };
1440    }
1441
1442    test_rot_imm! { verllvb_imm verllb u8 }
1443    test_rot_imm! { verllvh_imm verllh u16 }
1444    test_rot_imm! { verllvf_imm verllf u32 }
1445    test_rot_imm! { verllvg_imm verllg u64 }
1446
1447    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1448    pub trait VectorRli {
1449        unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self;
1450    }
1451
1452    macro_rules! impl_rot_imm {
1453        ($($ty:ident, $intr:ident),*) => {
1454            $(
1455                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1456                impl VectorRli for $ty {
1457                    #[inline]
1458                    #[target_feature(enable = "vector")]
1459                    unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self {
1460                        transmute($intr(transmute(self), bits))
1461                    }
1462                }
1463
1464                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1465                impl VectorRli for t_u!($ty) {
1466                    #[inline]
1467                    #[target_feature(enable = "vector")]
1468                    unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self {
1469                        $intr(self, bits)
1470                    }
1471                }
1472            )*
1473        }
1474    }
1475
1476    impl_rot_imm! {
1477        vector_signed_char, verllvb_imm,
1478        vector_signed_short, verllvh_imm,
1479        vector_signed_int, verllvf_imm,
1480        vector_signed_long_long, verllvg_imm
1481    }
1482
1483    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1484    pub trait VectorRlMask<Other> {
1485        unsafe fn vec_rl_mask<const IMM8: u8>(self, other: Other) -> Self;
1486    }
1487
1488    macro_rules! impl_rl_mask {
1489        ($($ty:ident, $intr:ident, $fun:ident),*) => {
1490            $(
1491                #[inline]
1492                #[target_feature(enable = "vector")]
1493                #[cfg_attr(test, assert_instr($intr, IMM8 = 6))]
1494                unsafe fn $fun<const IMM8: u8>(a: $ty, b: t_u!($ty)) -> $ty {
1495                    // mod by the number of bits in a's element type to prevent UB
1496                    $intr(a, a, transmute(b), const { (IMM8 % <l_t_t!($ty)>::BITS as u8) as i32 })
1497                }
1498
1499                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1500                impl VectorRlMask<t_u!($ty)> for $ty {
1501                    #[inline]
1502                    #[target_feature(enable = "vector")]
1503                    unsafe fn vec_rl_mask<const IMM8: u8>(self, other: t_u!($ty)) -> Self {
1504                        $fun::<IMM8>(self, other)
1505                    }
1506                }
1507
1508                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1509                impl VectorRlMask<t_u!($ty)> for t_u!($ty) {
1510                    #[inline]
1511                    #[target_feature(enable = "vector")]
1512                    unsafe fn vec_rl_mask<const IMM8: u8>(self, other: t_u!($ty)) -> Self {
1513                        transmute($fun::<IMM8>(transmute(self), transmute(other)))
1514                    }
1515                }
1516            )*
1517        }
1518    }
1519
1520    impl_rl_mask! {
1521        vector_signed_char, verimb, test_verimb,
1522        vector_signed_short, verimh, test_verimh,
1523        vector_signed_int, verimf, test_verimf,
1524        vector_signed_long_long, verimg, test_verimg
1525    }
1526
1527    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1528    pub trait VectorReve {
1529        unsafe fn vec_reve(self) -> Self;
1530    }
1531
1532    #[repr(simd)]
1533    struct ReverseMask<const N: usize>([u32; N]);
1534
1535    impl<const N: usize> ReverseMask<N> {
1536        const fn new() -> Self {
1537            let mut index = [0; N];
1538            let mut i = 0;
1539            while i < N {
1540                index[i] = (N - i - 1) as u32;
1541                i += 1;
1542            }
1543            ReverseMask(index)
1544        }
1545    }
1546
1547    macro_rules! impl_reve {
1548        ($($ty:ident, $fun:ident, $instr:ident),*) => {
1549            $(
1550                #[inline]
1551                #[target_feature(enable = "vector")]
1552                #[cfg_attr(test, assert_instr($instr))]
1553                unsafe fn $fun(a: $ty) -> $ty {
1554                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1555                    simd_shuffle(a, a, const { ShuffleMask::<N>::reverse() })
1556                }
1557
1558                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1559                impl VectorReve for $ty {
1560                    #[inline]
1561                    #[target_feature(enable = "vector")]
1562                    unsafe fn vec_reve(self) -> Self {
1563                        $fun(self)
1564                    }
1565                }
1566
1567                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1568                impl VectorReve for t_u!($ty) {
1569                    #[inline]
1570                    #[target_feature(enable = "vector")]
1571                    unsafe fn vec_reve(self) -> Self {
1572                        transmute($fun(transmute(self)))
1573                    }
1574                }
1575
1576                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1577                impl VectorReve for t_b!($ty) {
1578                    #[inline]
1579                    #[target_feature(enable = "vector")]
1580                    unsafe fn vec_reve(self) -> Self {
1581                        transmute($fun(transmute(self)))
1582                    }
1583                }
1584            )*
1585        }
1586    }
1587
1588    impl_reve! {
1589        vector_signed_char, reveb, vperm,
1590        vector_signed_short, reveh, vperm,
1591        vector_signed_int, revef, vperm,
1592        vector_signed_long_long, reveg, vpdi
1593    }
1594
1595    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1596    impl VectorReve for vector_float {
1597        #[inline]
1598        #[target_feature(enable = "vector")]
1599        unsafe fn vec_reve(self) -> Self {
1600            transmute(transmute::<_, vector_signed_int>(self).vec_reve())
1601        }
1602    }
1603
1604    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1605    impl VectorReve for vector_double {
1606        #[inline]
1607        #[target_feature(enable = "vector")]
1608        unsafe fn vec_reve(self) -> Self {
1609            transmute(transmute::<_, vector_signed_long_long>(self).vec_reve())
1610        }
1611    }
1612
1613    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1614    pub trait VectorRevb {
1615        unsafe fn vec_revb(self) -> Self;
1616    }
1617
1618    test_impl! { bswapb (a: vector_signed_char) -> vector_signed_char [simd_bswap, _] }
1619    test_impl! { bswaph (a: vector_signed_short) -> vector_signed_short [simd_bswap, vperm] }
1620    test_impl! { bswapf (a: vector_signed_int) -> vector_signed_int [simd_bswap, vperm] }
1621    test_impl! { bswapg (a: vector_signed_long_long) -> vector_signed_long_long [simd_bswap, vperm] }
1622
1623    impl_vec_trait! { [VectorRevb vec_revb]+ bswapb (vector_unsigned_char) }
1624    impl_vec_trait! { [VectorRevb vec_revb]+ bswapb (vector_signed_char) }
1625    impl_vec_trait! { [VectorRevb vec_revb]+ bswaph (vector_unsigned_short) }
1626    impl_vec_trait! { [VectorRevb vec_revb]+ bswaph (vector_signed_short) }
1627    impl_vec_trait! { [VectorRevb vec_revb]+ bswapf (vector_unsigned_int) }
1628    impl_vec_trait! { [VectorRevb vec_revb]+ bswapf (vector_signed_int) }
1629    impl_vec_trait! { [VectorRevb vec_revb]+ bswapg (vector_unsigned_long_long) }
1630    impl_vec_trait! { [VectorRevb vec_revb]+ bswapg (vector_signed_long_long) }
1631
1632    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1633    impl VectorRevb for vector_float {
1634        #[inline]
1635        #[target_feature(enable = "vector")]
1636        unsafe fn vec_revb(self) -> Self {
1637            transmute(transmute::<_, vector_signed_int>(self).vec_revb())
1638        }
1639    }
1640
1641    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1642    impl VectorRevb for vector_double {
1643        #[inline]
1644        #[target_feature(enable = "vector")]
1645        unsafe fn vec_revb(self) -> Self {
1646            transmute(transmute::<_, vector_signed_long_long>(self).vec_revb())
1647        }
1648    }
1649
1650    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1651    pub trait VectorMergel {
1652        unsafe fn vec_mergel(self, other: Self) -> Self;
1653    }
1654
1655    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1656    pub trait VectorMergeh {
1657        unsafe fn vec_mergeh(self, other: Self) -> Self;
1658    }
1659
1660    macro_rules! impl_merge {
1661        ($($ty:ident, $mergel:ident, $mergeh:ident),*) => {
1662            $(
1663                #[inline]
1664                #[target_feature(enable = "vector")]
1665                #[cfg_attr(test, assert_instr($mergel))]
1666                unsafe fn $mergel(a: $ty, b: $ty) -> $ty {
1667                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1668                    simd_shuffle(a, b, const { ShuffleMask::<N>::merge_low() })
1669                }
1670
1671                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1672                impl VectorMergel for $ty {
1673                    #[inline]
1674                    #[target_feature(enable = "vector")]
1675                    unsafe fn vec_mergel(self, other: Self) -> Self {
1676                        $mergel(self, other)
1677                    }
1678                }
1679
1680                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1681                impl VectorMergel for t_u!($ty) {
1682                    #[inline]
1683                    #[target_feature(enable = "vector")]
1684                    unsafe fn vec_mergel(self, other: Self) -> Self {
1685                        transmute($mergel(transmute(self), transmute(other)))
1686                    }
1687                }
1688
1689                #[inline]
1690                #[target_feature(enable = "vector")]
1691                #[cfg_attr(test, assert_instr($mergeh))]
1692                unsafe fn $mergeh(a: $ty, b: $ty) -> $ty {
1693                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1694                    simd_shuffle(a, b, const { ShuffleMask::<N>::merge_high() })
1695                }
1696
1697                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1698                impl VectorMergeh for $ty {
1699                    #[inline]
1700                    #[target_feature(enable = "vector")]
1701                    unsafe fn vec_mergeh(self, other: Self) -> Self {
1702                        $mergeh(self, other)
1703                    }
1704                }
1705
1706                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1707                impl VectorMergeh for t_u!($ty) {
1708                    #[inline]
1709                    #[target_feature(enable = "vector")]
1710                    unsafe fn vec_mergeh(self, other: Self) -> Self {
1711                        transmute($mergeh(transmute(self), transmute(other)))
1712                    }
1713                }
1714            )*
1715        }
1716    }
1717
1718    impl_merge! {
1719        vector_signed_char, vmrlb, vmrhb,
1720        vector_signed_short, vmrlh, vmrhh,
1721        vector_signed_int, vmrlf, vmrhf,
1722        vector_signed_long_long, vmrlg, vmrhg
1723    }
1724
1725    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1726    pub trait VectorPerm {
1727        unsafe fn vec_perm(self, other: Self, c: vector_unsigned_char) -> Self;
1728    }
1729
1730    macro_rules! impl_merge {
1731        ($($ty:ident),*) => {
1732            $(
1733                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1734                impl VectorPerm for $ty {
1735                    #[inline]
1736                    #[target_feature(enable = "vector")]
1737                    unsafe fn vec_perm(self, other: Self, c: vector_unsigned_char) -> Self {
1738                        transmute(vperm(transmute(self), transmute(other), c))
1739                    }
1740                }
1741            )*
1742        }
1743    }
1744
1745    impl_merge! {
1746        vector_signed_char,
1747        vector_signed_short,
1748        vector_signed_int,
1749        vector_signed_long_long,
1750        vector_unsigned_char,
1751        vector_unsigned_short,
1752        vector_unsigned_int,
1753        vector_unsigned_long_long,
1754        vector_bool_char,
1755        vector_bool_short,
1756        vector_bool_int,
1757        vector_bool_long_long,
1758        vector_float,
1759        vector_double
1760    }
1761
1762    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1763    pub trait VectorSumU128 {
1764        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char;
1765    }
1766
1767    #[inline]
1768    #[target_feature(enable = "vector")]
1769    #[cfg_attr(test, assert_instr(vsumqf))]
1770    pub unsafe fn vec_vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128 {
1771        transmute(vsumqf(a, b))
1772    }
1773
1774    #[inline]
1775    #[target_feature(enable = "vector")]
1776    #[cfg_attr(test, assert_instr(vsumqg))]
1777    pub unsafe fn vec_vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128 {
1778        transmute(vsumqg(a, b))
1779    }
1780
1781    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1782    impl VectorSumU128 for vector_unsigned_int {
1783        #[inline]
1784        #[target_feature(enable = "vector")]
1785        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char {
1786            transmute(vec_vsumqf(self, other))
1787        }
1788    }
1789
1790    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1791    impl VectorSumU128 for vector_unsigned_long_long {
1792        #[inline]
1793        #[target_feature(enable = "vector")]
1794        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char {
1795            transmute(vec_vsumqg(self, other))
1796        }
1797    }
1798
1799    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1800    pub trait VectorSum2 {
1801        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long;
1802    }
1803
1804    test_impl! { vec_vsumgh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_long_long [vsumgh, vsumgh] }
1805    test_impl! { vec_vsumgf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [vsumgf, vsumgf] }
1806
1807    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1808    impl VectorSum2 for vector_unsigned_short {
1809        #[inline]
1810        #[target_feature(enable = "vector")]
1811        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long {
1812            vec_vsumgh(self, other)
1813        }
1814    }
1815
1816    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1817    impl VectorSum2 for vector_unsigned_int {
1818        #[inline]
1819        #[target_feature(enable = "vector")]
1820        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long {
1821            vec_vsumgf(self, other)
1822        }
1823    }
1824
1825    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1826    pub trait VectorSum4 {
1827        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int;
1828    }
1829
1830    test_impl! { vec_vsumb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_int [vsumb, vsumb] }
1831    test_impl! { vec_vsumh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int [vsumh, vsumh] }
1832
1833    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1834    impl VectorSum4 for vector_unsigned_char {
1835        #[inline]
1836        #[target_feature(enable = "vector")]
1837        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int {
1838            vec_vsumb(self, other)
1839        }
1840    }
1841
1842    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1843    impl VectorSum4 for vector_unsigned_short {
1844        #[inline]
1845        #[target_feature(enable = "vector")]
1846        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int {
1847            vec_vsumh(self, other)
1848        }
1849    }
1850
1851    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1852    pub trait VectorSubc<Other> {
1853        type Result;
1854        unsafe fn vec_subc(self, b: Other) -> Self::Result;
1855    }
1856
1857    test_impl! { vec_vscbib (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vscbib, vscbib] }
1858    test_impl! { vec_vscbih (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vscbih, vscbih] }
1859    test_impl! { vec_vscbif (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vscbif, vscbif] }
1860    test_impl! { vec_vscbig (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vscbig, vscbig] }
1861
1862    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbib (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1863    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbih (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1864    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbif (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1865    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbig (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1866
1867    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1868    pub trait VectorSqrt {
1869        unsafe fn vec_sqrt(self) -> Self;
1870    }
1871
1872    test_impl! { vec_sqrt_f32 (v: vector_float) -> vector_float [ simd_fsqrt, "vector-enhancements-1" vfsqsb ] }
1873    test_impl! { vec_sqrt_f64 (v: vector_double) -> vector_double [ simd_fsqrt, vfsqdb ] }
1874
1875    impl_vec_trait! { [VectorSqrt vec_sqrt] vec_sqrt_f32 (vector_float) }
1876    impl_vec_trait! { [VectorSqrt vec_sqrt] vec_sqrt_f64 (vector_double) }
1877
1878    macro_rules! vfae_wrapper {
1879        ($($name:ident $ty:ident)*) => {
1880            $(
1881                #[inline]
1882                #[target_feature(enable = "vector")]
1883                #[cfg_attr(test, assert_instr($name, IMM = 0))]
1884                unsafe fn $name<const IMM: i32>(
1885                    a: $ty,
1886                    b: $ty,
1887                ) -> $ty {
1888                    super::$name(a, b, IMM)
1889                }
1890            )*
1891        }
1892     }
1893
1894    vfae_wrapper! {
1895       vfaeb vector_signed_char
1896       vfaeh vector_signed_short
1897       vfaef vector_signed_int
1898
1899       vfaezb vector_signed_char
1900       vfaezh vector_signed_short
1901       vfaezf vector_signed_int
1902    }
1903
1904    macro_rules! impl_vfae {
1905        ([idx_cc $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1906            impl_vfae! { [idx_cc $Trait $m] $imm
1907                $b vector_signed_char vector_signed_char
1908                $b vector_unsigned_char vector_unsigned_char
1909                $b vector_bool_char vector_unsigned_char
1910
1911                $h vector_signed_short vector_signed_short
1912                $h vector_unsigned_short vector_unsigned_short
1913                $h vector_bool_short vector_unsigned_short
1914
1915                $f vector_signed_int vector_signed_int
1916                $f vector_unsigned_int vector_unsigned_int
1917                $f vector_bool_int vector_unsigned_int
1918            }
1919        };
1920        ([idx_cc $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident $r:ident)*) => {
1921            $(
1922                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1923                impl $Trait<Self> for $ty {
1924                    type Result = $r;
1925                    #[inline]
1926                    #[target_feature(enable = "vector")]
1927                    unsafe fn $m(self, b: Self) -> (Self::Result, i32) {
1928                        let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b));
1929                        (transmute(x), y)
1930                    }
1931                }
1932            )*
1933        };
1934        ([cc $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1935            impl_vfae! { [cc $Trait $m] $imm
1936                $b vector_signed_char
1937                $b vector_unsigned_char
1938                $b vector_bool_char
1939
1940                $h vector_signed_short
1941                $h vector_unsigned_short
1942                $h vector_bool_short
1943
1944                $f vector_signed_int
1945                $f vector_unsigned_int
1946                $f vector_bool_int
1947            }
1948        };
1949        ([cc $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident)*) => {
1950            $(
1951                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1952                impl $Trait<Self> for $ty {
1953                    type Result = t_b!($ty);
1954                    #[inline]
1955                    #[target_feature(enable = "vector")]
1956                    unsafe fn $m(self, b: Self) -> (Self::Result, i32) {
1957                        let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b));
1958                        (transmute(x), y)
1959                    }
1960                }
1961            )*
1962        };
1963        ([idx $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1964            impl_vfae! { [idx $Trait $m] $imm
1965                $b vector_signed_char vector_signed_char
1966                $b vector_unsigned_char vector_unsigned_char
1967                $b vector_bool_char vector_unsigned_char
1968
1969                $h vector_signed_short vector_signed_short
1970                $h vector_unsigned_short vector_unsigned_short
1971                $h vector_bool_short vector_unsigned_short
1972
1973                $f vector_signed_int vector_signed_int
1974                $f vector_unsigned_int vector_unsigned_int
1975                $f vector_bool_int vector_unsigned_int
1976            }
1977        };
1978        ([idx $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident $r:ident)*) => {
1979            $(
1980                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1981                impl $Trait<Self> for $ty {
1982                    type Result = $r;
1983                    #[inline]
1984                    #[target_feature(enable = "vector")]
1985                    unsafe fn $m(self, b: Self) -> Self::Result {
1986                        transmute($fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)))
1987                    }
1988                }
1989            )*
1990        };
1991        ([$Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1992            impl_vfae! { [$Trait $m] $imm
1993                $b vector_signed_char
1994                $b vector_unsigned_char
1995                $b vector_bool_char
1996
1997                $h vector_signed_short
1998                $h vector_unsigned_short
1999                $h vector_bool_short
2000
2001                $f vector_signed_int
2002                $f vector_unsigned_int
2003                $f vector_bool_int
2004            }
2005        };
2006        ([$Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident)*) => {
2007            $(
2008                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2009                impl $Trait<Self> for $ty {
2010                    type Result = t_b!($ty);
2011                    #[inline]
2012                    #[target_feature(enable = "vector")]
2013                    unsafe fn $m(self, b: Self) -> Self::Result {
2014                        transmute($fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)))
2015                    }
2016                }
2017            )*
2018        };
2019    }
2020
2021    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2022    pub trait VectorFindAnyEq<Other> {
2023        type Result;
2024        unsafe fn vec_find_any_eq(self, other: Other) -> Self::Result;
2025    }
2026
2027    impl_vfae! { [VectorFindAnyEq vec_find_any_eq] Eq vfaeb vfaeh vfaef }
2028
2029    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2030    pub trait VectorFindAnyNe<Other> {
2031        type Result;
2032        unsafe fn vec_find_any_ne(self, other: Other) -> Self::Result;
2033    }
2034
2035    impl_vfae! { [VectorFindAnyNe vec_find_any_ne] Ne vfaeb vfaeh vfaef }
2036
2037    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2038    pub trait VectorFindAnyEqOrZeroIdx<Other> {
2039        type Result;
2040        unsafe fn vec_find_any_eq_or_0_idx(self, other: Other) -> Self::Result;
2041    }
2042
2043    impl_vfae! { [idx VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx] EqIdx
2044        vfaezb vector_signed_char vector_signed_char
2045        vfaezb vector_unsigned_char vector_unsigned_char
2046        vfaezb vector_bool_char vector_unsigned_char
2047
2048        vfaezh vector_signed_short vector_signed_short
2049        vfaezh vector_unsigned_short vector_unsigned_short
2050        vfaezh vector_bool_short vector_unsigned_short
2051
2052        vfaezf vector_signed_int vector_signed_int
2053        vfaezf vector_unsigned_int vector_unsigned_int
2054        vfaezf vector_bool_int vector_unsigned_int
2055    }
2056
2057    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2058    pub trait VectorFindAnyNeOrZeroIdx<Other> {
2059        type Result;
2060        unsafe fn vec_find_any_ne_or_0_idx(self, other: Other) -> Self::Result;
2061    }
2062
2063    impl_vfae! { [idx VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx] NeIdx
2064        vfaezb vector_signed_char vector_signed_char
2065        vfaezb vector_unsigned_char vector_unsigned_char
2066        vfaezb vector_bool_char vector_unsigned_char
2067
2068        vfaezh vector_signed_short vector_signed_short
2069        vfaezh vector_unsigned_short vector_unsigned_short
2070        vfaezh vector_bool_short vector_unsigned_short
2071
2072        vfaezf vector_signed_int vector_signed_int
2073        vfaezf vector_unsigned_int vector_unsigned_int
2074        vfaezf vector_bool_int vector_unsigned_int
2075    }
2076
2077    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2078    pub trait VectorFindAnyEqIdx<Other> {
2079        type Result;
2080        unsafe fn vec_find_any_eq_idx(self, other: Other) -> Self::Result;
2081    }
2082
2083    impl_vfae! { [idx VectorFindAnyEqIdx vec_find_any_eq_idx] EqIdx vfaeb vfaeh vfaef }
2084
2085    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2086    pub trait VectorFindAnyNeIdx<Other> {
2087        type Result;
2088        unsafe fn vec_find_any_ne_idx(self, other: Other) -> Self::Result;
2089    }
2090
2091    impl_vfae! { [idx VectorFindAnyNeIdx vec_find_any_ne_idx] NeIdx vfaeb vfaeh vfaef }
2092
2093    macro_rules! vfaes_wrapper {
2094        ($($name:ident $ty:ident)*) => {
2095            $(
2096                #[inline]
2097                #[target_feature(enable = "vector")]
2098                #[cfg_attr(test, assert_instr($name, IMM = 0))]
2099                unsafe fn $name<const IMM: i32>(
2100                    a: $ty,
2101                    b: $ty,
2102                ) -> PackedTuple<$ty, i32> {
2103                    super::$name(a, b, IMM)
2104                }
2105            )*
2106        }
2107     }
2108
2109    vfaes_wrapper! {
2110        vfaebs vector_signed_char
2111        vfaehs vector_signed_short
2112        vfaefs vector_signed_int
2113
2114        vfaezbs vector_signed_char
2115        vfaezhs vector_signed_short
2116        vfaezfs vector_signed_int
2117    }
2118
2119    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2120    pub trait VectorFindAnyEqCC<Other> {
2121        type Result;
2122        unsafe fn vec_find_any_eq_cc(self, other: Other) -> (Self::Result, i32);
2123    }
2124
2125    impl_vfae! { [cc VectorFindAnyEqCC vec_find_any_eq_cc] Eq vfaebs vfaehs vfaefs }
2126
2127    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2128    pub trait VectorFindAnyNeCC<Other> {
2129        type Result;
2130        unsafe fn vec_find_any_ne_cc(self, other: Other) -> (Self::Result, i32);
2131    }
2132
2133    impl_vfae! { [cc VectorFindAnyNeCC vec_find_any_ne_cc] Ne vfaebs vfaehs vfaefs }
2134
2135    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2136    pub trait VectorFindAnyEqIdxCC<Other> {
2137        type Result;
2138        unsafe fn vec_find_any_eq_idx_cc(self, other: Other) -> (Self::Result, i32);
2139    }
2140
2141    impl_vfae! { [idx_cc VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc] EqIdx vfaebs vfaehs vfaefs }
2142
2143    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2144    pub trait VectorFindAnyNeIdxCC<Other> {
2145        type Result;
2146        unsafe fn vec_find_any_ne_idx_cc(self, other: Other) -> (Self::Result, i32);
2147    }
2148
2149    impl_vfae! { [idx_cc VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc] NeIdx vfaebs vfaehs vfaefs }
2150
2151    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2152    pub trait VectorFindAnyEqOrZeroIdxCC<Other> {
2153        type Result;
2154        unsafe fn vec_find_any_eq_or_0_idx_cc(self, other: Other) -> (Self::Result, i32);
2155    }
2156
2157    impl_vfae! { [idx_cc VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc] EqIdx vfaezbs vfaezhs vfaezfs }
2158
2159    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2160    pub trait VectorFindAnyNeOrZeroIdxCC<Other> {
2161        type Result;
2162        unsafe fn vec_find_any_ne_or_0_idx_cc(self, other: Other) -> (Self::Result, i32);
2163    }
2164
2165    impl_vfae! { [idx_cc VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc] NeIdx vfaezbs vfaezhs vfaezfs }
2166
2167    #[inline]
2168    #[target_feature(enable = "vector")]
2169    #[cfg_attr(test, assert_instr(vl))]
2170    unsafe fn test_vector_load(offset: isize, ptr: *const i32) -> vector_signed_int {
2171        ptr.byte_offset(offset)
2172            .cast::<vector_signed_int>()
2173            .read_unaligned()
2174    }
2175
2176    #[inline]
2177    #[target_feature(enable = "vector")]
2178    #[cfg_attr(test, assert_instr(vst))]
2179    unsafe fn test_vector_store(vector: vector_signed_int, offset: isize, ptr: *mut i32) {
2180        ptr.byte_offset(offset)
2181            .cast::<vector_signed_int>()
2182            .write_unaligned(vector)
2183    }
2184
2185    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2186    pub trait VectorLoad: Sized {
2187        type ElementType;
2188
2189        #[inline]
2190        #[target_feature(enable = "vector")]
2191        unsafe fn vec_xl(offset: isize, ptr: *const Self::ElementType) -> Self {
2192            ptr.byte_offset(offset).cast::<Self>().read_unaligned()
2193        }
2194
2195        unsafe fn vec_load_len(ptr: *const Self::ElementType, byte_count: u32) -> Self;
2196
2197        unsafe fn vec_load_bndry<const BLOCK_BOUNDARY: u16>(
2198            ptr: *const Self::ElementType,
2199        ) -> MaybeUninit<Self>;
2200    }
2201
2202    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2203    pub trait VectorStore: Sized {
2204        type ElementType;
2205
2206        #[inline]
2207        #[target_feature(enable = "vector")]
2208        unsafe fn vec_xst(self, offset: isize, ptr: *mut Self::ElementType) {
2209            ptr.byte_offset(offset).cast::<Self>().write_unaligned(self)
2210        }
2211
2212        unsafe fn vec_store_len(self, ptr: *mut Self::ElementType, byte_count: u32);
2213    }
2214
2215    macro_rules! impl_load_store {
2216        ($($ty:ident)*) => {
2217            $(
2218                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2219                impl VectorLoad for t_t_l!($ty) {
2220                    type ElementType = $ty;
2221
2222                    #[inline]
2223                    #[target_feature(enable = "vector")]
2224                    unsafe fn vec_load_len(ptr: *const Self::ElementType, byte_count: u32) -> Self {
2225                        transmute(vll( byte_count, ptr.cast(),))
2226                    }
2227
2228                    #[inline]
2229                    #[target_feature(enable = "vector")]
2230                    unsafe fn vec_load_bndry<const BLOCK_BOUNDARY: u16>(ptr: *const Self::ElementType) -> MaybeUninit<Self> {
2231                        transmute(vlbb(ptr.cast(), const { validate_block_boundary(BLOCK_BOUNDARY) }))
2232                    }
2233
2234                }
2235
2236                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2237                impl VectorStore for t_t_l!($ty) {
2238                    type ElementType = $ty;
2239
2240                    #[inline]
2241                    #[target_feature(enable = "vector")]
2242                    unsafe fn vec_store_len(self, ptr: *mut Self::ElementType, byte_count: u32) {
2243                        vstl(transmute(self), byte_count, ptr.cast())
2244                    }
2245                }
2246            )*
2247        }
2248    }
2249
2250    impl_load_store! { i8 u8 i16 u16 i32 u32 i64 u64 f32 f64 }
2251
2252    #[inline]
2253    #[target_feature(enable = "vector")]
2254    #[cfg_attr(test, assert_instr(vll))]
2255    unsafe fn test_vec_load_len(ptr: *const i32, byte_count: u32) -> vector_signed_int {
2256        vector_signed_int::vec_load_len(ptr, byte_count)
2257    }
2258
2259    #[inline]
2260    #[target_feature(enable = "vector")]
2261    #[cfg_attr(test, assert_instr(vlbb))]
2262    unsafe fn test_vec_load_bndry(ptr: *const i32) -> MaybeUninit<vector_signed_int> {
2263        vector_signed_int::vec_load_bndry::<512>(ptr)
2264    }
2265
2266    #[inline]
2267    #[target_feature(enable = "vector")]
2268    #[cfg_attr(test, assert_instr(vstl))]
2269    unsafe fn test_vec_store_len(vector: vector_signed_int, ptr: *mut i32, byte_count: u32) {
2270        vector.vec_store_len(ptr, byte_count)
2271    }
2272
2273    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2274    pub trait VectorLoadPair: Sized {
2275        type ElementType;
2276
2277        unsafe fn vec_load_pair(a: Self::ElementType, b: Self::ElementType) -> Self;
2278    }
2279
2280    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2281    impl VectorLoadPair for vector_signed_long_long {
2282        type ElementType = i64;
2283
2284        #[inline]
2285        #[target_feature(enable = "vector")]
2286        unsafe fn vec_load_pair(a: i64, b: i64) -> Self {
2287            vector_signed_long_long([a, b])
2288        }
2289    }
2290
2291    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2292    impl VectorLoadPair for vector_unsigned_long_long {
2293        type ElementType = u64;
2294
2295        #[inline]
2296        #[target_feature(enable = "vector")]
2297        unsafe fn vec_load_pair(a: u64, b: u64) -> Self {
2298            vector_unsigned_long_long([a, b])
2299        }
2300    }
2301
2302    #[inline]
2303    #[target_feature(enable = "vector")]
2304    unsafe fn pack<T, const N: usize>(a: T, b: T) -> T {
2305        simd_shuffle(a, b, const { ShuffleMask::<N>::pack() })
2306    }
2307
2308    #[inline]
2309    #[target_feature(enable = "vector")]
2310    #[cfg_attr(test, assert_instr(vpkh))]
2311    unsafe fn vpkh(a: i16x8, b: i16x8) -> i8x16 {
2312        let a: i8x16 = transmute(a);
2313        let b: i8x16 = transmute(b);
2314        simd_shuffle(a, b, const { ShuffleMask::<16>::pack() })
2315    }
2316    #[inline]
2317    #[target_feature(enable = "vector")]
2318    #[cfg_attr(test, assert_instr(vpkf))]
2319    unsafe fn vpkf(a: i32x4, b: i32x4) -> i16x8 {
2320        let a: i16x8 = transmute(a);
2321        let b: i16x8 = transmute(b);
2322        simd_shuffle(a, b, const { ShuffleMask::<8>::pack() })
2323    }
2324    #[inline]
2325    #[target_feature(enable = "vector")]
2326    #[cfg_attr(test, assert_instr(vpkg))]
2327    unsafe fn vpkg(a: i64x2, b: i64x2) -> i32x4 {
2328        let a: i32x4 = transmute(a);
2329        let b: i32x4 = transmute(b);
2330        simd_shuffle(a, b, const { ShuffleMask::<4>::pack() })
2331    }
2332
2333    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2334    pub trait VectorPack<Other> {
2335        type Result;
2336        unsafe fn vec_pack(self, b: Other) -> Self::Result;
2337    }
2338
2339    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_signed_short, vector_signed_short) -> vector_signed_char }
2340    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2341    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_bool_short, vector_bool_short) -> vector_bool_char }
2342    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_signed_int, vector_signed_int) -> vector_signed_short }
2343    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2344    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_bool_int, vector_bool_int) -> vector_bool_short }
2345    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_signed_long_long, vector_signed_long_long) -> vector_signed_int }
2346    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2347    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_bool_long_long, vector_bool_long_long) -> vector_bool_int }
2348
2349    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2350    pub trait VectorPacks<Other> {
2351        type Result;
2352        unsafe fn vec_packs(self, b: Other) -> Self::Result;
2353    }
2354
2355    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/153655
2356    // Other targets can use a min/max for the saturation + a truncation.
2357
2358    impl_vec_trait! { [VectorPacks vec_packs] vpksh (vector_signed_short, vector_signed_short) -> vector_signed_char }
2359    impl_vec_trait! { [VectorPacks vec_packs] vpklsh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2360    impl_vec_trait! { [VectorPacks vec_packs] vpksf (vector_signed_int, vector_signed_int) -> vector_signed_short }
2361    impl_vec_trait! { [VectorPacks vec_packs] vpklsf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2362    impl_vec_trait! { [VectorPacks vec_packs] vpksg (vector_signed_long_long, vector_signed_long_long) -> vector_signed_int }
2363    impl_vec_trait! { [VectorPacks vec_packs] vpklsg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2364
2365    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2366    pub trait VectorPacksu<Other> {
2367        type Result;
2368        unsafe fn vec_packsu(self, b: Other) -> Self::Result;
2369    }
2370
2371    unsafe fn simd_smax<T: Copy>(a: T, b: T) -> T {
2372        simd_select::<T, T>(simd_gt::<T, T>(a, b), a, b)
2373    }
2374
2375    #[inline]
2376    #[target_feature(enable = "vector")]
2377    #[cfg_attr(test, assert_instr(vpklsh))]
2378    unsafe fn vpacksuh(a: vector_signed_short, b: vector_signed_short) -> vector_unsigned_char {
2379        vpklsh(
2380            simd_smax(a, vector_signed_short([0; 8])),
2381            simd_smax(b, vector_signed_short([0; 8])),
2382        )
2383    }
2384    #[inline]
2385    #[target_feature(enable = "vector")]
2386    #[cfg_attr(test, assert_instr(vpklsf))]
2387    unsafe fn vpacksuf(a: vector_signed_int, b: vector_signed_int) -> vector_unsigned_short {
2388        vpklsf(
2389            simd_smax(a, vector_signed_int([0; 4])),
2390            simd_smax(b, vector_signed_int([0; 4])),
2391        )
2392    }
2393    #[inline]
2394    #[target_feature(enable = "vector")]
2395    #[cfg_attr(test, assert_instr(vpklsg))]
2396    unsafe fn vpacksug(
2397        a: vector_signed_long_long,
2398        b: vector_signed_long_long,
2399    ) -> vector_unsigned_int {
2400        vpklsg(
2401            simd_smax(a, vector_signed_long_long([0; 2])),
2402            simd_smax(b, vector_signed_long_long([0; 2])),
2403        )
2404    }
2405
2406    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksuh (vector_signed_short, vector_signed_short) -> vector_unsigned_char }
2407    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2408    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksuf (vector_signed_int, vector_signed_int) -> vector_unsigned_short }
2409    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2410    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksug (vector_signed_long_long, vector_signed_long_long) -> vector_unsigned_int }
2411    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2412
2413    macro_rules! impl_vector_packs_cc {
2414        ($($intr:ident $ty:ident $outty:ident)*) => {
2415            $(
2416                #[inline]
2417                #[target_feature(enable = "vector")]
2418                #[cfg_attr(test, assert_instr($intr))]
2419                unsafe fn $intr(
2420                    a: $ty,
2421                    b: $ty,
2422                ) -> ($outty, i32) {
2423                    let PackedTuple { x, y } = super::$intr(a, b);
2424                    (x, y)
2425                }
2426
2427                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2428                impl VectorPacksCC for $ty {
2429                    type Result = $outty;
2430
2431                    #[inline]
2432                    #[target_feature(enable = "vector")]
2433                    unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32) {
2434                        $intr(self, b)
2435                    }
2436                }
2437            )*
2438        }
2439    }
2440
2441    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2442    pub trait VectorPacksCC {
2443        type Result;
2444        unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32);
2445    }
2446
2447    impl_vector_packs_cc! {
2448        vpkshs vector_signed_short vector_signed_char
2449        vpklshs vector_unsigned_short vector_unsigned_char
2450        vpksfs vector_signed_int vector_signed_short
2451        vpklsfs vector_unsigned_int vector_unsigned_short
2452        vpksgs vector_signed_long_long vector_signed_int
2453        vpklsgs vector_unsigned_long_long vector_unsigned_int
2454    }
2455
2456    macro_rules! impl_vector_packsu_cc {
2457        ($($intr:ident $ty:ident $outty:ident)*) => {
2458            $(
2459                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2460                impl VectorPacksuCC for $ty {
2461                    type Result = $outty;
2462
2463                    #[inline]
2464                    #[target_feature(enable = "vector")]
2465                    unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32) {
2466                        $intr(self, b)
2467                    }
2468                }
2469            )*
2470        }
2471    }
2472
2473    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2474    pub trait VectorPacksuCC {
2475        type Result;
2476        unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32);
2477    }
2478
2479    impl_vector_packsu_cc! {
2480        vpklshs vector_unsigned_short vector_unsigned_char
2481        vpklsfs vector_unsigned_int vector_unsigned_short
2482        vpklsgs vector_unsigned_long_long vector_unsigned_int
2483    }
2484
2485    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2486    pub trait VectorMadd {
2487        unsafe fn vec_madd(self, b: Self, c: Self) -> Self;
2488        unsafe fn vec_msub(self, b: Self, c: Self) -> Self;
2489    }
2490
2491    test_impl! { vfmasb (a: vector_float, b: vector_float, c: vector_float) -> vector_float [simd_fma, "vector-enhancements-1" vfmasb] }
2492    test_impl! { vfmadb (a: vector_double, b: vector_double, c: vector_double) -> vector_double [simd_fma, vfmadb] }
2493
2494    #[inline]
2495    unsafe fn simd_fms<T>(a: T, b: T, c: T) -> T {
2496        simd_fma(a, b, simd_neg(c))
2497    }
2498
2499    test_impl! { vfmssb (a: vector_float, b: vector_float, c: vector_float) -> vector_float [simd_fms, "vector-enhancements-1" vfmssb] }
2500    test_impl! { vfmsdb (a: vector_double, b: vector_double, c: vector_double) -> vector_double [simd_fms, vfmsdb] }
2501
2502    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2503    impl VectorMadd for vector_float {
2504        #[inline]
2505        #[target_feature(enable = "vector")]
2506        unsafe fn vec_madd(self, b: Self, c: Self) -> Self {
2507            vfmasb(self, b, c)
2508        }
2509
2510        #[inline]
2511        #[target_feature(enable = "vector")]
2512        unsafe fn vec_msub(self, b: Self, c: Self) -> Self {
2513            vfmssb(self, b, c)
2514        }
2515    }
2516
2517    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2518    impl VectorMadd for vector_double {
2519        #[inline]
2520        #[target_feature(enable = "vector")]
2521        unsafe fn vec_madd(self, b: Self, c: Self) -> Self {
2522            vfmadb(self, b, c)
2523        }
2524
2525        #[inline]
2526        #[target_feature(enable = "vector")]
2527        unsafe fn vec_msub(self, b: Self, c: Self) -> Self {
2528            vfmsdb(self, b, c)
2529        }
2530    }
2531
2532    macro_rules! impl_vec_unpack {
2533        ($mask:ident $instr:ident $src:ident $shuffled:ident $dst:ident $width:literal) => {
2534            #[inline]
2535            #[target_feature(enable = "vector")]
2536            #[cfg_attr(test, assert_instr($instr))]
2537            unsafe fn $instr(a: $src) -> $dst {
2538                simd_as(simd_shuffle::<_, _, $shuffled>(
2539                    a,
2540                    a,
2541                    const { ShuffleMask::<$width>::$mask() },
2542                ))
2543            }
2544        };
2545    }
2546
2547    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2548    pub trait VectorUnpackh {
2549        type Result;
2550        unsafe fn vec_unpackh(self) -> Self::Result;
2551    }
2552
2553    impl_vec_unpack!(unpack_high vuphb vector_signed_char i8x8 vector_signed_short 8);
2554    impl_vec_unpack!(unpack_high vuphh vector_signed_short i16x4 vector_signed_int 4);
2555    impl_vec_unpack!(unpack_high vuphf vector_signed_int i32x2 vector_signed_long_long 2);
2556
2557    impl_vec_unpack!(unpack_high vuplhb vector_unsigned_char u8x8 vector_unsigned_short 8);
2558    impl_vec_unpack!(unpack_high vuplhh vector_unsigned_short u16x4 vector_unsigned_int 4);
2559    impl_vec_unpack!(unpack_high vuplhf vector_unsigned_int u32x2 vector_unsigned_long_long 2);
2560
2561    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphb (vector_signed_char) -> vector_signed_short}
2562    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphh (vector_signed_short) -> vector_signed_int}
2563    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphf (vector_signed_int) -> vector_signed_long_long}
2564
2565    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhb (vector_unsigned_char) -> vector_unsigned_short}
2566    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhh (vector_unsigned_short) -> vector_unsigned_int}
2567    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhf (vector_unsigned_int) -> vector_unsigned_long_long}
2568
2569    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhb (vector_bool_char) -> vector_bool_short}
2570    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhh (vector_bool_short) -> vector_bool_int}
2571    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhf (vector_bool_int) -> vector_bool_long_long}
2572
2573    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2574    pub trait VectorUnpackl {
2575        type Result;
2576        unsafe fn vec_unpackl(self) -> Self::Result;
2577    }
2578
2579    // NOTE: `vuplh` is used for "unpack logical high", hence `vuplhw`.
2580    impl_vec_unpack!(unpack_low vuplb vector_signed_char i8x8 vector_signed_short 8);
2581    impl_vec_unpack!(unpack_low vuplhw vector_signed_short i16x4 vector_signed_int 4);
2582    impl_vec_unpack!(unpack_low vuplf vector_signed_int i32x2 vector_signed_long_long 2);
2583
2584    impl_vec_unpack!(unpack_low vupllb vector_unsigned_char u8x8 vector_unsigned_short 8);
2585    impl_vec_unpack!(unpack_low vupllh vector_unsigned_short u16x4 vector_unsigned_int 4);
2586    impl_vec_unpack!(unpack_low vupllf vector_unsigned_int u32x2 vector_unsigned_long_long 2);
2587
2588    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplb (vector_signed_char) -> vector_signed_short}
2589    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplhw (vector_signed_short) -> vector_signed_int}
2590    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplf (vector_signed_int) -> vector_signed_long_long}
2591
2592    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllb (vector_unsigned_char) -> vector_unsigned_short}
2593    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllh (vector_unsigned_short) -> vector_unsigned_int}
2594    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllf (vector_unsigned_int) -> vector_unsigned_long_long}
2595
2596    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllb (vector_bool_char) -> vector_bool_short}
2597    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllh (vector_bool_short) -> vector_bool_int}
2598    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllf (vector_bool_int) -> vector_bool_long_long}
2599
2600    test_impl! { vec_vavgb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vavgb, vavgb ] }
2601    test_impl! { vec_vavgh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vavgh, vavgh ] }
2602    test_impl! { vec_vavgf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vavgf, vavgf ] }
2603    test_impl! { vec_vavgg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [ vavgg, vavgg ] }
2604
2605    test_impl! { vec_vavglb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vavglb, vavglb ] }
2606    test_impl! { vec_vavglh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vavglh, vavglh ] }
2607    test_impl! { vec_vavglf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vavglf, vavglf ] }
2608    test_impl! { vec_vavglg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [ vavglg, vavglg ] }
2609
2610    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2611    pub trait VectorAvg<Other> {
2612        type Result;
2613        unsafe fn vec_avg(self, b: Other) -> Self::Result;
2614    }
2615
2616    impl_vec_trait! { [VectorAvg vec_avg] 2 (vec_vavglb, vec_vavgb, vec_vavglh, vec_vavgh, vec_vavglf, vec_vavgf, vec_vavglg, vec_vavgg) }
2617
2618    macro_rules! impl_mul {
2619        ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty) -> $r:ty) => {
2620            #[unstable(feature = "stdarch_s390x", issue = "135681")]
2621            impl $Trait<$r> for $a {
2622                #[inline]
2623                #[target_feature(enable = "vector")]
2624                unsafe fn $m(self, b: $b) -> $r {
2625                    $fun(transmute(self), transmute(b))
2626                }
2627            }
2628        };
2629        ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty, $c:ty) -> $r:ty) => {
2630            #[unstable(feature = "stdarch_s390x", issue = "135681")]
2631            impl $Trait for $a {
2632                type Result = $r;
2633                #[inline]
2634                #[target_feature(enable = "vector")]
2635                unsafe fn $m(self, b: $b, c: $c) -> $r {
2636                    $fun(self, b, c)
2637                }
2638            }
2639        };
2640    }
2641
2642    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2643    pub trait VectorMule<Result> {
2644        unsafe fn vec_mule(self, b: Self) -> Result;
2645    }
2646
2647    macro_rules! impl_vec_mul_even_odd {
2648        ($mask:ident $instr:ident $src:ident $shuffled:ident $dst:ident $width:literal) => {
2649            #[inline]
2650            #[target_feature(enable = "vector")]
2651            #[cfg_attr(test, assert_instr($instr))]
2652            unsafe fn $instr(a: $src, b: $src) -> $dst {
2653                let elems_a: $dst = simd_as(simd_shuffle::<_, _, $shuffled>(
2654                    a,
2655                    a, // this argument is ignored entirely.
2656                    const { ShuffleMask::<$width>::$mask() },
2657                ));
2658
2659                let elems_b: $dst = simd_as(simd_shuffle::<_, _, $shuffled>(
2660                    b,
2661                    b, // this argument is ignored entirely.
2662                    const { ShuffleMask::<$width>::$mask() },
2663                ));
2664
2665                simd_mul(elems_a, elems_b)
2666            }
2667        };
2668    }
2669
2670    impl_vec_mul_even_odd! { even vmeb vector_signed_char i8x8 vector_signed_short 8 }
2671    impl_vec_mul_even_odd! { even vmeh vector_signed_short i16x4 vector_signed_int 4 }
2672    impl_vec_mul_even_odd! { even vmef vector_signed_int i32x2 vector_signed_long_long 2 }
2673
2674    impl_vec_mul_even_odd! { even vmleb vector_unsigned_char u8x8 vector_unsigned_short 8 }
2675    impl_vec_mul_even_odd! { even vmleh vector_unsigned_short u16x4 vector_unsigned_int 4 }
2676    impl_vec_mul_even_odd! { even vmlef vector_unsigned_int u32x2 vector_unsigned_long_long 2 }
2677
2678    impl_mul!([VectorMule vec_mule] vmeb (vector_signed_char, vector_signed_char) -> vector_signed_short );
2679    impl_mul!([VectorMule vec_mule] vmeh (vector_signed_short, vector_signed_short) -> vector_signed_int);
2680    impl_mul!([VectorMule vec_mule] vmef (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
2681
2682    impl_mul!([VectorMule vec_mule] vmleb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2683    impl_mul!([VectorMule vec_mule] vmleh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2684    impl_mul!([VectorMule vec_mule] vmlef (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2685
2686    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2687    pub trait VectorMulo<Result> {
2688        unsafe fn vec_mulo(self, b: Self) -> Result;
2689    }
2690
2691    impl_vec_mul_even_odd! { odd vmob vector_signed_char i8x8 vector_signed_short 8 }
2692    impl_vec_mul_even_odd! { odd vmoh vector_signed_short i16x4 vector_signed_int 4 }
2693    impl_vec_mul_even_odd! { odd vmof vector_signed_int i32x2 vector_signed_long_long 2 }
2694
2695    impl_vec_mul_even_odd! { odd vmlob vector_unsigned_char u8x8 vector_unsigned_short 8 }
2696    impl_vec_mul_even_odd! { odd vmloh vector_unsigned_short u16x4 vector_unsigned_int 4 }
2697    impl_vec_mul_even_odd! { odd vmlof vector_unsigned_int u32x2 vector_unsigned_long_long 2 }
2698
2699    impl_mul!([VectorMulo vec_mulo] vmob (vector_signed_char, vector_signed_char) -> vector_signed_short );
2700    impl_mul!([VectorMulo vec_mulo] vmoh (vector_signed_short, vector_signed_short) -> vector_signed_int);
2701    impl_mul!([VectorMulo vec_mulo] vmof (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
2702
2703    impl_mul!([VectorMulo vec_mulo] vmlob (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2704    impl_mul!([VectorMulo vec_mulo] vmloh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2705    impl_mul!([VectorMulo vec_mulo] vmlof (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2706
2707    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2708    pub trait VectorMulh<Result> {
2709        unsafe fn vec_mulh(self, b: Self) -> Result;
2710    }
2711
2712    test_impl! { vec_vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vmhb, vmhb ] }
2713    test_impl! { vec_vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vmhh, vmhh ] }
2714    test_impl! { vec_vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vmhf, vmhf ] }
2715
2716    test_impl! { vec_vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vmlhb, vmlhb ] }
2717    test_impl! { vec_vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vmlhh, vmlhh ] }
2718    test_impl! { vec_vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vmlhf, vmlhf ] }
2719
2720    impl_mul!([VectorMulh vec_mulh] vec_vmhb (vector_signed_char, vector_signed_char) -> vector_signed_char);
2721    impl_mul!([VectorMulh vec_mulh] vec_vmhh (vector_signed_short, vector_signed_short) -> vector_signed_short);
2722    impl_mul!([VectorMulh vec_mulh] vec_vmhf (vector_signed_int, vector_signed_int) -> vector_signed_int);
2723
2724    impl_mul!([VectorMulh vec_mulh] vec_vmlhb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char);
2725    impl_mul!([VectorMulh vec_mulh] vec_vmlhh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2726    impl_mul!([VectorMulh vec_mulh] vec_vmlhf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int);
2727
2728    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2729    pub trait VectorMeadd {
2730        type Result;
2731        unsafe fn vec_meadd(self, b: Self, c: Self::Result) -> Self::Result;
2732    }
2733
2734    test_impl! { vec_vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaeb, vmaeb ] }
2735    test_impl! { vec_vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaeh, vmaeh ] }
2736    test_impl! { vec_vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaef, vmaef ] }
2737
2738    test_impl! { vec_vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmaleb, vmaleb ] }
2739    test_impl! { vec_vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaleh, vmaleh ] }
2740    test_impl! { vec_vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalef, vmalef ] }
2741
2742    impl_mul!([VectorMeadd vec_meadd] vec_vmaeb (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short );
2743    impl_mul!([VectorMeadd vec_meadd] vec_vmaeh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int);
2744    impl_mul!([VectorMeadd vec_meadd] vec_vmaef (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long );
2745
2746    impl_mul!([VectorMeadd vec_meadd] vec_vmaleb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short );
2747    impl_mul!([VectorMeadd vec_meadd] vec_vmaleh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int);
2748    impl_mul!([VectorMeadd vec_meadd] vec_vmalef (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long );
2749
2750    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2751    pub trait VectorMoadd {
2752        type Result;
2753        unsafe fn vec_moadd(self, b: Self, c: Self::Result) -> Self::Result;
2754    }
2755
2756    test_impl! { vec_vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaob, vmaob ] }
2757    test_impl! { vec_vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaoh, vmaoh ] }
2758    test_impl! { vec_vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaof, vmaof ] }
2759
2760    test_impl! { vec_vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmalob, vmalob ] }
2761    test_impl! { vec_vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaloh, vmaloh ] }
2762    test_impl! { vec_vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalof, vmalof ] }
2763
2764    impl_mul!([VectorMoadd vec_moadd] vec_vmaob (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short );
2765    impl_mul!([VectorMoadd vec_moadd] vec_vmaoh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int);
2766    impl_mul!([VectorMoadd vec_moadd] vec_vmaof (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long );
2767
2768    impl_mul!([VectorMoadd vec_moadd] vec_vmalob (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short );
2769    impl_mul!([VectorMoadd vec_moadd] vec_vmaloh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int);
2770    impl_mul!([VectorMoadd vec_moadd] vec_vmalof (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long );
2771
2772    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2773    pub trait VectorMhadd {
2774        type Result;
2775        unsafe fn vec_mhadd(self, b: Self, c: Self::Result) -> Self::Result;
2776    }
2777
2778    test_impl! { vec_vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [ vmahb, vmahb ] }
2779    test_impl! { vec_vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[ vmahh, vmahh ] }
2780    test_impl! { vec_vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [ vmahf, vmahf ] }
2781
2782    test_impl! { vec_vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [ vmalhb, vmalhb ] }
2783    test_impl! { vec_vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[ vmalhh, vmalhh ] }
2784    test_impl! { vec_vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [ vmalhf, vmalhf ] }
2785
2786    impl_mul!([VectorMhadd vec_mhadd] vec_vmahb (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char );
2787    impl_mul!([VectorMhadd vec_mhadd] vec_vmahh (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short);
2788    impl_mul!([VectorMhadd vec_mhadd] vec_vmahf (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int );
2789
2790    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char );
2791    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2792    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int );
2793
2794    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2795    pub trait VectorMladd {
2796        type Result;
2797        unsafe fn vec_mladd(self, b: Self, c: Self::Result) -> Self::Result;
2798    }
2799
2800    #[inline]
2801    #[target_feature(enable = "vector")]
2802    unsafe fn simd_mladd<T>(a: T, b: T, c: T) -> T {
2803        simd_add(simd_mul(a, b), c)
2804    }
2805
2806    test_impl! { vec_vmal_ib(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [simd_mladd, vmalb ] }
2807    test_impl! { vec_vmal_ih(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[simd_mladd, vmalhw ] }
2808    test_impl! { vec_vmal_if(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [simd_mladd, vmalf ] }
2809
2810    test_impl! { vec_vmal_ub(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [simd_mladd, vmalb ] }
2811    test_impl! { vec_vmal_uh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[simd_mladd, vmalhw ] }
2812    test_impl! { vec_vmal_uf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [simd_mladd, vmalf ] }
2813
2814    impl_mul!([VectorMladd vec_mladd] vec_vmal_ib (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char );
2815    impl_mul!([VectorMladd vec_mladd] vec_vmal_ih (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short);
2816    impl_mul!([VectorMladd vec_mladd] vec_vmal_if (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int );
2817
2818    impl_mul!([VectorMladd vec_mladd] vec_vmal_ub (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char );
2819    impl_mul!([VectorMladd vec_mladd] vec_vmal_uh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2820    impl_mul!([VectorMladd vec_mladd] vec_vmal_uf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int );
2821
2822    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2823    pub trait VectorGfmsum<Result> {
2824        unsafe fn vec_gfmsum(self, b: Self) -> Result;
2825    }
2826
2827    test_impl! { vec_vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vgfmb, vgfmb ] }
2828    test_impl! { vec_vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vgfmh, vgfmh] }
2829    test_impl! { vec_vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vgfmf, vgfmf ] }
2830
2831    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2832    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2833    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2834
2835    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2836    pub trait VectorGfmsumAccum {
2837        type Result;
2838        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result;
2839    }
2840
2841    test_impl! { vec_vgfmab(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vgfmab, vgfmab ] }
2842    test_impl! { vec_vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vgfmah, vgfmah] }
2843    test_impl! { vec_vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vgfmaf, vgfmaf ] }
2844
2845    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2846    impl VectorGfmsumAccum for vector_unsigned_char {
2847        type Result = vector_unsigned_short;
2848        #[inline]
2849        #[target_feature(enable = "vector")]
2850        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2851            vec_vgfmab(self, b, c)
2852        }
2853    }
2854    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2855    impl VectorGfmsumAccum for vector_unsigned_short {
2856        type Result = vector_unsigned_int;
2857        #[inline]
2858        #[target_feature(enable = "vector")]
2859        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2860            vec_vgfmah(self, b, c)
2861        }
2862    }
2863    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2864    impl VectorGfmsumAccum for vector_unsigned_int {
2865        type Result = vector_unsigned_long_long;
2866        #[inline]
2867        #[target_feature(enable = "vector")]
2868        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2869            vec_vgfmaf(self, b, c)
2870        }
2871    }
2872
2873    #[inline]
2874    #[target_feature(enable = "vector")]
2875    #[cfg_attr(test, assert_instr(vgef, D = 3))]
2876    unsafe fn vgef<const D: u32>(
2877        a: vector_unsigned_int,
2878        b: vector_unsigned_int,
2879        c: *const u32,
2880    ) -> vector_unsigned_int {
2881        static_assert_uimm_bits!(D, 2);
2882        let offset: u32 = simd_extract(b, D);
2883        let ptr = c.byte_add(offset as usize);
2884        let value = ptr.read();
2885        simd_insert(a, D, value)
2886    }
2887
2888    #[inline]
2889    #[target_feature(enable = "vector")]
2890    #[cfg_attr(test, assert_instr(vgeg, D = 1))]
2891    unsafe fn vgeg<const D: u32>(
2892        a: vector_unsigned_long_long,
2893        b: vector_unsigned_long_long,
2894        c: *const u64,
2895    ) -> vector_unsigned_long_long {
2896        static_assert_uimm_bits!(D, 1);
2897        let offset: u64 = simd_extract(b, D);
2898        let ptr = c.byte_add(offset as usize);
2899        let value = ptr.read();
2900        simd_insert(a, D, value)
2901    }
2902
2903    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2904    pub trait VectorGatherElement {
2905        type Element;
2906        type Offset;
2907        unsafe fn vec_gather_element<const D: u32>(
2908            self,
2909            b: Self::Offset,
2910            c: *const Self::Element,
2911        ) -> Self;
2912    }
2913
2914    macro_rules! impl_vec_gather_element {
2915        ($($instr:ident $ty:ident)*) => {
2916            $(
2917                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2918                impl VectorGatherElement for $ty {
2919                    type Element = l_t_t!($ty);
2920                    type Offset = t_u!($ty);
2921
2922                    #[inline]
2923                    #[target_feature(enable = "vector")]
2924                    unsafe fn vec_gather_element<const D: u32>(self, b: Self::Offset, c: *const Self::Element) -> Self {
2925                        transmute($instr::<D>(transmute(self), b, c.cast()))
2926                    }
2927                }
2928            )*
2929        }
2930    }
2931
2932    impl_vec_gather_element! {
2933        vgef vector_signed_int
2934        vgef vector_bool_int
2935        vgef vector_unsigned_int
2936
2937        vgeg vector_signed_long_long
2938        vgeg vector_bool_long_long
2939        vgeg vector_unsigned_long_long
2940
2941        vgef vector_float
2942        vgeg vector_double
2943    }
2944
2945    #[inline]
2946    #[target_feature(enable = "vector")]
2947    #[cfg_attr(test, assert_instr(vscef, D = 3))]
2948    unsafe fn vscef<const D: u32>(a: vector_unsigned_int, b: vector_unsigned_int, c: *mut u32) {
2949        static_assert_uimm_bits!(D, 2);
2950        let value = simd_extract(a, D);
2951        let offset: u32 = simd_extract(b, D);
2952        let ptr = c.byte_add(offset as usize);
2953        ptr.write(value);
2954    }
2955
2956    #[inline]
2957    #[target_feature(enable = "vector")]
2958    #[cfg_attr(test, assert_instr(vsceg, D = 1))]
2959    unsafe fn vsceg<const D: u32>(
2960        a: vector_unsigned_long_long,
2961        b: vector_unsigned_long_long,
2962        c: *mut u64,
2963    ) {
2964        static_assert_uimm_bits!(D, 1);
2965        let value = simd_extract(a, D);
2966        let offset: u64 = simd_extract(b, D);
2967        let ptr = c.byte_add(offset as usize);
2968        ptr.write(value);
2969    }
2970
2971    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2972    pub trait VectorScatterElement {
2973        type Element;
2974        type Offset;
2975        unsafe fn vec_scatter_element<const D: u32>(self, b: Self::Offset, c: *mut Self::Element);
2976    }
2977
2978    macro_rules! impl_vec_scatter_element {
2979        ($($instr:ident $ty:ident)*) => {
2980            $(
2981                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2982                impl VectorScatterElement for $ty {
2983                    type Element = l_t_t!($ty);
2984                    type Offset = t_u!($ty);
2985
2986                    #[inline]
2987                    #[target_feature(enable = "vector")]
2988                    unsafe fn vec_scatter_element<const D: u32>(self, b: Self::Offset, c: *mut Self::Element) {
2989                        $instr::<D>(transmute(self), b, c.cast())
2990                    }
2991                }
2992            )*
2993        }
2994    }
2995
2996    impl_vec_scatter_element! {
2997        vscef vector_signed_int
2998        vscef vector_bool_int
2999        vscef vector_unsigned_int
3000
3001        vsceg vector_signed_long_long
3002        vsceg vector_bool_long_long
3003        vsceg vector_unsigned_long_long
3004
3005        vscef vector_float
3006        vsceg vector_double
3007    }
3008
3009    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3010    pub trait VectorSel<Mask>: Sized {
3011        unsafe fn vec_sel(self, b: Self, c: Mask) -> Self;
3012    }
3013
3014    macro_rules! impl_vec_sel {
3015        ($($ty:ident)*) => {
3016            $(
3017                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3018                impl VectorSel<t_u!($ty)> for $ty {
3019                    #[inline]
3020                    #[target_feature(enable = "vector")]
3021                    unsafe fn vec_sel(self, b: Self, c: t_u!($ty)) -> Self {
3022                        let b = simd_and(transmute(b), c);
3023                        let a = simd_and(transmute(self), simd_xor(c, transmute(vector_signed_char([!0; 16]))));
3024                        transmute(simd_or(a, b))
3025                    }
3026                }
3027
3028                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3029                impl VectorSel<t_b!($ty)> for $ty {
3030                    #[inline]
3031                    #[target_feature(enable = "vector")]
3032                    unsafe fn vec_sel(self, b: Self, c: t_b!($ty)) -> Self {
3033                        // defer to the implementation with an unsigned mask
3034                        self.vec_sel(b, transmute::<_, t_u!($ty)>(c))
3035                    }
3036                }
3037            )*
3038        }
3039    }
3040
3041    impl_vec_sel! {
3042        vector_signed_char
3043        vector_signed_short
3044        vector_signed_int
3045        vector_signed_long_long
3046
3047        vector_unsigned_char
3048        vector_unsigned_short
3049        vector_unsigned_int
3050        vector_unsigned_long_long
3051
3052        vector_bool_char
3053        vector_bool_short
3054        vector_bool_int
3055        vector_bool_long_long
3056
3057        vector_float
3058        vector_double
3059    }
3060
3061    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3062    pub trait VectorFpTestDataClass {
3063        type Result;
3064        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32);
3065    }
3066
3067    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3068    impl VectorFpTestDataClass for vector_float {
3069        type Result = vector_bool_int;
3070
3071        #[inline]
3072        #[target_feature(enable = "vector")]
3073        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32) {
3074            let PackedTuple { x, y } = vftcisb(self, CLASS);
3075            (x, y)
3076        }
3077    }
3078
3079    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3080    impl VectorFpTestDataClass for vector_double {
3081        type Result = vector_bool_long_long;
3082
3083        #[inline]
3084        #[target_feature(enable = "vector")]
3085        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32) {
3086            let PackedTuple { x, y } = vftcidb(self, CLASS);
3087            (x, y)
3088        }
3089    }
3090
3091    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3092    pub trait VectorCompare {
3093        unsafe fn vec_all_lt(self, other: Self) -> i32;
3094        unsafe fn vec_all_le(self, other: Self) -> i32;
3095        unsafe fn vec_all_gt(self, other: Self) -> i32;
3096        unsafe fn vec_all_ge(self, other: Self) -> i32;
3097    }
3098
3099    // NOTE: this implementation is currently non-optimal, but it does work for floats even with
3100    // only `vector` enabled.
3101    //
3102    // - https://github.com/llvm/llvm-project/issues/129434
3103    // - https://github.com/llvm/llvm-project/issues/130424
3104    macro_rules! impl_vec_compare {
3105        ($($ty:ident)*) => {
3106            $(
3107                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3108                impl VectorCompare for $ty {
3109                    #[inline]
3110                    #[target_feature(enable = "vector")]
3111                    unsafe fn vec_all_lt(self, other: Self) -> i32 {
3112                        simd_reduce_all(simd_lt::<_, t_b!($ty)>(self, other)) as i32
3113                    }
3114                    #[inline]
3115                    #[target_feature(enable = "vector")]
3116                    unsafe fn vec_all_le(self, other: Self) -> i32 {
3117                        simd_reduce_all(simd_le::<_, t_b!($ty)>(self, other)) as i32
3118                    }
3119                    #[inline]
3120                    #[target_feature(enable = "vector")]
3121                    unsafe fn vec_all_gt(self, other: Self) -> i32 {
3122                        simd_reduce_all(simd_gt::<_, t_b!($ty)>(self, other)) as i32
3123                    }
3124                    #[inline]
3125                    #[target_feature(enable = "vector")]
3126                    unsafe fn vec_all_ge(self, other: Self) -> i32 {
3127                        simd_reduce_all(simd_ge::<_, t_b!($ty)>(self, other)) as i32
3128                    }
3129                }
3130            )*
3131        }
3132    }
3133
3134    impl_vec_compare! {
3135        vector_signed_char
3136        vector_unsigned_char
3137
3138        vector_signed_short
3139        vector_unsigned_short
3140
3141        vector_signed_int
3142        vector_unsigned_int
3143        vector_float
3144
3145        vector_signed_long_long
3146        vector_unsigned_long_long
3147        vector_double
3148    }
3149
3150    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3151    pub trait VectorTestMask {
3152        type Mask;
3153        unsafe fn vec_test_mask(self, other: Self::Mask) -> i32;
3154    }
3155
3156    macro_rules! impl_vec_test_mask {
3157        ($($instr:ident $ty:ident)*) => {
3158            $(
3159                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3160                impl VectorTestMask for $ty {
3161                    type Mask = t_u!($ty);
3162
3163                    #[inline]
3164                    #[target_feature(enable = "vector")]
3165                    unsafe fn vec_test_mask(self, other: Self::Mask) -> i32 {
3166                        vtm(transmute(self), transmute(other))
3167                    }
3168                }
3169            )*
3170        }
3171    }
3172
3173    impl_vec_test_mask! {
3174        vector_signed_char
3175        vector_signed_short
3176        vector_signed_int
3177        vector_signed_long_long
3178
3179        vector_unsigned_char
3180        vector_unsigned_short
3181        vector_unsigned_int
3182        vector_unsigned_long_long
3183
3184        vector_float
3185        vector_double
3186    }
3187
3188    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3189    pub trait VectorSearchString {
3190        unsafe fn vec_search_string_cc(
3191            self,
3192            b: Self,
3193            c: vector_unsigned_char,
3194        ) -> (vector_unsigned_char, i32);
3195
3196        unsafe fn vec_search_string_until_zero_cc(
3197            self,
3198            b: Self,
3199            c: vector_unsigned_char,
3200        ) -> (vector_unsigned_char, i32);
3201    }
3202
3203    macro_rules! impl_vec_search_string{
3204        ($($intr_s:ident $intr_sz:ident $ty:ident)*) => {
3205            $(
3206                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3207                impl VectorSearchString for $ty {
3208                    #[inline]
3209                    #[target_feature(enable = "vector-enhancements-2")]
3210                    unsafe fn vec_search_string_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) {
3211                        let PackedTuple { x,y } = $intr_s(transmute(self), transmute(b), c);
3212                        (x, y)
3213                    }
3214
3215                    #[inline]
3216                    #[target_feature(enable = "vector-enhancements-2")]
3217                    unsafe fn vec_search_string_until_zero_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) {
3218                        let PackedTuple { x,y } = $intr_sz(transmute(self), transmute(b), c);
3219                        (x, y)
3220                    }
3221                }
3222
3223            )*
3224        }
3225    }
3226
3227    impl_vec_search_string! {
3228        vstrsb vstrszb vector_signed_char
3229        vstrsb vstrszb vector_bool_char
3230        vstrsb vstrszb vector_unsigned_char
3231
3232        vstrsh vstrszh vector_signed_short
3233        vstrsh vstrszh vector_bool_short
3234        vstrsh vstrszh vector_unsigned_short
3235
3236        vstrsf vstrszf vector_signed_int
3237        vstrsf vstrszf vector_bool_int
3238        vstrsf vstrszf vector_unsigned_int
3239    }
3240
3241    #[inline]
3242    #[target_feature(enable = "vector")]
3243    #[cfg_attr(test, assert_instr(vcdgb))]
3244    pub unsafe fn vcdgb(a: vector_signed_long_long) -> vector_double {
3245        simd_as(a)
3246    }
3247
3248    #[inline]
3249    #[target_feature(enable = "vector")]
3250    #[cfg_attr(test, assert_instr(vcdlgb))]
3251    pub unsafe fn vcdlgb(a: vector_unsigned_long_long) -> vector_double {
3252        simd_as(a)
3253    }
3254
3255    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3256    pub trait VectorDouble {
3257        unsafe fn vec_double(self) -> vector_double;
3258    }
3259
3260    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3261    impl VectorDouble for vector_signed_long_long {
3262        #[inline]
3263        #[target_feature(enable = "vector")]
3264        unsafe fn vec_double(self) -> vector_double {
3265            vcdgb(self)
3266        }
3267    }
3268
3269    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3270    impl VectorDouble for vector_unsigned_long_long {
3271        #[inline]
3272        #[target_feature(enable = "vector")]
3273        unsafe fn vec_double(self) -> vector_double {
3274            vcdlgb(self)
3275        }
3276    }
3277
3278    #[inline]
3279    #[target_feature(enable = "vector")]
3280    #[cfg_attr(
3281        all(test, target_feature = "vector-enhancements-2"),
3282        assert_instr(vcefb)
3283    )]
3284    pub unsafe fn vcefb(a: vector_signed_int) -> vector_float {
3285        simd_as(a)
3286    }
3287
3288    #[inline]
3289    #[target_feature(enable = "vector")]
3290    #[cfg_attr(
3291        all(test, target_feature = "vector-enhancements-2"),
3292        assert_instr(vcelfb)
3293    )]
3294    pub unsafe fn vcelfb(a: vector_unsigned_int) -> vector_float {
3295        simd_as(a)
3296    }
3297
3298    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3299    pub trait VectorFloat {
3300        unsafe fn vec_float(self) -> vector_float;
3301    }
3302
3303    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3304    impl VectorFloat for vector_signed_int {
3305        #[inline]
3306        #[target_feature(enable = "vector")]
3307        unsafe fn vec_float(self) -> vector_float {
3308            vcefb(self)
3309        }
3310    }
3311
3312    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3313    impl VectorFloat for vector_unsigned_int {
3314        #[inline]
3315        #[target_feature(enable = "vector")]
3316        unsafe fn vec_float(self) -> vector_float {
3317            vcelfb(self)
3318        }
3319    }
3320
3321    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3322    pub trait VectorExtendSigned64 {
3323        unsafe fn vec_extend_s64(self) -> vector_signed_long_long;
3324    }
3325
3326    #[inline]
3327    #[target_feature(enable = "vector")]
3328    #[cfg_attr(test, assert_instr(vsegb))]
3329    pub unsafe fn vsegb(a: vector_signed_char) -> vector_signed_long_long {
3330        simd_as(simd_shuffle::<_, _, i8x2>(
3331            a,
3332            a,
3333            const { u32x2::from_array([7, 15]) },
3334        ))
3335    }
3336
3337    #[inline]
3338    #[target_feature(enable = "vector")]
3339    #[cfg_attr(test, assert_instr(vsegh))]
3340    pub unsafe fn vsegh(a: vector_signed_short) -> vector_signed_long_long {
3341        simd_as(simd_shuffle::<_, _, i16x2>(
3342            a,
3343            a,
3344            const { u32x2::from_array([3, 7]) },
3345        ))
3346    }
3347
3348    #[inline]
3349    #[target_feature(enable = "vector")]
3350    #[cfg_attr(test, assert_instr(vsegf))]
3351    pub unsafe fn vsegf(a: vector_signed_int) -> vector_signed_long_long {
3352        simd_as(simd_shuffle::<_, _, i32x2>(
3353            a,
3354            a,
3355            const { u32x2::from_array([1, 3]) },
3356        ))
3357    }
3358
3359    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3360    impl VectorExtendSigned64 for vector_signed_char {
3361        #[inline]
3362        #[target_feature(enable = "vector")]
3363        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3364            vsegb(self)
3365        }
3366    }
3367    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3368    impl VectorExtendSigned64 for vector_signed_short {
3369        #[inline]
3370        #[target_feature(enable = "vector")]
3371        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3372            vsegh(self)
3373        }
3374    }
3375    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3376    impl VectorExtendSigned64 for vector_signed_int {
3377        #[inline]
3378        #[target_feature(enable = "vector")]
3379        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3380            vsegf(self)
3381        }
3382    }
3383
3384    // NOTE: VectorSigned and VectorUnsigned make strong safety assumptions around floats.
3385    // This is what C provides, but even IBM does not clearly document these constraints.
3386    //
3387    // https://doc.rust-lang.org/std/intrinsics/simd/fn.simd_cast.html
3388
3389    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3390    pub trait VectorSigned {
3391        type Result;
3392        unsafe fn vec_signed(self) -> Self::Result;
3393    }
3394
3395    test_impl! { vcgsb (a: vector_float) -> vector_signed_int [simd_cast, "vector-enhancements-2" vcgsb] }
3396    test_impl! { vcgdb (a: vector_double) -> vector_signed_long_long [simd_cast, vcgdb] }
3397
3398    impl_vec_trait! { [VectorSigned vec_signed] vcgsb (vector_float) -> vector_signed_int }
3399    impl_vec_trait! { [VectorSigned vec_signed] vcgdb (vector_double) -> vector_signed_long_long }
3400
3401    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3402    pub trait VectorUnsigned {
3403        type Result;
3404        unsafe fn vec_unsigned(self) -> Self::Result;
3405    }
3406
3407    test_impl! { vclgsb (a: vector_float) -> vector_unsigned_int [simd_cast, "vector-enhancements-2" vclgsb] }
3408    test_impl! { vclgdb (a: vector_double) -> vector_unsigned_long_long [simd_cast, vclgdb] }
3409
3410    impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgsb (vector_float) -> vector_unsigned_int }
3411    impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgdb (vector_double) -> vector_unsigned_long_long }
3412
3413    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3414    pub trait VectorCopyUntilZero {
3415        unsafe fn vec_cp_until_zero(self) -> Self;
3416    }
3417
3418    test_impl! { vec_vistrb (a: vector_unsigned_char) -> vector_unsigned_char [vistrb, vistrb] }
3419    test_impl! { vec_vistrh (a: vector_unsigned_short) -> vector_unsigned_short [vistrh, vistrh] }
3420    test_impl! { vec_vistrf (a: vector_unsigned_int) -> vector_unsigned_int [vistrf, vistrf] }
3421
3422    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_signed_char) }
3423    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_bool_char) }
3424    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_unsigned_char) }
3425
3426    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_signed_short) }
3427    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_bool_short) }
3428    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_unsigned_short) }
3429
3430    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_signed_int) }
3431    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_bool_int) }
3432    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_unsigned_int) }
3433
3434    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3435    pub trait VectorCopyUntilZeroCC: Sized {
3436        unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32);
3437    }
3438
3439    test_impl! { vec_vistrbs (a: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32> [vistrbs, vistrbs] }
3440    test_impl! { vec_vistrhs (a: vector_unsigned_short) -> PackedTuple<vector_unsigned_short, i32> [vistrhs, vistrhs] }
3441    test_impl! { vec_vistrfs (a: vector_unsigned_int) -> PackedTuple<vector_unsigned_int, i32> [vistrfs, vistrfs] }
3442
3443    macro_rules! impl_vec_copy_until_zero_cc {
3444        ($($intr:ident $ty:ident)*) => {
3445            $(
3446                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3447                impl VectorCopyUntilZeroCC for $ty {
3448                    #[inline]
3449                    #[target_feature(enable = "vector")]
3450                    unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32) {
3451                        let PackedTuple { x,y } = $intr(transmute(self));
3452                        (transmute(x), y)
3453                    }
3454                }
3455
3456            )*
3457        }
3458    }
3459
3460    impl_vec_copy_until_zero_cc! {
3461        vec_vistrbs vector_signed_char
3462        vec_vistrbs vector_bool_char
3463        vec_vistrbs vector_unsigned_char
3464
3465        vec_vistrhs vector_signed_short
3466        vec_vistrhs vector_bool_short
3467        vec_vistrhs vector_unsigned_short
3468
3469        vec_vistrfs vector_signed_int
3470        vec_vistrfs vector_bool_int
3471        vec_vistrfs vector_unsigned_int
3472    }
3473
3474    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3475    pub trait VectorSrdb {
3476        unsafe fn vec_srdb<const C: u32>(self, b: Self) -> Self;
3477    }
3478
3479    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3480    pub trait VectorSld {
3481        unsafe fn vec_sld<const C: u32>(self, b: Self) -> Self;
3482
3483        unsafe fn vec_sldw<const C: u32>(self, b: Self) -> Self;
3484
3485        unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self;
3486    }
3487
3488    #[inline]
3489    #[target_feature(enable = "vector")]
3490    #[cfg_attr(test, assert_instr(vsldb))]
3491    unsafe fn test_vec_sld(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int {
3492        a.vec_sld::<13>(b)
3493    }
3494
3495    #[inline]
3496    #[target_feature(enable = "vector")]
3497    #[cfg_attr(test, assert_instr(vsldb))]
3498    unsafe fn test_vec_sldw(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int {
3499        a.vec_sldw::<3>(b)
3500    }
3501
3502    #[inline]
3503    #[target_feature(enable = "vector-enhancements-2")]
3504    #[cfg_attr(test, assert_instr(vsld))]
3505    unsafe fn test_vec_sldb(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int {
3506        a.vec_sldb::<7>(b)
3507    }
3508
3509    #[inline]
3510    #[target_feature(enable = "vector-enhancements-2")]
3511    #[cfg_attr(test, assert_instr(vsrd))]
3512    unsafe fn test_vec_srdb(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int {
3513        a.vec_srdb::<7>(b)
3514    }
3515
3516    macro_rules! impl_vec_sld {
3517        ($($ty:ident)*) => {
3518            $(
3519                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3520                impl VectorSld for $ty {
3521                    #[inline]
3522                    #[target_feature(enable = "vector")]
3523                    unsafe fn vec_sld<const C: u32>(self, b: Self) -> Self {
3524                        static_assert_uimm_bits!(C, 4);
3525                        transmute(u128::funnel_shl(transmute(self), transmute(b), C  * 8))
3526                    }
3527
3528                    #[inline]
3529                    #[target_feature(enable = "vector")]
3530                    unsafe fn vec_sldw<const C: u32>(self, b: Self) -> Self {
3531                        static_assert_uimm_bits!(C, 2);
3532                        transmute(u128::funnel_shl(transmute(self), transmute(b), C * 4 * 8))
3533                    }
3534
3535                    #[inline]
3536                    #[target_feature(enable = "vector-enhancements-2")]
3537                    unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self {
3538                        static_assert_uimm_bits!(C, 3);
3539                        transmute(u128::funnel_shl(transmute(self), transmute(b), C))
3540                    }
3541                }
3542
3543                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3544                impl VectorSrdb for $ty {
3545                    #[inline]
3546                    #[target_feature(enable = "vector-enhancements-2")]
3547                    unsafe fn vec_srdb<const C: u32>(self, b: Self) -> Self {
3548                        static_assert_uimm_bits!(C, 3);
3549                        transmute(vsrd(transmute(self), transmute(b), C))
3550                        // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129955#issuecomment-3207488190
3551                        // LLVM currently rewrites `fshr` to `fshl`, and the logic in the s390x
3552                        // backend cannot deal with that yet.
3553                        // #[link_name = "llvm.fshr.i128"] fn fshr_i128(a: u128, b: u128, c: u128) -> u128;
3554                        // transmute(fshr_i128(transmute(self), transmute(b), const { C as u128 }))
3555                    }
3556                }
3557            )*
3558        }
3559    }
3560
3561    impl_vec_sld! {
3562        vector_signed_char
3563        vector_bool_char
3564        vector_unsigned_char
3565
3566        vector_signed_short
3567        vector_bool_short
3568        vector_unsigned_short
3569
3570        vector_signed_int
3571        vector_bool_int
3572        vector_unsigned_int
3573
3574        vector_signed_long_long
3575        vector_bool_long_long
3576        vector_unsigned_long_long
3577
3578        vector_float
3579        vector_double
3580    }
3581
3582    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3583    pub trait VectorCompareRange: Sized {
3584        type Result;
3585
3586        unsafe fn vstrc<const IMM: u32>(self, b: Self, c: Self) -> Self::Result;
3587        unsafe fn vstrcz<const IMM: u32>(self, b: Self, c: Self) -> Self::Result;
3588        unsafe fn vstrcs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32);
3589        unsafe fn vstrczs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32);
3590    }
3591
3592    const fn validate_compare_range_imm(imm: u32) {
3593        if !matches!(imm, 0 | 4 | 8 | 12) {
3594            panic!("IMM needs to be one of 0, 4, 8, 12");
3595        }
3596    }
3597
3598    macro_rules! impl_compare_range {
3599        ($($ty:ident $vstrc:ident $vstrcs:ident $vstrcz:ident $vstrczs:ident)*) => {
3600            $(
3601                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3602                impl VectorCompareRange for $ty {
3603                    type Result = t_b!($ty);
3604
3605                    #[inline]
3606                    #[target_feature(enable = "vector")]
3607                    unsafe fn vstrc<const IMM: u32>(self, b: Self, c: Self) -> Self::Result {
3608                        const { validate_compare_range_imm };
3609                        $vstrc(self, b, c, IMM)
3610                    }
3611
3612                    #[inline]
3613                    #[target_feature(enable = "vector")]
3614                    unsafe fn vstrcz<const IMM: u32>(self, b: Self, c: Self) -> Self::Result {
3615                        const { validate_compare_range_imm };
3616                        $vstrcz(self, b, c, IMM)
3617                    }
3618
3619                    #[inline]
3620                    #[target_feature(enable = "vector")]
3621                    unsafe fn vstrcs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32) {
3622                        const { validate_compare_range_imm };
3623                        let PackedTuple { x, y } = $vstrcs(self, b, c, IMM);
3624                        (x,y)
3625                    }
3626
3627                    #[inline]
3628                    #[target_feature(enable = "vector")]
3629                    unsafe fn vstrczs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32) {
3630                        const { validate_compare_range_imm };
3631                        let PackedTuple { x, y } = $vstrczs(self, b, c, IMM);
3632                        (x,y)
3633                    }
3634                }
3635            )*
3636        }
3637    }
3638
3639    impl_compare_range! {
3640        vector_unsigned_char    vstrcb vstrcbs vstrczb vstrczbs
3641        vector_unsigned_short   vstrch vstrchs vstrczh vstrczhs
3642        vector_unsigned_int     vstrcf vstrcfs vstrczf vstrczfs
3643    }
3644
3645    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3646    pub trait VectorComparePredicate: Sized {
3647        type Result;
3648
3649        #[inline]
3650        #[target_feature(enable = "vector")]
3651        unsafe fn vec_cmpgt(self, other: Self) -> Self::Result {
3652            simd_gt(self, other)
3653        }
3654
3655        #[inline]
3656        #[target_feature(enable = "vector")]
3657        unsafe fn vec_cmpge(self, other: Self) -> Self::Result {
3658            simd_ge(self, other)
3659        }
3660
3661        #[inline]
3662        #[target_feature(enable = "vector")]
3663        unsafe fn vec_cmplt(self, other: Self) -> Self::Result {
3664            simd_lt(self, other)
3665        }
3666
3667        #[inline]
3668        #[target_feature(enable = "vector")]
3669        unsafe fn vec_cmple(self, other: Self) -> Self::Result {
3670            simd_le(self, other)
3671        }
3672    }
3673
3674    macro_rules! impl_compare_predicate {
3675        ($($ty:ident)*) => {
3676            $(
3677                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3678                impl VectorComparePredicate for $ty {
3679                    type Result = t_b!($ty);
3680                }
3681            )*
3682        }
3683    }
3684
3685    impl_compare_predicate! {
3686        vector_signed_char
3687        vector_unsigned_char
3688
3689        vector_signed_short
3690        vector_unsigned_short
3691
3692        vector_signed_int
3693        vector_unsigned_int
3694        vector_float
3695
3696        vector_signed_long_long
3697        vector_unsigned_long_long
3698        vector_double
3699    }
3700
3701    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3702    pub trait VectorEquality: Sized {
3703        type Result;
3704
3705        #[inline]
3706        #[target_feature(enable = "vector")]
3707        unsafe fn vec_cmpeq(self, other: Self) -> Self::Result {
3708            simd_eq(self, other)
3709        }
3710
3711        #[inline]
3712        #[target_feature(enable = "vector")]
3713        unsafe fn vec_cmpne(self, other: Self) -> Self::Result {
3714            simd_ne(self, other)
3715        }
3716    }
3717
3718    macro_rules! impl_compare_equality {
3719        ($($ty:ident)*) => {
3720            $(
3721                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3722                impl VectorEquality for $ty {
3723                    type Result = t_b!($ty);
3724                }
3725            )*
3726        }
3727    }
3728
3729    impl_compare_equality! {
3730        vector_bool_char
3731        vector_signed_char
3732        vector_unsigned_char
3733
3734        vector_bool_short
3735        vector_signed_short
3736        vector_unsigned_short
3737
3738        vector_bool_int
3739        vector_signed_int
3740        vector_unsigned_int
3741        vector_float
3742
3743        vector_bool_long_long
3744        vector_signed_long_long
3745        vector_unsigned_long_long
3746        vector_double
3747    }
3748
3749    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3750    pub trait VectorEqualityIdx: Sized {
3751        type Result;
3752
3753        unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result;
3754        unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result;
3755
3756        unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32);
3757        unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32);
3758
3759        unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result;
3760        unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result;
3761
3762        unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32);
3763        unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32);
3764    }
3765
3766    macro_rules! impl_compare_equality_idx {
3767        ($($ty:ident $ret:ident
3768                $cmpeq:ident $cmpne:ident
3769                $cmpeq_or_0:ident $cmpne_or_0:ident
3770                $cmpeq_cc:ident $cmpne_cc:ident
3771                $cmpeq_or_0_cc:ident $cmpne_or_0_cc:ident
3772        )*) => {
3773            $(
3774                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3775                impl VectorEqualityIdx for $ty {
3776                    type Result = $ret;
3777
3778                    #[inline]
3779                    #[target_feature(enable = "vector")]
3780                    unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result {
3781                        transmute($cmpeq(transmute(self), transmute(other)))
3782                    }
3783
3784                    #[inline]
3785                    #[target_feature(enable = "vector")]
3786                    unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result {
3787                        transmute($cmpne(transmute(self), transmute(other)))
3788                    }
3789
3790                    #[inline]
3791                    #[target_feature(enable = "vector")]
3792                    unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result {
3793                        transmute($cmpeq_or_0(transmute(self), transmute(other)))
3794                    }
3795
3796                    #[inline]
3797                    #[target_feature(enable = "vector")]
3798                    unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result {
3799                        transmute($cmpne_or_0(transmute(self), transmute(other)))
3800                    }
3801
3802                    #[inline]
3803                    #[target_feature(enable = "vector")]
3804                    unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32) {
3805                        let PackedTuple { x, y } = $cmpeq_cc(transmute(self), transmute(other));
3806                        (transmute(x), y)
3807                    }
3808
3809                    #[inline]
3810                    #[target_feature(enable = "vector")]
3811                    unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32) {
3812                        let PackedTuple { x, y } = $cmpne_cc(transmute(self), transmute(other));
3813                        (transmute(x),y)
3814                    }
3815
3816                    #[inline]
3817                    #[target_feature(enable = "vector")]
3818                    unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) {
3819                        let PackedTuple { x, y } = $cmpeq_or_0_cc(transmute(self), transmute(other));
3820                        (transmute(x), y)
3821                    }
3822
3823                    #[inline]
3824                    #[target_feature(enable = "vector")]
3825                    unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) {
3826                        let PackedTuple { x, y } = $cmpne_or_0_cc(transmute(self), transmute(other));
3827                        (transmute(x),y)
3828                    }
3829                }
3830            )*
3831        }
3832    }
3833
3834    impl_compare_equality_idx! {
3835        vector_signed_char vector_signed_char               vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3836        vector_bool_char vector_unsigned_char               vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3837        vector_unsigned_char vector_unsigned_char           vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3838        vector_signed_short vector_signed_short             vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3839        vector_bool_short  vector_unsigned_short            vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3840        vector_unsigned_short vector_unsigned_short         vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3841        vector_signed_int vector_signed_int                 vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3842        vector_bool_int  vector_unsigned_int                vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3843        vector_unsigned_int vector_unsigned_int             vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3844    }
3845
3846    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3847    pub trait VectorExtract {
3848        type ElementType;
3849
3850        unsafe fn vec_extract(a: Self, b: i32) -> Self::ElementType;
3851    }
3852
3853    #[inline]
3854    #[target_feature(enable = "vector")]
3855    #[cfg_attr(test, assert_instr(vlgvb))]
3856    unsafe fn vlgvb(a: vector_unsigned_char, b: i32) -> u8 {
3857        simd_extract_dyn(a, b as u32 % 16)
3858    }
3859
3860    #[inline]
3861    #[target_feature(enable = "vector")]
3862    #[cfg_attr(test, assert_instr(vlgvh))]
3863    unsafe fn vlgvh(a: vector_unsigned_short, b: i32) -> u16 {
3864        simd_extract_dyn(a, b as u32 % 8)
3865    }
3866
3867    #[inline]
3868    #[target_feature(enable = "vector")]
3869    #[cfg_attr(test, assert_instr(vlgvf))]
3870    unsafe fn vlgvf(a: vector_unsigned_int, b: i32) -> u32 {
3871        simd_extract_dyn(a, b as u32 % 4)
3872    }
3873
3874    #[inline]
3875    #[target_feature(enable = "vector")]
3876    #[cfg_attr(test, assert_instr(vlgvg))]
3877    unsafe fn vlgvg(a: vector_unsigned_long_long, b: i32) -> u64 {
3878        simd_extract_dyn(a, b as u32 % 2)
3879    }
3880
3881    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3882    pub trait VectorInsert {
3883        type ElementType;
3884
3885        unsafe fn vec_insert(a: Self::ElementType, b: Self, c: i32) -> Self;
3886    }
3887
3888    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3889    pub trait VectorPromote: Sized {
3890        type ElementType;
3891
3892        unsafe fn vec_promote(a: Self::ElementType, b: i32) -> MaybeUninit<Self>;
3893    }
3894
3895    #[inline]
3896    #[target_feature(enable = "vector")]
3897    #[cfg_attr(test, assert_instr(vlvgb))]
3898    unsafe fn vlvgb(a: u8, b: vector_unsigned_char, c: i32) -> vector_unsigned_char {
3899        simd_insert_dyn(b, c as u32 % 16, a)
3900    }
3901
3902    #[inline]
3903    #[target_feature(enable = "vector")]
3904    #[cfg_attr(test, assert_instr(vlvgh))]
3905    unsafe fn vlvgh(a: u16, b: vector_unsigned_short, c: i32) -> vector_unsigned_short {
3906        simd_insert_dyn(b, c as u32 % 8, a)
3907    }
3908
3909    #[inline]
3910    #[target_feature(enable = "vector")]
3911    #[cfg_attr(test, assert_instr(vlvgf))]
3912    unsafe fn vlvgf(a: u32, b: vector_unsigned_int, c: i32) -> vector_unsigned_int {
3913        simd_insert_dyn(b, c as u32 % 4, a)
3914    }
3915
3916    #[inline]
3917    #[target_feature(enable = "vector")]
3918    #[cfg_attr(test, assert_instr(vlvgg))]
3919    unsafe fn vlvgg(a: u64, b: vector_unsigned_long_long, c: i32) -> vector_unsigned_long_long {
3920        simd_insert_dyn(b, c as u32 % 2, a)
3921    }
3922
3923    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3924    pub trait VectorInsertAndZero {
3925        type ElementType;
3926
3927        unsafe fn vec_insert_and_zero(a: *const Self::ElementType) -> Self;
3928    }
3929
3930    #[inline]
3931    #[target_feature(enable = "vector")]
3932    #[cfg_attr(test, assert_instr(vllezb))]
3933    unsafe fn vllezb(x: *const u8) -> vector_unsigned_char {
3934        vector_unsigned_char([0, 0, 0, 0, 0, 0, 0, *x, 0, 0, 0, 0, 0, 0, 0, 0])
3935    }
3936
3937    #[inline]
3938    #[target_feature(enable = "vector")]
3939    #[cfg_attr(test, assert_instr(vllezh))]
3940    unsafe fn vllezh(x: *const u16) -> vector_unsigned_short {
3941        vector_unsigned_short([0, 0, 0, *x, 0, 0, 0, 0])
3942    }
3943
3944    #[inline]
3945    #[target_feature(enable = "vector")]
3946    #[cfg_attr(test, assert_instr(vllezf))]
3947    unsafe fn vllezf(x: *const u32) -> vector_unsigned_int {
3948        vector_unsigned_int([0, *x, 0, 0])
3949    }
3950
3951    #[inline]
3952    #[target_feature(enable = "vector")]
3953    #[cfg_attr(test, assert_instr(vllezg))]
3954    unsafe fn vllezg(x: *const u64) -> vector_unsigned_long_long {
3955        vector_unsigned_long_long([*x, 0])
3956    }
3957
3958    macro_rules! impl_extract_insert {
3959        ($($ty:ident $extract_intr:ident $insert_intr:ident $insert_and_zero_intr:ident)*) => {
3960            $(
3961                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3962                impl VectorExtract for $ty {
3963                    type ElementType = l_t_t!($ty);
3964
3965                    #[inline]
3966                    #[target_feature(enable = "vector")]
3967                    unsafe fn vec_extract(a: Self, b: i32) -> Self::ElementType {
3968                        transmute($extract_intr(transmute(a), b))
3969                    }
3970                }
3971
3972                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3973                impl VectorInsert for $ty {
3974                    type ElementType = l_t_t!($ty);
3975
3976                    #[inline]
3977                    #[target_feature(enable = "vector")]
3978                    unsafe fn vec_insert(a: Self::ElementType, b: Self, c: i32) -> Self {
3979                        transmute($insert_intr(transmute(a), transmute(b), c))
3980                    }
3981                }
3982
3983                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3984                impl VectorInsertAndZero for $ty {
3985                    type ElementType = l_t_t!($ty);
3986
3987                    #[inline]
3988                    #[target_feature(enable = "vector")]
3989                    unsafe fn vec_insert_and_zero(a: *const Self::ElementType) -> Self {
3990                        transmute($insert_and_zero_intr(a.cast()))
3991                    }
3992                }
3993
3994                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3995                impl VectorPromote for $ty {
3996                    type ElementType = l_t_t!($ty);
3997
3998                    #[inline]
3999                    #[target_feature(enable = "vector")]
4000                    unsafe fn vec_promote(a: Self::ElementType, c: i32) -> MaybeUninit<Self> {
4001                        // Rust does not currently support `MaybeUninit` element types to simd
4002                        // vectors. In C/LLVM that is allowed (using poison values). So rust will
4003                        // use an extra instruction to zero the memory.
4004                        let b = MaybeUninit::<$ty>::zeroed();
4005                        MaybeUninit::new(transmute($insert_intr(transmute(a), transmute(b), c)))
4006                    }
4007                }
4008            )*
4009        }
4010
4011    }
4012
4013    impl_extract_insert! {
4014        vector_signed_char          vlgvb vlvgb vllezb
4015        vector_unsigned_char        vlgvb vlvgb vllezb
4016        vector_signed_short         vlgvh vlvgh vllezh
4017        vector_unsigned_short       vlgvh vlvgh vllezh
4018        vector_signed_int           vlgvf vlvgf vllezf
4019        vector_unsigned_int         vlgvf vlvgf vllezf
4020        vector_signed_long_long     vlgvg vlvgg vllezg
4021        vector_unsigned_long_long   vlgvg vlvgg vllezg
4022        vector_float                vlgvf vlvgf vllezf
4023        vector_double               vlgvg vlvgg vllezg
4024    }
4025}
4026
4027/// Load Count to Block Boundary
4028#[inline]
4029#[target_feature(enable = "vector")]
4030#[unstable(feature = "stdarch_s390x", issue = "135681")]
4031#[cfg_attr(test, assert_instr(lcbb, BLOCK_BOUNDARY = 512))]
4032unsafe fn __lcbb<const BLOCK_BOUNDARY: u16>(ptr: *const u8) -> u32 {
4033    lcbb(ptr, const { validate_block_boundary(BLOCK_BOUNDARY) })
4034}
4035
4036/// Vector Add
4037#[inline]
4038#[target_feature(enable = "vector")]
4039#[unstable(feature = "stdarch_s390x", issue = "135681")]
4040pub unsafe fn vec_add<T: sealed::VectorAdd<U>, U>(a: T, b: U) -> T::Result {
4041    a.vec_add(b)
4042}
4043
4044/// Vector Subtract
4045#[inline]
4046#[target_feature(enable = "vector")]
4047#[unstable(feature = "stdarch_s390x", issue = "135681")]
4048pub unsafe fn vec_sub<T: sealed::VectorSub<U>, U>(a: T, b: U) -> T::Result {
4049    a.vec_sub(b)
4050}
4051
4052/// Vector Multiply
4053///
4054/// ## Purpose
4055/// Compute the products of corresponding elements of two vectors.
4056///
4057/// ## Result value
4058/// Each element of r receives the product of the corresponding elements of a and b.
4059#[inline]
4060#[target_feature(enable = "vector")]
4061#[unstable(feature = "stdarch_s390x", issue = "135681")]
4062pub unsafe fn vec_mul<T: sealed::VectorMul>(a: T, b: T) -> T {
4063    a.vec_mul(b)
4064}
4065
4066/// Vector Count Leading Zeros
4067#[inline]
4068#[target_feature(enable = "vector")]
4069#[unstable(feature = "stdarch_s390x", issue = "135681")]
4070pub unsafe fn vec_cntlz<T: sealed::CountBits>(a: T) -> T::Result {
4071    a.vec_cntlz()
4072}
4073
4074/// Vector Count Trailing Zeros
4075#[inline]
4076#[target_feature(enable = "vector")]
4077#[unstable(feature = "stdarch_s390x", issue = "135681")]
4078pub unsafe fn vec_cnttz<T: sealed::CountBits>(a: T) -> T::Result {
4079    a.vec_cnttz()
4080}
4081
4082/// Vector Population Count
4083///
4084/// Computes the population count (number of set bits) in each element of the input.
4085#[inline]
4086#[target_feature(enable = "vector")]
4087#[unstable(feature = "stdarch_s390x", issue = "135681")]
4088pub unsafe fn vec_popcnt<T: sealed::CountBits>(a: T) -> T::Result {
4089    a.vec_popcnt()
4090}
4091
4092/// Vector Maximum
4093#[inline]
4094#[target_feature(enable = "vector")]
4095#[unstable(feature = "stdarch_s390x", issue = "135681")]
4096pub unsafe fn vec_max<T: sealed::VectorMax<U>, U>(a: T, b: U) -> T::Result {
4097    a.vec_max(b)
4098}
4099
4100/// Vector  Minimum
4101#[inline]
4102#[target_feature(enable = "vector")]
4103#[unstable(feature = "stdarch_s390x", issue = "135681")]
4104pub unsafe fn vec_min<T: sealed::VectorMin<U>, U>(a: T, b: U) -> T::Result {
4105    a.vec_min(b)
4106}
4107
4108/// Vector Absolute
4109#[inline]
4110#[target_feature(enable = "vector")]
4111#[unstable(feature = "stdarch_s390x", issue = "135681")]
4112pub unsafe fn vec_abs<T: sealed::VectorAbs>(a: T) -> T {
4113    a.vec_abs()
4114}
4115
4116/// Vector Negative Absolute
4117#[inline]
4118#[target_feature(enable = "vector")]
4119#[unstable(feature = "stdarch_s390x", issue = "135681")]
4120pub unsafe fn vec_nabs<T: sealed::VectorNabs>(a: T) -> T {
4121    a.vec_nabs()
4122}
4123
4124/// Vector Negative Multiply Add
4125#[inline]
4126#[target_feature(enable = "vector")]
4127#[unstable(feature = "stdarch_s390x", issue = "135681")]
4128pub unsafe fn vec_nmadd<T: sealed::VectorNmadd>(a: T, b: T, c: T) -> T {
4129    a.vec_nmadd(b, c)
4130}
4131
4132/// Vector Negative Multiply Subtract
4133#[inline]
4134#[target_feature(enable = "vector")]
4135#[unstable(feature = "stdarch_s390x", issue = "135681")]
4136pub unsafe fn vec_nmsub<T: sealed::VectorNmsub>(a: T, b: T, c: T) -> T {
4137    a.vec_nmsub(b, c)
4138}
4139
4140/// Vector Square Root
4141#[inline]
4142#[target_feature(enable = "vector")]
4143#[unstable(feature = "stdarch_s390x", issue = "135681")]
4144pub unsafe fn vec_sqrt<T: sealed::VectorSqrt>(a: T) -> T {
4145    a.vec_sqrt()
4146}
4147
4148/// Vector Splat
4149#[inline]
4150#[target_feature(enable = "vector")]
4151#[unstable(feature = "stdarch_s390x", issue = "135681")]
4152pub unsafe fn vec_splat<T: sealed::VectorSplat, const IMM: u32>(a: T) -> T {
4153    a.vec_splat::<IMM>()
4154}
4155
4156/// Vector Splats
4157#[inline]
4158#[target_feature(enable = "vector")]
4159#[unstable(feature = "stdarch_s390x", issue = "135681")]
4160pub unsafe fn vec_splats<T: sealed::VectorSplats<U>, U>(a: T) -> U {
4161    a.vec_splats()
4162}
4163
4164/// Vector AND
4165#[inline]
4166#[target_feature(enable = "vector")]
4167#[unstable(feature = "stdarch_s390x", issue = "135681")]
4168pub unsafe fn vec_and<T: sealed::VectorAnd<U>, U>(a: T, b: U) -> T::Result {
4169    a.vec_and(b)
4170}
4171
4172/// Vector OR
4173#[inline]
4174#[target_feature(enable = "vector")]
4175#[unstable(feature = "stdarch_s390x", issue = "135681")]
4176pub unsafe fn vec_or<T: sealed::VectorOr<U>, U>(a: T, b: U) -> T::Result {
4177    a.vec_or(b)
4178}
4179
4180/// Vector XOR
4181#[inline]
4182#[target_feature(enable = "vector")]
4183#[unstable(feature = "stdarch_s390x", issue = "135681")]
4184pub unsafe fn vec_xor<T: sealed::VectorXor<U>, U>(a: T, b: U) -> T::Result {
4185    a.vec_xor(b)
4186}
4187
4188/// Vector NOR
4189#[inline]
4190#[target_feature(enable = "vector")]
4191#[unstable(feature = "stdarch_s390x", issue = "135681")]
4192pub unsafe fn vec_nor<T: sealed::VectorNor<U>, U>(a: T, b: U) -> T::Result {
4193    a.vec_nor(b)
4194}
4195
4196/// Vector NAND
4197#[inline]
4198#[target_feature(enable = "vector")]
4199#[unstable(feature = "stdarch_s390x", issue = "135681")]
4200pub unsafe fn vec_nand<T: sealed::VectorNand<U>, U>(a: T, b: U) -> T::Result {
4201    a.vec_nand(b)
4202}
4203
4204/// Vector XNOR
4205#[inline]
4206#[target_feature(enable = "vector")]
4207#[unstable(feature = "stdarch_s390x", issue = "135681")]
4208pub unsafe fn vec_eqv<T: sealed::VectorEqv<U>, U>(a: T, b: U) -> T::Result {
4209    a.vec_eqv(b)
4210}
4211
4212/// Vector ANDC
4213#[inline]
4214#[target_feature(enable = "vector")]
4215#[unstable(feature = "stdarch_s390x", issue = "135681")]
4216pub unsafe fn vec_andc<T: sealed::VectorAndc<U>, U>(a: T, b: U) -> T::Result {
4217    a.vec_andc(b)
4218}
4219
4220/// Vector OR with Complement
4221///
4222/// ## Purpose
4223/// Performs a bitwise OR of the first vector with the bitwise-complemented second vector.
4224///
4225/// ## Result value
4226/// r is the bitwise OR of a and the bitwise complement of b.
4227#[inline]
4228#[target_feature(enable = "vector")]
4229#[unstable(feature = "stdarch_s390x", issue = "135681")]
4230pub unsafe fn vec_orc<T: sealed::VectorOrc<U>, U>(a: T, b: U) -> T::Result {
4231    a.vec_orc(b)
4232}
4233
4234/// Vector Floor
4235#[inline]
4236#[target_feature(enable = "vector")]
4237#[unstable(feature = "stdarch_s390x", issue = "135681")]
4238pub unsafe fn vec_floor<T: sealed::VectorFloor>(a: T) -> T {
4239    a.vec_floor()
4240}
4241
4242/// Vector Ceil
4243#[inline]
4244#[target_feature(enable = "vector")]
4245#[unstable(feature = "stdarch_s390x", issue = "135681")]
4246pub unsafe fn vec_ceil<T: sealed::VectorCeil>(a: T) -> T {
4247    a.vec_ceil()
4248}
4249
4250/// Vector Truncate
4251///
4252/// Returns a vector containing the truncated values of the corresponding elements of the given vector.
4253/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value.
4254#[inline]
4255#[target_feature(enable = "vector")]
4256#[unstable(feature = "stdarch_s390x", issue = "135681")]
4257pub unsafe fn vec_trunc<T: sealed::VectorTrunc>(a: T) -> T {
4258    a.vec_trunc()
4259}
4260
4261/// Vector Round
4262///
4263/// Returns a vector containing the rounded values to the nearest representable floating-point integer,
4264/// using IEEE round-to-nearest rounding, of the corresponding elements of the given vector
4265#[inline]
4266#[target_feature(enable = "vector")]
4267#[unstable(feature = "stdarch_s390x", issue = "135681")]
4268pub unsafe fn vec_round<T: sealed::VectorRound>(a: T) -> T {
4269    a.vec_round()
4270}
4271
4272/// Vector Round to Current
4273///
4274/// Returns a vector by using the current rounding mode to round every
4275/// floating-point element in the given vector to integer.
4276#[inline]
4277#[target_feature(enable = "vector")]
4278#[unstable(feature = "stdarch_s390x", issue = "135681")]
4279pub unsafe fn vec_roundc<T: sealed::VectorRoundc>(a: T) -> T {
4280    a.vec_roundc()
4281}
4282
4283/// Vector Round toward Negative Infinity
4284///
4285/// Returns a vector containing the largest representable floating-point integral values less
4286/// than or equal to the values of the corresponding elements of the given vector.
4287#[inline]
4288#[target_feature(enable = "vector")]
4289#[unstable(feature = "stdarch_s390x", issue = "135681")]
4290pub unsafe fn vec_roundm<T: sealed::VectorFloor>(a: T) -> T {
4291    // the IBM docs note
4292    //
4293    // > vec_roundm provides the same functionality as vec_floor, except that vec_roundz would not trigger the IEEE-inexact exception.
4294    //
4295    // but in practice `vec_floor` also does not trigger that exception, so both are equivalent
4296    a.vec_floor()
4297}
4298
4299/// Vector Round toward Positive Infinity
4300///
4301/// Returns a vector containing the smallest representable floating-point integral values greater
4302/// than or equal to the values of the corresponding elements of the given vector.
4303#[inline]
4304#[target_feature(enable = "vector")]
4305#[unstable(feature = "stdarch_s390x", issue = "135681")]
4306pub unsafe fn vec_roundp<T: sealed::VectorCeil>(a: T) -> T {
4307    // the IBM docs note
4308    //
4309    // > vec_roundp provides the same functionality as vec_ceil, except that vec_roundz would not trigger the IEEE-inexact exception.
4310    //
4311    // but in practice `vec_ceil` also does not trigger that exception, so both are equivalent
4312    a.vec_ceil()
4313}
4314
4315/// Vector Round toward Zero
4316///
4317/// Returns a vector containing the truncated values of the corresponding elements of the given vector.
4318/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value.
4319#[inline]
4320#[target_feature(enable = "vector")]
4321#[unstable(feature = "stdarch_s390x", issue = "135681")]
4322pub unsafe fn vec_roundz<T: sealed::VectorTrunc>(a: T) -> T {
4323    // the IBM docs note
4324    //
4325    // > vec_roundz provides the same functionality as vec_trunc, except that vec_roundz would not trigger the IEEE-inexact exception.
4326    //
4327    // but in practice `vec_trunc` also does not trigger that exception, so both are equivalent
4328    a.vec_trunc()
4329}
4330
4331/// Vector Round to Integer
4332///
4333/// Returns a vector by using the current rounding mode to round every floating-point element in the given vector to integer.
4334#[inline]
4335#[target_feature(enable = "vector")]
4336#[unstable(feature = "stdarch_s390x", issue = "135681")]
4337pub unsafe fn vec_rint<T: sealed::VectorRint>(a: T) -> T {
4338    a.vec_rint()
4339}
4340
4341/// Vector Average
4342#[inline]
4343#[target_feature(enable = "vector")]
4344#[unstable(feature = "stdarch_s390x", issue = "135681")]
4345pub unsafe fn vec_avg<T: sealed::VectorAvg<U>, U>(a: T, b: U) -> T::Result {
4346    a.vec_avg(b)
4347}
4348
4349/// Vector Shift Left
4350#[inline]
4351#[target_feature(enable = "vector")]
4352#[unstable(feature = "stdarch_s390x", issue = "135681")]
4353pub unsafe fn vec_sl<T: sealed::VectorSl<U>, U>(a: T, b: U) -> T::Result {
4354    a.vec_sl(b)
4355}
4356
4357/// Vector Shift Right
4358#[inline]
4359#[target_feature(enable = "vector")]
4360#[unstable(feature = "stdarch_s390x", issue = "135681")]
4361pub unsafe fn vec_sr<T: sealed::VectorSr<U>, U>(a: T, b: U) -> T::Result {
4362    a.vec_sr(b)
4363}
4364
4365/// Vector Shift Right Algebraic
4366#[inline]
4367#[target_feature(enable = "vector")]
4368#[unstable(feature = "stdarch_s390x", issue = "135681")]
4369pub unsafe fn vec_sra<T: sealed::VectorSra<U>, U>(a: T, b: U) -> T::Result {
4370    a.vec_sra(b)
4371}
4372
4373/// Vector Shift Left by Byte
4374#[inline]
4375#[target_feature(enable = "vector")]
4376#[unstable(feature = "stdarch_s390x", issue = "135681")]
4377pub unsafe fn vec_slb<T: sealed::VectorSlb<U>, U>(a: T, b: U) -> T::Result {
4378    a.vec_slb(b)
4379}
4380
4381/// Vector Shift Right by Byte
4382#[inline]
4383#[target_feature(enable = "vector")]
4384#[unstable(feature = "stdarch_s390x", issue = "135681")]
4385pub unsafe fn vec_srb<T: sealed::VectorSrb<U>, U>(a: T, b: U) -> T::Result {
4386    a.vec_srb(b)
4387}
4388
4389/// Vector Shift Right Algebraic by Byte
4390#[inline]
4391#[target_feature(enable = "vector")]
4392#[unstable(feature = "stdarch_s390x", issue = "135681")]
4393pub unsafe fn vec_srab<T: sealed::VectorSrab<U>, U>(a: T, b: U) -> T::Result {
4394    a.vec_srab(b)
4395}
4396
4397/// Vector Element Rotate Left
4398#[inline]
4399#[target_feature(enable = "vector")]
4400#[unstable(feature = "stdarch_s390x", issue = "135681")]
4401pub unsafe fn vec_rl<T: sealed::VectorRl<U>, U>(a: T, b: U) -> T::Result {
4402    a.vec_rl(b)
4403}
4404
4405/// Vector Shift Left
4406///
4407/// Performs a left shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4408/// element of a left by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros.
4409#[inline]
4410#[target_feature(enable = "vector")]
4411#[unstable(feature = "stdarch_s390x", issue = "135681")]
4412pub unsafe fn vec_sll<T>(a: T, b: vector_unsigned_char) -> T
4413where
4414    T: sealed::VectorSll<vector_unsigned_char, Result = T>,
4415{
4416    a.vec_sll(b)
4417}
4418
4419/// Vector Shift Right
4420///
4421/// Performs a right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4422/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros.
4423#[inline]
4424#[target_feature(enable = "vector")]
4425#[unstable(feature = "stdarch_s390x", issue = "135681")]
4426pub unsafe fn vec_srl<T>(a: T, b: vector_unsigned_char) -> T
4427where
4428    T: sealed::VectorSrl<vector_unsigned_char, Result = T>,
4429{
4430    a.vec_srl(b)
4431}
4432
4433/// Vector Shift Right Arithmetic
4434///
4435/// Performs an algebraic right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4436/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by copies of
4437/// the most significant bit of the element of a.
4438#[inline]
4439#[target_feature(enable = "vector")]
4440#[unstable(feature = "stdarch_s390x", issue = "135681")]
4441pub unsafe fn vec_sral<T>(a: T, b: vector_unsigned_char) -> T
4442where
4443    T: sealed::VectorSral<vector_unsigned_char, Result = T>,
4444{
4445    a.vec_sral(b)
4446}
4447
4448/// Vector Element Rotate Left Immediate
4449///
4450/// Rotates each element of a vector left by a given number of bits. Each element of the result is obtained by rotating the corresponding element
4451/// of a left by the number of bits specified by b, modulo the number of bits in the element.
4452#[inline]
4453#[target_feature(enable = "vector")]
4454#[unstable(feature = "stdarch_s390x", issue = "135681")]
4455pub unsafe fn vec_rli<T: sealed::VectorRli>(a: T, bits: core::ffi::c_ulong) -> T {
4456    a.vec_rli(bits)
4457}
4458
4459/// Vector Reverse Elements
4460///
4461/// Returns a vector with the elements of the input vector in reversed order.
4462#[inline]
4463#[target_feature(enable = "vector")]
4464#[unstable(feature = "stdarch_s390x", issue = "135681")]
4465pub unsafe fn vec_reve<T: sealed::VectorReve>(a: T) -> T {
4466    a.vec_reve()
4467}
4468
4469/// Vector Byte Reverse
4470///
4471/// Returns a vector where each vector element contains the corresponding byte-reversed vector element of the input vector.
4472#[inline]
4473#[target_feature(enable = "vector")]
4474#[unstable(feature = "stdarch_s390x", issue = "135681")]
4475pub unsafe fn vec_revb<T: sealed::VectorRevb>(a: T) -> T {
4476    a.vec_revb()
4477}
4478
4479/// Vector Merge High
4480///
4481/// Merges the most significant ("high") halves of two vectors.
4482#[inline]
4483#[target_feature(enable = "vector")]
4484#[unstable(feature = "stdarch_s390x", issue = "135681")]
4485pub unsafe fn vec_mergeh<T: sealed::VectorMergeh>(a: T, b: T) -> T {
4486    a.vec_mergeh(b)
4487}
4488
4489/// Vector Merge Low
4490///
4491/// Merges the least significant ("low") halves of two vectors.
4492#[inline]
4493#[target_feature(enable = "vector")]
4494#[unstable(feature = "stdarch_s390x", issue = "135681")]
4495pub unsafe fn vec_mergel<T: sealed::VectorMergel>(a: T, b: T) -> T {
4496    a.vec_mergel(b)
4497}
4498
4499/// Vector Pack
4500#[inline]
4501#[target_feature(enable = "vector")]
4502#[unstable(feature = "stdarch_s390x", issue = "135681")]
4503pub unsafe fn vec_pack<T: sealed::VectorPack<U>, U>(a: T, b: U) -> T::Result {
4504    a.vec_pack(b)
4505}
4506
4507/// Vector Pack Saturated
4508#[inline]
4509#[target_feature(enable = "vector")]
4510#[unstable(feature = "stdarch_s390x", issue = "135681")]
4511pub unsafe fn vec_packs<T: sealed::VectorPacks<U>, U>(a: T, b: U) -> T::Result {
4512    a.vec_packs(b)
4513}
4514
4515/// Vector Pack Saturated Condition Code
4516#[inline]
4517#[target_feature(enable = "vector")]
4518#[unstable(feature = "stdarch_s390x", issue = "135681")]
4519pub unsafe fn vec_packs_cc<T: sealed::VectorPacksCC>(a: T, b: T) -> (T::Result, i32) {
4520    a.vec_packs_cc(b)
4521}
4522
4523/// Vector Pack Saturated Unsigned
4524#[inline]
4525#[target_feature(enable = "vector")]
4526#[unstable(feature = "stdarch_s390x", issue = "135681")]
4527pub unsafe fn vec_packsu<T: sealed::VectorPacksu<U>, U>(a: T, b: U) -> T::Result {
4528    a.vec_packsu(b)
4529}
4530
4531/// Vector Pack Saturated Unsigned Condition Code
4532#[inline]
4533#[target_feature(enable = "vector")]
4534#[unstable(feature = "stdarch_s390x", issue = "135681")]
4535pub unsafe fn vec_packsu_cc<T: sealed::VectorPacksuCC>(a: T, b: T) -> (T::Result, i32) {
4536    a.vec_packsu_cc(b)
4537}
4538
4539/// Vector Unpack High
4540#[inline]
4541#[target_feature(enable = "vector")]
4542#[unstable(feature = "stdarch_s390x", issue = "135681")]
4543pub unsafe fn vec_unpackh<T: sealed::VectorUnpackh>(a: T) -> <T as sealed::VectorUnpackh>::Result {
4544    a.vec_unpackh()
4545}
4546
4547/// Vector Unpack Low
4548#[inline]
4549#[target_feature(enable = "vector")]
4550#[unstable(feature = "stdarch_s390x", issue = "135681")]
4551pub unsafe fn vec_unpackl<T: sealed::VectorUnpackl>(a: T) -> <T as sealed::VectorUnpackl>::Result {
4552    a.vec_unpackl()
4553}
4554
4555/// Vector Generate Byte Mask
4556///
4557/// Generates byte masks for elements in the return vector. For each bit in a, if the bit is one, all bit positions
4558/// in the corresponding byte element of d are set to ones. Otherwise, if the bit is zero, the corresponding byte element is set to zero.
4559#[inline]
4560#[target_feature(enable = "vector")]
4561#[unstable(feature = "stdarch_s390x", issue = "135681")]
4562#[cfg_attr(test, assert_instr(vgbm, MASK = 0x00FF))]
4563pub unsafe fn vec_genmask<const MASK: u16>() -> vector_unsigned_char {
4564    vector_unsigned_char(const { genmask::<MASK>() })
4565}
4566
4567/// Vector Generate Mask (Byte)
4568#[inline]
4569#[target_feature(enable = "vector")]
4570#[unstable(feature = "stdarch_s390x", issue = "135681")]
4571#[cfg_attr(test, assert_instr(vrepib, L = 3, H = 5))]
4572pub unsafe fn vec_genmasks_8<const L: u8, const H: u8>() -> vector_unsigned_char {
4573    vector_unsigned_char(const { [genmasks(u8::BITS, L, H) as u8; 16] })
4574}
4575
4576/// Vector Generate Mask (Halfword)
4577#[inline]
4578#[target_feature(enable = "vector")]
4579#[unstable(feature = "stdarch_s390x", issue = "135681")]
4580#[cfg_attr(test, assert_instr(vrepih, L = 3, H = 5))]
4581pub unsafe fn vec_genmasks_16<const L: u8, const H: u8>() -> vector_unsigned_short {
4582    vector_unsigned_short(const { [genmasks(u16::BITS, L, H) as u16; 8] })
4583}
4584
4585/// Vector Generate Mask (Word)
4586#[inline]
4587#[target_feature(enable = "vector")]
4588#[unstable(feature = "stdarch_s390x", issue = "135681")]
4589#[cfg_attr(test, assert_instr(vgmf, L = 3, H = 5))]
4590pub unsafe fn vec_genmasks_32<const L: u8, const H: u8>() -> vector_unsigned_int {
4591    vector_unsigned_int(const { [genmasks(u32::BITS, L, H) as u32; 4] })
4592}
4593
4594/// Vector Generate Mask (Doubleword)
4595#[inline]
4596#[target_feature(enable = "vector")]
4597#[unstable(feature = "stdarch_s390x", issue = "135681")]
4598#[cfg_attr(test, assert_instr(vgmg, L = 3, H = 5))]
4599pub unsafe fn vec_genmasks_64<const L: u8, const H: u8>() -> vector_unsigned_long_long {
4600    vector_unsigned_long_long(const { [genmasks(u64::BITS, L, H); 2] })
4601}
4602
4603/// Vector Permute
4604///
4605/// Returns a vector that contains some elements of two vectors, in the order specified by a third vector.
4606/// Each byte of the result is selected by using the least significant 5 bits of the corresponding byte of c as an index into the concatenated bytes of a and b.
4607/// Note: The vector generate mask built-in function [`vec_genmask`] could help generate the mask c.
4608#[inline]
4609#[target_feature(enable = "vector")]
4610#[unstable(feature = "stdarch_s390x", issue = "135681")]
4611pub unsafe fn vec_perm<T: sealed::VectorPerm>(a: T, b: T, c: vector_unsigned_char) -> T {
4612    a.vec_perm(b, c)
4613}
4614
4615/// Vector Sum Across Quadword
4616///
4617/// Returns a vector containing the results of performing a sum across all the elements in each of the quadword of vector a,
4618/// and the rightmost word or doubleword element of the b. The result is an unsigned 128-bit integer.
4619#[inline]
4620#[target_feature(enable = "vector")]
4621#[unstable(feature = "stdarch_s390x", issue = "135681")]
4622pub unsafe fn vec_sum_u128<T: sealed::VectorSumU128>(a: T, b: T) -> vector_unsigned_char {
4623    a.vec_sum_u128(b)
4624}
4625
4626/// Vector Sum Across Doubleword
4627///
4628/// Returns a vector containing the results of performing a sum across all the elements in each of the doubleword of vector a,
4629/// and the rightmost sub-element of the corresponding doubleword of b.
4630#[inline]
4631#[target_feature(enable = "vector")]
4632#[unstable(feature = "stdarch_s390x", issue = "135681")]
4633pub unsafe fn vec_sum2<T: sealed::VectorSum2>(a: T, b: T) -> vector_unsigned_long_long {
4634    a.vec_sum2(b)
4635}
4636
4637/// Vector Sum Across Word
4638///
4639/// Returns a vector containing the results of performing a sum across all the elements in each of the word of vector a,
4640/// and the rightmost sub-element of the corresponding word of b.
4641#[inline]
4642#[target_feature(enable = "vector")]
4643#[unstable(feature = "stdarch_s390x", issue = "135681")]
4644pub unsafe fn vec_sum4<T: sealed::VectorSum4>(a: T, b: T) -> vector_unsigned_int {
4645    a.vec_sum4(b)
4646}
4647
4648/// Vector Addition unsigned 128-bits
4649///
4650/// Adds unsigned quadword values.
4651///
4652/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a + b.
4653#[inline]
4654#[target_feature(enable = "vector")]
4655#[unstable(feature = "stdarch_s390x", issue = "135681")]
4656#[cfg_attr(test, assert_instr(vaq))]
4657pub unsafe fn vec_add_u128(
4658    a: vector_unsigned_char,
4659    b: vector_unsigned_char,
4660) -> vector_unsigned_char {
4661    let a: u128 = transmute(a);
4662    let b: u128 = transmute(b);
4663    transmute(a.wrapping_add(b))
4664}
4665
4666/// Vector Subtract unsigned 128-bits
4667///
4668/// Subtracts unsigned quadword values.
4669///
4670/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a - b.
4671#[inline]
4672#[target_feature(enable = "vector")]
4673#[unstable(feature = "stdarch_s390x", issue = "135681")]
4674#[cfg_attr(test, assert_instr(vsq))]
4675pub unsafe fn vec_sub_u128(
4676    a: vector_unsigned_char,
4677    b: vector_unsigned_char,
4678) -> vector_unsigned_char {
4679    let a: u128 = transmute(a);
4680    let b: u128 = transmute(b);
4681
4682    transmute(a.wrapping_sub(b))
4683}
4684
4685/// Vector Subtract Carryout
4686///
4687/// Returns a vector containing the borrow produced by subtracting each of corresponding elements of b from a.
4688///
4689/// On each resulting element, the value is 0 if a borrow occurred, or 1 if no borrow occurred.
4690#[inline]
4691#[target_feature(enable = "vector")]
4692#[unstable(feature = "stdarch_s390x", issue = "135681")]
4693pub unsafe fn vec_subc<T: sealed::VectorSubc<U>, U>(a: T, b: U) -> T::Result {
4694    a.vec_subc(b)
4695}
4696
4697/// Vector Subtract Carryout unsigned 128-bits
4698///
4699/// Gets the carry bit of the 128-bit subtraction of two quadword values.
4700/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the borrow produced by subtracting b from a, as unsigned 128-bits integers.
4701/// If no borrow occurred, the bit 127 of d is 1; otherwise it is set to 0. All other bits of d are 0.
4702#[inline]
4703#[target_feature(enable = "vector")]
4704#[unstable(feature = "stdarch_s390x", issue = "135681")]
4705#[cfg_attr(test, assert_instr(vscbiq))]
4706pub unsafe fn vec_subc_u128(
4707    a: vector_unsigned_char,
4708    b: vector_unsigned_char,
4709) -> vector_unsigned_char {
4710    let a: u128 = transmute(a);
4711    let b: u128 = transmute(b);
4712    transmute(!a.overflowing_sub(b).1 as u128)
4713}
4714
4715/// Vector Add Compute Carryout unsigned 128-bits
4716#[inline]
4717#[target_feature(enable = "vector")]
4718#[unstable(feature = "stdarch_s390x", issue = "135681")]
4719#[cfg_attr(test, assert_instr(vaccq))]
4720pub unsafe fn vec_addc_u128(
4721    a: vector_unsigned_char,
4722    b: vector_unsigned_char,
4723) -> vector_unsigned_char {
4724    let a: u128 = transmute(a);
4725    let b: u128 = transmute(b);
4726    // FIXME(llvm) https://github.com/llvm/llvm-project/pull/153557
4727    // transmute(a.overflowing_add(b).1 as u128)
4728    transmute(vaccq(a, b))
4729}
4730
4731/// Vector Add With Carry unsigned 128-bits
4732#[inline]
4733#[target_feature(enable = "vector")]
4734#[unstable(feature = "stdarch_s390x", issue = "135681")]
4735#[cfg_attr(test, assert_instr(vacq))]
4736pub unsafe fn vec_adde_u128(
4737    a: vector_unsigned_char,
4738    b: vector_unsigned_char,
4739    c: vector_unsigned_char,
4740) -> vector_unsigned_char {
4741    let a: u128 = transmute(a);
4742    let b: u128 = transmute(b);
4743    let c: u128 = transmute(c);
4744    // FIXME(llvm) https://github.com/llvm/llvm-project/pull/153557
4745    //     let (d, _carry) = a.carrying_add(b, c & 1 != 0);
4746    //     transmute(d)
4747    transmute(vacq(a, b, c))
4748}
4749
4750/// Vector Add With Carry Compute Carry unsigned 128-bits
4751#[inline]
4752#[target_feature(enable = "vector")]
4753#[unstable(feature = "stdarch_s390x", issue = "135681")]
4754#[cfg_attr(test, assert_instr(vacccq))]
4755pub unsafe fn vec_addec_u128(
4756    a: vector_unsigned_char,
4757    b: vector_unsigned_char,
4758    c: vector_unsigned_char,
4759) -> vector_unsigned_char {
4760    let a: u128 = transmute(a);
4761    let b: u128 = transmute(b);
4762    let c: u128 = transmute(c);
4763    // FIXME(llvm) https://github.com/llvm/llvm-project/pull/153557
4764    // let (_d, carry) = a.carrying_add(b, c & 1 != 0);
4765    // transmute(carry as u128)
4766    transmute(vacccq(a, b, c))
4767}
4768
4769/// Vector Subtract with Carryout
4770///
4771/// Subtracts unsigned quadword values with carry bit from a previous operation.
4772///
4773/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the result of subtracting of b from a,
4774/// and the carryout bit from a previous operation.
4775///
4776/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored.
4777#[inline]
4778#[target_feature(enable = "vector")]
4779#[unstable(feature = "stdarch_s390x", issue = "135681")]
4780#[cfg_attr(test, assert_instr(vsbiq))]
4781pub unsafe fn vec_sube_u128(
4782    a: vector_unsigned_char,
4783    b: vector_unsigned_char,
4784    c: vector_unsigned_char,
4785) -> vector_unsigned_char {
4786    transmute(vsbiq(transmute(a), transmute(b), transmute(c)))
4787}
4788
4789/// Vector Subtract with Carryout, Carryout
4790///
4791/// Gets the carry bit of the 128-bit subtraction of two quadword values with carry bit from the previous operation.
4792///
4793/// It returns a vector containing the carryout produced from the result of subtracting of b from a,
4794/// and the carryout bit from a previous operation. If no borrow occurred, the 127-bit of d is 1, otherwise 0.
4795/// All other bits of d are 0.
4796///
4797/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored.
4798#[inline]
4799#[target_feature(enable = "vector")]
4800#[unstable(feature = "stdarch_s390x", issue = "135681")]
4801#[cfg_attr(test, assert_instr(vsbcbiq))]
4802pub unsafe fn vec_subec_u128(
4803    a: vector_unsigned_char,
4804    b: vector_unsigned_char,
4805    c: vector_unsigned_char,
4806) -> vector_unsigned_char {
4807    transmute(vsbcbiq(transmute(a), transmute(b), transmute(c)))
4808}
4809
4810/// Vector Splat Signed Byte
4811#[inline]
4812#[target_feature(enable = "vector")]
4813#[unstable(feature = "stdarch_s390x", issue = "135681")]
4814#[cfg_attr(test, assert_instr(vrepib, IMM = 42))]
4815pub unsafe fn vec_splat_s8<const IMM: i8>() -> vector_signed_char {
4816    vector_signed_char([IMM; 16])
4817}
4818
4819/// Vector Splat Signed Halfword
4820#[inline]
4821#[target_feature(enable = "vector")]
4822#[unstable(feature = "stdarch_s390x", issue = "135681")]
4823#[cfg_attr(test, assert_instr(vrepih, IMM = 42))]
4824pub unsafe fn vec_splat_s16<const IMM: i16>() -> vector_signed_short {
4825    vector_signed_short([IMM; 8])
4826}
4827
4828/// Vector Splat Signed Word
4829#[inline]
4830#[target_feature(enable = "vector")]
4831#[unstable(feature = "stdarch_s390x", issue = "135681")]
4832#[cfg_attr(test, assert_instr(vrepif, IMM = 42))]
4833pub unsafe fn vec_splat_s32<const IMM: i16>() -> vector_signed_int {
4834    vector_signed_int([IMM as i32; 4])
4835}
4836
4837/// Vector Splat Signed Doubleword
4838#[inline]
4839#[target_feature(enable = "vector")]
4840#[unstable(feature = "stdarch_s390x", issue = "135681")]
4841#[cfg_attr(test, assert_instr(vrepig, IMM = 42))]
4842pub unsafe fn vec_splat_s64<const IMM: i16>() -> vector_signed_long_long {
4843    vector_signed_long_long([IMM as i64; 2])
4844}
4845
4846/// Vector Splat Unsigned Byte
4847#[inline]
4848#[target_feature(enable = "vector")]
4849#[unstable(feature = "stdarch_s390x", issue = "135681")]
4850#[cfg_attr(test, assert_instr(vrepib, IMM = 42))]
4851pub unsafe fn vec_splat_u8<const IMM: u8>() -> vector_unsigned_char {
4852    vector_unsigned_char([IMM; 16])
4853}
4854
4855/// Vector Splat Unsigned Halfword
4856#[inline]
4857#[target_feature(enable = "vector")]
4858#[unstable(feature = "stdarch_s390x", issue = "135681")]
4859#[cfg_attr(test, assert_instr(vrepih, IMM = 42))]
4860pub unsafe fn vec_splat_u16<const IMM: i16>() -> vector_unsigned_short {
4861    vector_unsigned_short([IMM as u16; 8])
4862}
4863
4864/// Vector Splat Unsigned Word
4865#[inline]
4866#[target_feature(enable = "vector")]
4867#[unstable(feature = "stdarch_s390x", issue = "135681")]
4868#[cfg_attr(test, assert_instr(vrepif, IMM = 42))]
4869pub unsafe fn vec_splat_u32<const IMM: i16>() -> vector_unsigned_int {
4870    vector_unsigned_int([IMM as u32; 4])
4871}
4872
4873/// Vector Splat Unsigned Doubleword
4874#[inline]
4875#[target_feature(enable = "vector")]
4876#[unstable(feature = "stdarch_s390x", issue = "135681")]
4877#[cfg_attr(test, assert_instr(vrepig, IMM = 42))]
4878pub unsafe fn vec_splat_u64<const IMM: i16>() -> vector_unsigned_long_long {
4879    vector_unsigned_long_long([IMM as u64; 2])
4880}
4881
4882macro_rules! vec_find_any {
4883    ($($Trait:ident $fun:ident $doc:literal)*) => {
4884        $(
4885            #[inline]
4886            #[target_feature(enable = "vector")]
4887            #[unstable(feature = "stdarch_s390x", issue = "135681")]
4888            #[doc = $doc]
4889            pub unsafe fn $fun<T: sealed::$Trait<U>, U>(a: T, b: U) -> T::Result {
4890                a.$fun(b)
4891            }
4892        )*
4893    }
4894}
4895
4896vec_find_any! {
4897    VectorFindAnyEq vec_find_any_eq "Vector Find Any Element Equal with Condition Code"
4898    VectorFindAnyNe vec_find_any_ne "Vector Find Any Element Not Equal with Condition Code"
4899    VectorFindAnyEqIdx vec_find_any_eq_idx "Vector Find Any Element Equal Index with Condition Code"
4900    VectorFindAnyNeIdx vec_find_any_ne_idx "Vector Find Any Element Not Equal Index with Condition Code"
4901    VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx "Vector Find Any Element Equal or Zero Index with Condition Code"
4902    VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx "Vector Find Any Element Not Equal or Zero Index with Condition Code"
4903}
4904
4905macro_rules! vec_find_any_cc {
4906    ($($Trait:ident $fun:ident $doc:literal)*) => {
4907        $(
4908            #[inline]
4909            #[target_feature(enable = "vector")]
4910            #[unstable(feature = "stdarch_s390x", issue = "135681")]
4911            #[doc = $doc]
4912            pub unsafe fn $fun<T: sealed::$Trait<U>, U>(a: T, b: U) -> (T::Result, i32) {
4913                a.$fun(b)
4914            }
4915        )*
4916    }
4917}
4918
4919vec_find_any_cc! {
4920    VectorFindAnyEqCC vec_find_any_eq_cc "Vector Find Any Element Equal with Condition Code"
4921    VectorFindAnyNeCC vec_find_any_ne_cc "Vector Find Any Element Not Equal with Condition Code"
4922    VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc "Vector Find Any Element Equal Index with Condition Code"
4923    VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc "Vector Find Any Element Not Equal Index with Condition Code"
4924    VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc "Vector Find Any Element Equal or Zero Index with Condition Code"
4925    VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc "Vector Find Any Element Not Equal or Zero Index with Condition Code"
4926}
4927
4928/// Vector Load
4929#[inline]
4930#[target_feature(enable = "vector")]
4931#[unstable(feature = "stdarch_s390x", issue = "135681")]
4932pub unsafe fn vec_xl<T: sealed::VectorLoad>(offset: isize, ptr: *const T::ElementType) -> T {
4933    T::vec_xl(offset, ptr)
4934}
4935
4936/// Vector Load Pair
4937#[inline]
4938#[target_feature(enable = "vector")]
4939#[unstable(feature = "stdarch_s390x", issue = "135681")]
4940pub unsafe fn vec_load_pair<T: sealed::VectorLoadPair>(a: T::ElementType, b: T::ElementType) -> T {
4941    T::vec_load_pair(a, b)
4942}
4943
4944/// Vector Load to Block Boundary
4945#[inline]
4946#[target_feature(enable = "vector")]
4947#[unstable(feature = "stdarch_s390x", issue = "135681")]
4948pub unsafe fn vec_load_bndry<T: sealed::VectorLoad, const BLOCK_BOUNDARY: u16>(
4949    ptr: *const T::ElementType,
4950) -> MaybeUninit<T> {
4951    T::vec_load_bndry::<BLOCK_BOUNDARY>(ptr)
4952}
4953
4954/// Vector Store
4955#[inline]
4956#[target_feature(enable = "vector")]
4957#[unstable(feature = "stdarch_s390x", issue = "135681")]
4958pub unsafe fn vec_xst<T: sealed::VectorStore>(vector: T, offset: isize, ptr: *mut T::ElementType) {
4959    vector.vec_xst(offset, ptr)
4960}
4961
4962/// Vector Load with Length
4963#[inline]
4964#[target_feature(enable = "vector")]
4965#[unstable(feature = "stdarch_s390x", issue = "135681")]
4966pub unsafe fn vec_load_len<T: sealed::VectorLoad>(
4967    ptr: *const T::ElementType,
4968    byte_count: u32,
4969) -> T {
4970    T::vec_load_len(ptr, byte_count)
4971}
4972
4973/// Vector Store with Length
4974#[inline]
4975#[target_feature(enable = "vector")]
4976#[unstable(feature = "stdarch_s390x", issue = "135681")]
4977pub unsafe fn vec_store_len<T: sealed::VectorStore>(
4978    vector: T,
4979    ptr: *mut T::ElementType,
4980    byte_count: u32,
4981) {
4982    vector.vec_store_len(ptr, byte_count)
4983}
4984
4985/// Vector Load Rightmost with Length
4986#[inline]
4987#[target_feature(enable = "vector-packed-decimal")]
4988#[unstable(feature = "stdarch_s390x", issue = "135681")]
4989#[cfg_attr(test, assert_instr(vlrlr))]
4990pub unsafe fn vec_load_len_r(ptr: *const u8, byte_count: u32) -> vector_unsigned_char {
4991    vlrl(byte_count, ptr)
4992}
4993
4994/// Vector Store Rightmost with Length
4995#[inline]
4996#[target_feature(enable = "vector-packed-decimal")]
4997#[unstable(feature = "stdarch_s390x", issue = "135681")]
4998#[cfg_attr(test, assert_instr(vstrlr))]
4999pub unsafe fn vec_store_len_r(vector: vector_unsigned_char, ptr: *mut u8, byte_count: u32) {
5000    vstrl(vector, byte_count, ptr)
5001}
5002
5003/// Vector Multiply Add
5004#[inline]
5005#[target_feature(enable = "vector-packed-decimal")]
5006#[unstable(feature = "stdarch_s390x", issue = "135681")]
5007pub unsafe fn vec_madd<T: sealed::VectorMadd>(a: T, b: T, c: T) -> T {
5008    a.vec_madd(b, c)
5009}
5010
5011/// Vector Multiply Add
5012#[inline]
5013#[target_feature(enable = "vector-packed-decimal")]
5014#[unstable(feature = "stdarch_s390x", issue = "135681")]
5015pub unsafe fn vec_msub<T: sealed::VectorMadd>(a: T, b: T, c: T) -> T {
5016    a.vec_msub(b, c)
5017}
5018
5019/// Vector Multiply and Add Even
5020#[inline]
5021#[target_feature(enable = "vector-packed-decimal")]
5022#[unstable(feature = "stdarch_s390x", issue = "135681")]
5023pub unsafe fn vec_meadd<T: sealed::VectorMeadd>(a: T, b: T, c: T::Result) -> T::Result {
5024    a.vec_meadd(b, c)
5025}
5026
5027/// Vector Multiply and Add Odd
5028#[inline]
5029#[target_feature(enable = "vector-packed-decimal")]
5030#[unstable(feature = "stdarch_s390x", issue = "135681")]
5031pub unsafe fn vec_moadd<T: sealed::VectorMoadd>(a: T, b: T, c: T::Result) -> T::Result {
5032    a.vec_moadd(b, c)
5033}
5034
5035/// Vector Multiply and Add High
5036#[inline]
5037#[target_feature(enable = "vector-packed-decimal")]
5038#[unstable(feature = "stdarch_s390x", issue = "135681")]
5039pub unsafe fn vec_mhadd<T: sealed::VectorMhadd>(a: T, b: T, c: T::Result) -> T::Result {
5040    a.vec_mhadd(b, c)
5041}
5042
5043/// Vector Multiply and Add Low
5044#[inline]
5045#[target_feature(enable = "vector-packed-decimal")]
5046#[unstable(feature = "stdarch_s390x", issue = "135681")]
5047pub unsafe fn vec_mladd<T: sealed::VectorMladd>(a: T, b: T, c: T::Result) -> T::Result {
5048    a.vec_mladd(b, c)
5049}
5050
5051/// Vector Checksum
5052#[inline]
5053#[target_feature(enable = "vector")]
5054#[unstable(feature = "stdarch_s390x", issue = "135681")]
5055#[cfg_attr(test, assert_instr(vcksm))]
5056pub unsafe fn vec_checksum(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int {
5057    vcksm(a, b)
5058}
5059
5060/// Vector Multiply Even
5061#[inline]
5062#[target_feature(enable = "vector")]
5063#[unstable(feature = "stdarch_s390x", issue = "135681")]
5064pub unsafe fn vec_mule<T: sealed::VectorMule<U>, U>(a: T, b: T) -> U {
5065    a.vec_mule(b)
5066}
5067
5068/// Vector Multiply Odd
5069#[inline]
5070#[target_feature(enable = "vector")]
5071#[unstable(feature = "stdarch_s390x", issue = "135681")]
5072pub unsafe fn vec_mulo<T: sealed::VectorMulo<U>, U>(a: T, b: T) -> U {
5073    a.vec_mulo(b)
5074}
5075
5076/// Vector Multiply High
5077#[inline]
5078#[target_feature(enable = "vector")]
5079#[unstable(feature = "stdarch_s390x", issue = "135681")]
5080pub unsafe fn vec_mulh<T: sealed::VectorMulh<U>, U>(a: T, b: T) -> U {
5081    a.vec_mulh(b)
5082}
5083
5084/// Vector Galois Field Multiply Sum
5085#[inline]
5086#[target_feature(enable = "vector")]
5087#[unstable(feature = "stdarch_s390x", issue = "135681")]
5088pub unsafe fn vec_gfmsum<T: sealed::VectorGfmsum<U>, U>(a: T, b: T) -> U {
5089    a.vec_gfmsum(b)
5090}
5091
5092/// Vector Galois Field Multiply Sum
5093#[inline]
5094#[target_feature(enable = "vector")]
5095#[unstable(feature = "stdarch_s390x", issue = "135681")]
5096pub unsafe fn vec_gfmsum_accum<T: sealed::VectorGfmsumAccum>(
5097    a: T,
5098    b: T,
5099    c: T::Result,
5100) -> T::Result {
5101    a.vec_gfmsum_accum(b, c)
5102}
5103
5104/// Vector Galois Field Multiply Sum 128-bits
5105#[inline]
5106#[target_feature(enable = "vector")]
5107#[unstable(feature = "stdarch_s390x", issue = "135681")]
5108#[cfg_attr(test, assert_instr(vgfmg))]
5109pub unsafe fn vec_gfmsum_128(
5110    a: vector_unsigned_long_long,
5111    b: vector_unsigned_long_long,
5112) -> vector_unsigned_char {
5113    transmute(vgfmg(a, b))
5114}
5115
5116/// Vector Galois Field Multiply Sum and Accumulate 128-bits
5117#[inline]
5118#[target_feature(enable = "vector")]
5119#[unstable(feature = "stdarch_s390x", issue = "135681")]
5120#[cfg_attr(test, assert_instr(vgfmag))]
5121pub unsafe fn vec_gfmsum_accum_128(
5122    a: vector_unsigned_long_long,
5123    b: vector_unsigned_long_long,
5124    c: vector_unsigned_char,
5125) -> vector_unsigned_char {
5126    transmute(vgfmag(a, b, transmute(c)))
5127}
5128
5129/// Vector Bit Permute
5130#[inline]
5131#[target_feature(enable = "vector-enhancements-1")]
5132#[unstable(feature = "stdarch_s390x", issue = "135681")]
5133#[cfg_attr(test, assert_instr(vbperm))]
5134pub unsafe fn vec_bperm_u128(
5135    a: vector_unsigned_char,
5136    b: vector_unsigned_char,
5137) -> vector_unsigned_long_long {
5138    vbperm(a, b)
5139}
5140
5141/// Vector Gather Element
5142#[inline]
5143#[target_feature(enable = "vector")]
5144#[unstable(feature = "stdarch_s390x", issue = "135681")]
5145pub unsafe fn vec_gather_element<T: sealed::VectorGatherElement, const D: u32>(
5146    a: T,
5147    b: T::Offset,
5148    c: *const T::Element,
5149) -> T {
5150    a.vec_gather_element::<D>(b, c)
5151}
5152
5153/// Vector Select
5154#[inline]
5155#[target_feature(enable = "vector")]
5156#[unstable(feature = "stdarch_s390x", issue = "135681")]
5157pub unsafe fn vec_sel<T: sealed::VectorSel<U>, U>(a: T, b: T, c: U) -> T {
5158    a.vec_sel(b, c)
5159}
5160
5161#[unstable(feature = "stdarch_s390x", issue = "135681")]
5162pub const __VEC_CLASS_FP_ZERO_P: u32 = 1 << 11;
5163#[unstable(feature = "stdarch_s390x", issue = "135681")]
5164pub const __VEC_CLASS_FP_ZERO_N: u32 = 1 << 10;
5165#[unstable(feature = "stdarch_s390x", issue = "135681")]
5166pub const __VEC_CLASS_FP_ZERO: u32 = __VEC_CLASS_FP_ZERO_P | __VEC_CLASS_FP_ZERO_N;
5167#[unstable(feature = "stdarch_s390x", issue = "135681")]
5168pub const __VEC_CLASS_FP_NORMAL_P: u32 = 1 << 9;
5169#[unstable(feature = "stdarch_s390x", issue = "135681")]
5170pub const __VEC_CLASS_FP_NORMAL_N: u32 = 1 << 8;
5171#[unstable(feature = "stdarch_s390x", issue = "135681")]
5172pub const __VEC_CLASS_FP_NORMAL: u32 = __VEC_CLASS_FP_NORMAL_P | __VEC_CLASS_FP_NORMAL_N;
5173#[unstable(feature = "stdarch_s390x", issue = "135681")]
5174pub const __VEC_CLASS_FP_SUBNORMAL_P: u32 = 1 << 7;
5175#[unstable(feature = "stdarch_s390x", issue = "135681")]
5176pub const __VEC_CLASS_FP_SUBNORMAL_N: u32 = 1 << 6;
5177#[unstable(feature = "stdarch_s390x", issue = "135681")]
5178pub const __VEC_CLASS_FP_SUBNORMAL: u32 = __VEC_CLASS_FP_SUBNORMAL_P | __VEC_CLASS_FP_SUBNORMAL_N;
5179#[unstable(feature = "stdarch_s390x", issue = "135681")]
5180pub const __VEC_CLASS_FP_INFINITY_P: u32 = 1 << 5;
5181#[unstable(feature = "stdarch_s390x", issue = "135681")]
5182pub const __VEC_CLASS_FP_INFINITY_N: u32 = 1 << 4;
5183#[unstable(feature = "stdarch_s390x", issue = "135681")]
5184pub const __VEC_CLASS_FP_INFINITY: u32 = __VEC_CLASS_FP_INFINITY_P | __VEC_CLASS_FP_INFINITY_N;
5185#[unstable(feature = "stdarch_s390x", issue = "135681")]
5186pub const __VEC_CLASS_FP_QNAN_P: u32 = 1 << 3;
5187#[unstable(feature = "stdarch_s390x", issue = "135681")]
5188pub const __VEC_CLASS_FP_QNAN_N: u32 = 1 << 2;
5189#[unstable(feature = "stdarch_s390x", issue = "135681")]
5190pub const __VEC_CLASS_FP_QNAN: u32 = __VEC_CLASS_FP_QNAN_P | __VEC_CLASS_FP_QNAN_N;
5191#[unstable(feature = "stdarch_s390x", issue = "135681")]
5192pub const __VEC_CLASS_FP_SNAN_P: u32 = 1 << 1;
5193#[unstable(feature = "stdarch_s390x", issue = "135681")]
5194pub const __VEC_CLASS_FP_SNAN_N: u32 = 1 << 0;
5195#[unstable(feature = "stdarch_s390x", issue = "135681")]
5196pub const __VEC_CLASS_FP_SNAN: u32 = __VEC_CLASS_FP_SNAN_P | __VEC_CLASS_FP_SNAN_N;
5197#[unstable(feature = "stdarch_s390x", issue = "135681")]
5198pub const __VEC_CLASS_FP_NAN: u32 = __VEC_CLASS_FP_QNAN | __VEC_CLASS_FP_SNAN;
5199#[unstable(feature = "stdarch_s390x", issue = "135681")]
5200pub const __VEC_CLASS_FP_NOT_NORMAL: u32 =
5201    __VEC_CLASS_FP_NAN | __VEC_CLASS_FP_SUBNORMAL | __VEC_CLASS_FP_ZERO | __VEC_CLASS_FP_INFINITY;
5202
5203/// Vector Floating-Point Test Data Class
5204///
5205/// You can use the `__VEC_CLASS_FP_*` constants as the argument for this operand
5206#[inline]
5207#[target_feature(enable = "vector")]
5208#[unstable(feature = "stdarch_s390x", issue = "135681")]
5209pub unsafe fn vec_fp_test_data_class<T: sealed::VectorFpTestDataClass, const CLASS: u32>(
5210    a: T,
5211    c: *mut i32,
5212) -> T::Result {
5213    let (x, y) = a.vec_fp_test_data_class::<CLASS>();
5214    c.write(y);
5215    x
5216}
5217
5218/// All Elements Not a Number
5219#[inline]
5220#[target_feature(enable = "vector")]
5221#[unstable(feature = "stdarch_s390x", issue = "135681")]
5222pub unsafe fn vec_all_nan<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5223    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 0)
5224}
5225
5226/// All Elements Numeric
5227#[inline]
5228#[target_feature(enable = "vector")]
5229#[unstable(feature = "stdarch_s390x", issue = "135681")]
5230pub unsafe fn vec_all_numeric<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5231    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 3)
5232}
5233
5234/// Any Elements Not a Number
5235#[inline]
5236#[target_feature(enable = "vector")]
5237#[unstable(feature = "stdarch_s390x", issue = "135681")]
5238pub unsafe fn vec_any_nan<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5239    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 3)
5240}
5241
5242/// Any Elements Numeric
5243#[inline]
5244#[target_feature(enable = "vector")]
5245#[unstable(feature = "stdarch_s390x", issue = "135681")]
5246pub unsafe fn vec_any_numeric<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5247    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 0)
5248}
5249
5250/// Vector Test under Mask
5251#[inline]
5252#[target_feature(enable = "vector")]
5253#[unstable(feature = "stdarch_s390x", issue = "135681")]
5254pub unsafe fn vec_test_mask<T: sealed::VectorTestMask>(a: T, b: T::Mask) -> i32 {
5255    // I can't find much information about this, but this might just be a check for whether the
5256    // bitwise and of a and b is non-zero?
5257    a.vec_test_mask(b)
5258}
5259
5260/// Vector Search String
5261#[inline]
5262#[target_feature(enable = "vector")]
5263#[unstable(feature = "stdarch_s390x", issue = "135681")]
5264pub unsafe fn vec_search_string_cc<T: sealed::VectorSearchString>(
5265    a: T,
5266    b: T,
5267    c: vector_unsigned_char,
5268) -> (vector_unsigned_char, i32) {
5269    a.vec_search_string_cc(b, c)
5270}
5271
5272/// Vector Search String Until Zero
5273#[inline]
5274#[target_feature(enable = "vector")]
5275#[unstable(feature = "stdarch_s390x", issue = "135681")]
5276pub unsafe fn vec_search_string_until_zero_cc<T: sealed::VectorSearchString>(
5277    a: T,
5278    b: T,
5279    c: vector_unsigned_char,
5280) -> (vector_unsigned_char, i32) {
5281    a.vec_search_string_until_zero_cc(b, c)
5282}
5283
5284/// Vector Convert from float (even elements) to double
5285#[inline]
5286#[target_feature(enable = "vector-enhancements-1")]
5287#[unstable(feature = "stdarch_s390x", issue = "135681")]
5288// FIXME: this emits `vflls` where `vldeb` is expected
5289// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vldeb))]
5290pub unsafe fn vec_doublee(a: vector_float) -> vector_double {
5291    let even = simd_shuffle::<_, _, f32x2>(a, a, const { u32x2::from_array([0, 2]) });
5292    simd_as(even)
5293}
5294
5295/// Vector Convert from double to float (even elements)
5296#[inline]
5297#[target_feature(enable = "vector-enhancements-1")]
5298#[unstable(feature = "stdarch_s390x", issue = "135681")]
5299// FIXME: the C version uses a shuffle mask with poison; we can't do that
5300// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vledb))]
5301pub unsafe fn vec_floate(a: vector_double) -> vector_float {
5302    let truncated: f32x2 = simd_as(a);
5303    simd_shuffle(
5304        truncated,
5305        truncated,
5306        const { u32x4::from_array([0, 0, 1, 1]) },
5307    )
5308}
5309
5310/// Vector Convert from int to float
5311#[inline]
5312#[target_feature(enable = "vector")]
5313#[unstable(feature = "stdarch_s390x", issue = "135681")]
5314pub unsafe fn vec_float(a: impl sealed::VectorFloat) -> vector_float {
5315    a.vec_float()
5316}
5317
5318/// Vector Convert from long long to double
5319#[inline]
5320#[target_feature(enable = "vector")]
5321#[unstable(feature = "stdarch_s390x", issue = "135681")]
5322pub unsafe fn vec_double(a: impl sealed::VectorDouble) -> vector_double {
5323    a.vec_double()
5324}
5325
5326/// Vector Sign Extend to Doubleword
5327#[inline]
5328#[target_feature(enable = "vector")]
5329#[unstable(feature = "stdarch_s390x", issue = "135681")]
5330pub unsafe fn vec_extend_s64(a: impl sealed::VectorExtendSigned64) -> vector_signed_long_long {
5331    a.vec_extend_s64()
5332}
5333
5334/// Vector Convert floating point to signed
5335#[inline]
5336#[target_feature(enable = "vector")]
5337#[unstable(feature = "stdarch_s390x", issue = "135681")]
5338pub unsafe fn vec_signed<T: sealed::VectorSigned>(a: T) -> T::Result {
5339    a.vec_signed()
5340}
5341
5342/// Vector Convert floating point to unsigned
5343#[inline]
5344#[target_feature(enable = "vector")]
5345#[unstable(feature = "stdarch_s390x", issue = "135681")]
5346pub unsafe fn vec_unsigned<T: sealed::VectorUnsigned>(a: T) -> T::Result {
5347    a.vec_unsigned()
5348}
5349
5350/// Vector Copy Until Zero
5351#[inline]
5352#[target_feature(enable = "vector")]
5353#[unstable(feature = "stdarch_s390x", issue = "135681")]
5354pub unsafe fn vec_cp_until_zero<T: sealed::VectorCopyUntilZero>(a: T) -> T {
5355    a.vec_cp_until_zero()
5356}
5357
5358/// Vector Copy Until Zero
5359#[inline]
5360#[target_feature(enable = "vector")]
5361#[unstable(feature = "stdarch_s390x", issue = "135681")]
5362pub unsafe fn vec_cp_until_zero_cc<T: sealed::VectorCopyUntilZeroCC>(a: T) -> (T, i32) {
5363    a.vec_cp_until_zero_cc()
5364}
5365
5366/// Vector Multiply Sum Logical
5367#[inline]
5368#[target_feature(enable = "vector-enhancements-1")]
5369#[unstable(feature = "stdarch_s390x", issue = "135681")]
5370#[cfg_attr(
5371    all(test, target_feature = "vector-enhancements-1"),
5372    assert_instr(vmslg, D = 4)
5373)]
5374pub unsafe fn vec_msum_u128<const D: u32>(
5375    a: vector_unsigned_long_long,
5376    b: vector_unsigned_long_long,
5377    c: vector_unsigned_char,
5378) -> vector_unsigned_char {
5379    const {
5380        if !matches!(D, 0 | 4 | 8 | 12) {
5381            panic!("D needs to be one of 0, 4, 8, 12");
5382        }
5383    };
5384    transmute(vmslg(a, b, transmute(c), D))
5385}
5386
5387/// Vector Shift Left Double by Byte
5388#[inline]
5389#[target_feature(enable = "vector")]
5390#[unstable(feature = "stdarch_s390x", issue = "135681")]
5391pub unsafe fn vec_sld<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5392    static_assert_uimm_bits!(C, 4);
5393    a.vec_sld::<C>(b)
5394}
5395
5396/// Vector Shift Left Double by Word
5397#[inline]
5398#[target_feature(enable = "vector")]
5399#[unstable(feature = "stdarch_s390x", issue = "135681")]
5400pub unsafe fn vec_sldw<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5401    static_assert_uimm_bits!(C, 2);
5402    a.vec_sldw::<C>(b)
5403}
5404
5405/// Vector Shift Left Double by Bit
5406#[inline]
5407#[target_feature(enable = "vector-enhancements-2")]
5408#[unstable(feature = "stdarch_s390x", issue = "135681")]
5409pub unsafe fn vec_sldb<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5410    static_assert_uimm_bits!(C, 3);
5411    a.vec_sldb::<C>(b)
5412}
5413
5414/// Vector Shift Right Double by Bit
5415#[inline]
5416#[target_feature(enable = "vector-enhancements-2")]
5417#[unstable(feature = "stdarch_s390x", issue = "135681")]
5418pub unsafe fn vec_srdb<T: sealed::VectorSrdb, const C: u32>(a: T, b: T) -> T {
5419    static_assert_uimm_bits!(C, 3);
5420    a.vec_srdb::<C>(b)
5421}
5422
5423/// Vector Compare Ranges
5424#[inline]
5425#[target_feature(enable = "vector")]
5426#[unstable(feature = "stdarch_s390x", issue = "135681")]
5427pub unsafe fn vec_cmprg<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5428    a.vstrc::<{ FindImm::Eq as u32 }>(b, c)
5429}
5430
5431/// Vector Compare Not in Ranges
5432#[inline]
5433#[target_feature(enable = "vector")]
5434#[unstable(feature = "stdarch_s390x", issue = "135681")]
5435pub unsafe fn vec_cmpnrg<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5436    a.vstrc::<{ FindImm::Ne as u32 }>(b, c)
5437}
5438
5439/// Vector Compare Ranges Index
5440#[inline]
5441#[target_feature(enable = "vector")]
5442#[unstable(feature = "stdarch_s390x", issue = "135681")]
5443pub unsafe fn vec_cmprg_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5444    a.vstrc::<{ FindImm::EqIdx as u32 }>(b, c)
5445}
5446
5447/// Vector Compare Not in Ranges Index
5448#[inline]
5449#[target_feature(enable = "vector")]
5450#[unstable(feature = "stdarch_s390x", issue = "135681")]
5451pub unsafe fn vec_cmpnrg_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5452    a.vstrc::<{ FindImm::NeIdx as u32 }>(b, c)
5453}
5454
5455/// Vector Compare Ranges with Condition Code
5456#[inline]
5457#[target_feature(enable = "vector")]
5458#[unstable(feature = "stdarch_s390x", issue = "135681")]
5459pub unsafe fn vec_cmprg_cc<T: sealed::VectorCompareRange>(
5460    a: T,
5461    b: T,
5462    c: T,
5463    d: *mut i32,
5464) -> T::Result {
5465    let (x, y) = a.vstrcs::<{ FindImm::Eq as u32 }>(b, c);
5466    d.write(y);
5467    x
5468}
5469
5470/// Vector Compare Not in Ranges with Condition Code
5471#[inline]
5472#[target_feature(enable = "vector")]
5473#[unstable(feature = "stdarch_s390x", issue = "135681")]
5474pub unsafe fn vec_cmpnrg_cc<T: sealed::VectorCompareRange>(
5475    a: T,
5476    b: T,
5477    c: T,
5478    d: *mut i32,
5479) -> T::Result {
5480    let (x, y) = a.vstrcs::<{ FindImm::Ne as u32 }>(b, c);
5481    d.write(y);
5482    x
5483}
5484
5485/// Vector Compare Ranges Index with Condition Code
5486#[inline]
5487#[target_feature(enable = "vector")]
5488#[unstable(feature = "stdarch_s390x", issue = "135681")]
5489pub unsafe fn vec_cmprg_idx_cc<T: sealed::VectorCompareRange>(
5490    a: T,
5491    b: T,
5492    c: T,
5493    d: *mut i32,
5494) -> T::Result {
5495    let (x, y) = a.vstrcs::<{ FindImm::EqIdx as u32 }>(b, c);
5496    d.write(y);
5497    x
5498}
5499
5500/// Vector Compare Not in Ranges Index with Condition Code
5501#[inline]
5502#[target_feature(enable = "vector")]
5503#[unstable(feature = "stdarch_s390x", issue = "135681")]
5504pub unsafe fn vec_cmpnrg_idx_cc<T: sealed::VectorCompareRange>(
5505    a: T,
5506    b: T,
5507    c: T,
5508    d: *mut i32,
5509) -> T::Result {
5510    let (x, y) = a.vstrcs::<{ FindImm::NeIdx as u32 }>(b, c);
5511    d.write(y);
5512    x
5513}
5514
5515/// Vector Compare Ranges or Zero Index
5516#[inline]
5517#[target_feature(enable = "vector")]
5518#[unstable(feature = "stdarch_s390x", issue = "135681")]
5519pub unsafe fn vec_cmprg_or_0_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5520    a.vstrcz::<{ FindImm::EqIdx as u32 }>(b, c)
5521}
5522
5523/// Vector Compare Not in Ranges or Zero Index
5524#[inline]
5525#[target_feature(enable = "vector")]
5526#[unstable(feature = "stdarch_s390x", issue = "135681")]
5527pub unsafe fn vec_cmpnrg_or_0_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5528    a.vstrcz::<{ FindImm::NeIdx as u32 }>(b, c)
5529}
5530
5531/// Vector Compare Ranges or Zero Index with Condition Code
5532#[inline]
5533#[target_feature(enable = "vector")]
5534#[unstable(feature = "stdarch_s390x", issue = "135681")]
5535pub unsafe fn vec_cmprg_or_0_idx_cc<T: sealed::VectorCompareRange>(
5536    a: T,
5537    b: T,
5538    c: T,
5539    d: *mut i32,
5540) -> T::Result {
5541    let (x, y) = a.vstrczs::<{ FindImm::EqIdx as u32 }>(b, c);
5542    d.write(y);
5543    x
5544}
5545
5546/// Vector Compare Not in Ranges or Zero Index with Condition Code
5547#[inline]
5548#[target_feature(enable = "vector")]
5549#[unstable(feature = "stdarch_s390x", issue = "135681")]
5550pub unsafe fn vec_cmpnrg_or_0_idx_cc<T: sealed::VectorCompareRange>(
5551    a: T,
5552    b: T,
5553    c: T,
5554    d: *mut i32,
5555) -> T::Result {
5556    let (x, y) = a.vstrczs::<{ FindImm::NeIdx as u32 }>(b, c);
5557    d.write(y);
5558    x
5559}
5560
5561/// Vector Compare Equal
5562#[inline]
5563#[target_feature(enable = "vector")]
5564#[unstable(feature = "stdarch_s390x", issue = "135681")]
5565pub unsafe fn vec_cmpeq<T: sealed::VectorEquality>(a: T, b: T) -> T::Result {
5566    a.vec_cmpeq(b)
5567}
5568
5569/// Vector Compare Not Equal
5570#[inline]
5571#[target_feature(enable = "vector")]
5572#[unstable(feature = "stdarch_s390x", issue = "135681")]
5573pub unsafe fn vec_cmpne<T: sealed::VectorEquality>(a: T, b: T) -> T::Result {
5574    a.vec_cmpne(b)
5575}
5576
5577/// Vector Compare Greater Than
5578#[inline]
5579#[target_feature(enable = "vector")]
5580#[unstable(feature = "stdarch_s390x", issue = "135681")]
5581pub unsafe fn vec_cmpgt<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5582    a.vec_cmpgt(b)
5583}
5584
5585/// Vector Compare Greater Than or Equal
5586#[inline]
5587#[target_feature(enable = "vector")]
5588#[unstable(feature = "stdarch_s390x", issue = "135681")]
5589pub unsafe fn vec_cmpge<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5590    a.vec_cmpge(b)
5591}
5592
5593/// Vector Compare Less
5594#[inline]
5595#[target_feature(enable = "vector")]
5596#[unstable(feature = "stdarch_s390x", issue = "135681")]
5597pub unsafe fn vec_cmplt<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5598    a.vec_cmplt(b)
5599}
5600
5601/// Vector Compare Less Than or Equal
5602#[inline]
5603#[target_feature(enable = "vector")]
5604#[unstable(feature = "stdarch_s390x", issue = "135681")]
5605pub unsafe fn vec_cmple<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5606    a.vec_cmple(b)
5607}
5608
5609/// Vector Compare Equal Index
5610#[inline]
5611#[target_feature(enable = "vector")]
5612#[unstable(feature = "stdarch_s390x", issue = "135681")]
5613pub unsafe fn vec_cmpeq_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5614    a.vec_cmpeq_idx(b)
5615}
5616/// Vector Compare Not Equal Index
5617#[inline]
5618#[target_feature(enable = "vector")]
5619#[unstable(feature = "stdarch_s390x", issue = "135681")]
5620pub unsafe fn vec_cmpne_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5621    a.vec_cmpne_idx(b)
5622}
5623/// Vector Compare Equal Index with Condition Code
5624#[inline]
5625#[target_feature(enable = "vector")]
5626#[unstable(feature = "stdarch_s390x", issue = "135681")]
5627pub unsafe fn vec_cmpeq_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5628    a.vec_cmpeq_idx_cc(b)
5629}
5630/// Vector Compare Not Equal Index with Condition Code
5631#[inline]
5632#[target_feature(enable = "vector")]
5633#[unstable(feature = "stdarch_s390x", issue = "135681")]
5634pub unsafe fn vec_cmpne_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5635    a.vec_cmpne_idx_cc(b)
5636}
5637/// Vector Compare Equal or Zero Index
5638#[inline]
5639#[target_feature(enable = "vector")]
5640#[unstable(feature = "stdarch_s390x", issue = "135681")]
5641pub unsafe fn vec_cmpeq_or_0_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5642    a.vec_cmpeq_or_0_idx(b)
5643}
5644/// Vector Compare Not Equal or Zero Index
5645#[inline]
5646#[target_feature(enable = "vector")]
5647#[unstable(feature = "stdarch_s390x", issue = "135681")]
5648pub unsafe fn vec_cmpne_or_0_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5649    a.vec_cmpne_or_0_idx(b)
5650}
5651/// Vector Compare Equal or Zero Index with Condition Code
5652#[inline]
5653#[target_feature(enable = "vector")]
5654#[unstable(feature = "stdarch_s390x", issue = "135681")]
5655pub unsafe fn vec_cmpeq_or_0_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5656    a.vec_cmpeq_or_0_idx_cc(b)
5657}
5658/// Vector Compare Not Equal or Zero Index with Condition Code
5659#[inline]
5660#[target_feature(enable = "vector")]
5661#[unstable(feature = "stdarch_s390x", issue = "135681")]
5662pub unsafe fn vec_cmpne_or_0_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5663    a.vec_cmpne_or_0_idx_cc(b)
5664}
5665
5666/// All Elements Equal
5667#[inline]
5668#[target_feature(enable = "vector")]
5669#[unstable(feature = "stdarch_s390x", issue = "135681")]
5670pub unsafe fn vec_all_eq<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5671    simd_reduce_all(vec_cmpeq(a, b)) as i32 as i32
5672}
5673
5674/// All Elements Not Equal
5675#[inline]
5676#[target_feature(enable = "vector")]
5677#[unstable(feature = "stdarch_s390x", issue = "135681")]
5678pub unsafe fn vec_all_ne<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5679    simd_reduce_all(vec_cmpne(a, b)) as i32
5680}
5681
5682/// Any Element Equal
5683#[inline]
5684#[target_feature(enable = "vector")]
5685#[unstable(feature = "stdarch_s390x", issue = "135681")]
5686pub unsafe fn vec_any_eq<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5687    simd_reduce_any(vec_cmpeq(a, b)) as i32
5688}
5689
5690/// Any Element Not Equal
5691#[inline]
5692#[target_feature(enable = "vector")]
5693#[unstable(feature = "stdarch_s390x", issue = "135681")]
5694pub unsafe fn vec_any_ne<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5695    simd_reduce_any(vec_cmpne(a, b)) as i32
5696}
5697
5698/// All Elements Less Than
5699#[inline]
5700#[target_feature(enable = "vector")]
5701#[unstable(feature = "stdarch_s390x", issue = "135681")]
5702pub unsafe fn vec_all_lt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5703    a.vec_all_lt(b)
5704}
5705
5706/// All Elements Less Than or Equal
5707#[inline]
5708#[target_feature(enable = "vector")]
5709#[unstable(feature = "stdarch_s390x", issue = "135681")]
5710pub unsafe fn vec_all_le<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5711    a.vec_all_le(b)
5712}
5713
5714/// All Elements Greater Than
5715#[inline]
5716#[target_feature(enable = "vector")]
5717#[unstable(feature = "stdarch_s390x", issue = "135681")]
5718pub unsafe fn vec_all_gt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5719    a.vec_all_gt(b)
5720}
5721
5722/// All Elements Greater Than or Equal
5723#[inline]
5724#[target_feature(enable = "vector")]
5725#[unstable(feature = "stdarch_s390x", issue = "135681")]
5726pub unsafe fn vec_all_ge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5727    a.vec_all_ge(b)
5728}
5729
5730/// All Elements Not Less Than
5731#[inline]
5732#[target_feature(enable = "vector")]
5733#[unstable(feature = "stdarch_s390x", issue = "135681")]
5734pub unsafe fn vec_all_nlt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5735    vec_all_ge(a, b)
5736}
5737
5738/// All Elements Not Less Than or Equal
5739#[inline]
5740#[target_feature(enable = "vector")]
5741#[unstable(feature = "stdarch_s390x", issue = "135681")]
5742pub unsafe fn vec_all_nle<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5743    vec_all_gt(a, b)
5744}
5745
5746/// All Elements Not Greater Than
5747#[inline]
5748#[target_feature(enable = "vector")]
5749#[unstable(feature = "stdarch_s390x", issue = "135681")]
5750pub unsafe fn vec_all_ngt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5751    vec_all_le(a, b)
5752}
5753
5754/// All Elements Not Greater Than or Equal
5755#[inline]
5756#[target_feature(enable = "vector")]
5757#[unstable(feature = "stdarch_s390x", issue = "135681")]
5758pub unsafe fn vec_all_nge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5759    vec_all_lt(a, b)
5760}
5761
5762/// Any Elements Less Than
5763#[inline]
5764#[target_feature(enable = "vector")]
5765#[unstable(feature = "stdarch_s390x", issue = "135681")]
5766pub unsafe fn vec_any_lt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5767    !vec_all_ge(a, b)
5768}
5769
5770/// Any Elements Less Than or Equal
5771#[inline]
5772#[target_feature(enable = "vector")]
5773#[unstable(feature = "stdarch_s390x", issue = "135681")]
5774pub unsafe fn vec_any_le<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5775    !vec_all_gt(a, b)
5776}
5777
5778/// Any Elements Greater Than
5779#[inline]
5780#[target_feature(enable = "vector")]
5781#[unstable(feature = "stdarch_s390x", issue = "135681")]
5782pub unsafe fn vec_any_gt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5783    !vec_all_le(a, b)
5784}
5785
5786/// Any Elements Greater Than or Equal
5787#[inline]
5788#[target_feature(enable = "vector")]
5789#[unstable(feature = "stdarch_s390x", issue = "135681")]
5790pub unsafe fn vec_any_ge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5791    !vec_all_lt(a, b)
5792}
5793
5794/// Any Elements Not Less Than
5795#[inline]
5796#[target_feature(enable = "vector")]
5797#[unstable(feature = "stdarch_s390x", issue = "135681")]
5798pub unsafe fn vec_any_nlt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5799    vec_any_ge(a, b)
5800}
5801
5802/// Any Elements Not Less Than or Equal
5803#[inline]
5804#[target_feature(enable = "vector")]
5805#[unstable(feature = "stdarch_s390x", issue = "135681")]
5806pub unsafe fn vec_any_nle<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5807    vec_any_gt(a, b)
5808}
5809
5810/// Any Elements Not Greater Than
5811#[inline]
5812#[target_feature(enable = "vector")]
5813#[unstable(feature = "stdarch_s390x", issue = "135681")]
5814pub unsafe fn vec_any_ngt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5815    vec_any_le(a, b)
5816}
5817
5818/// Any Elements Not Greater Than or Equal
5819#[inline]
5820#[target_feature(enable = "vector")]
5821#[unstable(feature = "stdarch_s390x", issue = "135681")]
5822pub unsafe fn vec_any_nge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5823    vec_any_lt(a, b)
5824}
5825
5826/// Vector Extract
5827#[inline]
5828#[target_feature(enable = "vector")]
5829#[unstable(feature = "stdarch_s390x", issue = "135681")]
5830pub unsafe fn vec_extract<T: sealed::VectorExtract>(a: T, b: i32) -> T::ElementType {
5831    T::vec_extract(a, b)
5832}
5833
5834/// Vector Insert
5835#[inline]
5836#[target_feature(enable = "vector")]
5837#[unstable(feature = "stdarch_s390x", issue = "135681")]
5838pub unsafe fn vec_insert<T: sealed::VectorInsert>(a: T::ElementType, b: T, c: i32) -> T {
5839    T::vec_insert(a, b, c)
5840}
5841
5842/// Vector Insert and Zero
5843#[inline]
5844#[target_feature(enable = "vector")]
5845#[unstable(feature = "stdarch_s390x", issue = "135681")]
5846pub unsafe fn vec_insert_and_zero<T: sealed::VectorInsertAndZero>(a: *const T::ElementType) -> T {
5847    T::vec_insert_and_zero(a)
5848}
5849
5850/// Vector Promote
5851#[inline]
5852#[target_feature(enable = "vector")]
5853#[unstable(feature = "stdarch_s390x", issue = "135681")]
5854pub unsafe fn vec_promote<T: sealed::VectorPromote>(a: T::ElementType, b: i32) -> MaybeUninit<T> {
5855    T::vec_promote(a, b)
5856}
5857
5858#[cfg(test)]
5859mod tests {
5860    use super::*;
5861
5862    use std::mem::transmute;
5863
5864    use crate::core_arch::simd::*;
5865    use stdarch_test::simd_test;
5866
5867    impl<const N: usize> ShuffleMask<N> {
5868        fn as_array(&self) -> &[u32; N] {
5869            unsafe { std::mem::transmute(self) }
5870        }
5871    }
5872
5873    #[test]
5874    fn reverse_mask() {
5875        assert_eq!(ShuffleMask::<4>::reverse().as_array(), &[3, 2, 1, 0]);
5876    }
5877
5878    #[test]
5879    fn mergel_mask() {
5880        assert_eq!(ShuffleMask::<4>::merge_low().as_array(), &[2, 6, 3, 7]);
5881    }
5882
5883    #[test]
5884    fn mergeh_mask() {
5885        assert_eq!(ShuffleMask::<4>::merge_high().as_array(), &[0, 4, 1, 5]);
5886    }
5887
5888    #[test]
5889    fn pack_mask() {
5890        assert_eq!(ShuffleMask::<4>::pack().as_array(), &[1, 3, 5, 7]);
5891    }
5892
5893    #[test]
5894    fn test_vec_mask() {
5895        assert_eq!(
5896            genmask::<0x00FF>(),
5897            [
5898                0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
5899            ]
5900        );
5901    }
5902
5903    #[test]
5904    fn test_genmasks() {
5905        assert_eq!(genmasks(u8::BITS, 3, 5), 28);
5906        assert_eq!(genmasks(u8::BITS, 3, 7), 31);
5907
5908        // If a or b is greater than 8, the operation is performed as if the value gets modulo by 8.
5909        assert_eq!(genmasks(u8::BITS, 3 + 8, 7 + 8), 31);
5910        // If a is greater than b, the operation is perform as if b equals 7.
5911        assert_eq!(genmasks(u8::BITS, 5, 4), genmasks(u8::BITS, 5, 7));
5912
5913        assert_eq!(
5914            genmasks(u16::BITS, 4, 12) as u16,
5915            u16::from_be_bytes([15, -8i8 as u8])
5916        );
5917        assert_eq!(
5918            genmasks(u32::BITS, 4, 29) as u32,
5919            u32::from_be_bytes([15, 0xFF, 0xFF, -4i8 as u8])
5920        );
5921    }
5922
5923    macro_rules! test_vec_1 {
5924        { $name: ident, $fn:ident, f32x4, [$($a:expr),+], ~[$($d:expr),+] } => {
5925            #[simd_test(enable = "vector")]
5926            unsafe fn $name() {
5927                let a: vector_float = transmute(f32x4::new($($a),+));
5928
5929                let d: vector_float = transmute(f32x4::new($($d),+));
5930                let r = transmute(vec_cmple(vec_abs(vec_sub($fn(a), d)), vec_splats(f32::EPSILON)));
5931                let e = m32x4::new(true, true, true, true);
5932                assert_eq!(e, r);
5933            }
5934        };
5935        { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($d:expr),+] } => {
5936            test_vec_1! { $name, $fn, $ty -> $ty, [$($a),+], [$($d),+] }
5937        };
5938        { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($d:expr),+] } => {
5939            #[simd_test(enable = "vector")]
5940            unsafe fn $name() {
5941                let a: s_t_l!($ty) = transmute($ty::new($($a),+));
5942
5943                let d = $ty_out::new($($d),+);
5944                let r : $ty_out = transmute($fn(a));
5945                assert_eq!(d, r);
5946            }
5947        }
5948    }
5949
5950    macro_rules! test_vec_2 {
5951        { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5952            test_vec_2! { $name, $fn, $ty -> $ty, [$($a),+], [$($b),+], [$($d),+] }
5953        };
5954        { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5955            test_vec_2! { $name, $fn, $ty, $ty -> $ty, [$($a),+], [$($b),+], [$($d),+] }
5956         };
5957        { $name: ident, $fn:ident, $ty1: ident, $ty2: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5958            #[simd_test(enable = "vector")]
5959            unsafe fn $name() {
5960                let a: s_t_l!($ty1) = transmute($ty1::new($($a),+));
5961                let b: s_t_l!($ty2) = transmute($ty2::new($($b),+));
5962
5963                let d = $ty_out::new($($d),+);
5964                let r : $ty_out = transmute($fn(a, b));
5965                assert_eq!(d, r);
5966            }
5967         };
5968         { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], $d:expr } => {
5969            #[simd_test(enable = "vector")]
5970            unsafe fn $name() {
5971                let a: s_t_l!($ty) = transmute($ty::new($($a),+));
5972                let b: s_t_l!($ty) = transmute($ty::new($($b),+));
5973
5974                let r : $ty_out = transmute($fn(a, b));
5975                assert_eq!($d, r);
5976            }
5977         }
5978   }
5979
5980    #[simd_test(enable = "vector")]
5981    unsafe fn vec_add_i32x4_i32x4() {
5982        let x = i32x4::new(1, 2, 3, 4);
5983        let y = i32x4::new(4, 3, 2, 1);
5984        let x: vector_signed_int = transmute(x);
5985        let y: vector_signed_int = transmute(y);
5986        let z = vec_add(x, y);
5987        assert_eq!(i32x4::splat(5), transmute(z));
5988    }
5989
5990    macro_rules! test_vec_sub {
5991        { $name: ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5992            test_vec_2! {$name, vec_sub, $ty, [$($a),+], [$($b),+], [$($d),+] }
5993        }
5994    }
5995
5996    test_vec_sub! { test_vec_sub_f32x4, f32x4,
5997    [-1.0, 0.0, 1.0, 2.0],
5998    [2.0, 1.0, -1.0, -2.0],
5999    [-3.0, -1.0, 2.0, 4.0] }
6000
6001    test_vec_sub! { test_vec_sub_f64x2, f64x2,
6002    [-1.0, 0.0],
6003    [2.0, 1.0],
6004    [-3.0, -1.0] }
6005
6006    test_vec_sub! { test_vec_sub_i64x2, i64x2,
6007    [-1, 0],
6008    [2, 1],
6009    [-3, -1] }
6010
6011    test_vec_sub! { test_vec_sub_u64x2, u64x2,
6012    [0, 1],
6013    [1, 0],
6014    [u64::MAX, 1] }
6015
6016    test_vec_sub! { test_vec_sub_i32x4, i32x4,
6017    [-1, 0, 1, 2],
6018    [2, 1, -1, -2],
6019    [-3, -1, 2, 4] }
6020
6021    test_vec_sub! { test_vec_sub_u32x4, u32x4,
6022    [0, 0, 1, 2],
6023    [2, 1, 0, 0],
6024    [4294967294, 4294967295, 1, 2] }
6025
6026    test_vec_sub! { test_vec_sub_i16x8, i16x8,
6027    [-1, 0, 1, 2, -1, 0, 1, 2],
6028    [2, 1, -1, -2, 2, 1, -1, -2],
6029    [-3, -1, 2, 4, -3, -1, 2, 4] }
6030
6031    test_vec_sub! { test_vec_sub_u16x8, u16x8,
6032    [0, 0, 1, 2, 0, 0, 1, 2],
6033    [2, 1, 0, 0, 2, 1, 0, 0],
6034    [65534, 65535, 1, 2, 65534, 65535, 1, 2] }
6035
6036    test_vec_sub! { test_vec_sub_i8x16, i8x16,
6037    [-1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2],
6038    [2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2],
6039    [-3, -1, 2, 4, -3, -1, 2, 4, -3, -1, 2, 4, -3, -1, 2, 4] }
6040
6041    test_vec_sub! { test_vec_sub_u8x16, u8x16,
6042    [0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 1, 2],
6043    [2, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 0],
6044    [254, 255, 1, 2, 254, 255, 1, 2, 254, 255, 1, 2, 254, 255, 1, 2] }
6045
6046    macro_rules! test_vec_mul {
6047        { $name: ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
6048            test_vec_2! {$name, vec_mul, $ty, [$($a),+], [$($b),+], [$($d),+] }
6049        }
6050    }
6051
6052    test_vec_mul! { test_vec_mul_f32x4, f32x4,
6053    [-1.0, 0.0, 1.0, 2.0],
6054    [2.0, 1.0, -1.0, -2.0],
6055    [-2.0, 0.0, -1.0, -4.0] }
6056
6057    test_vec_mul! { test_vec_mul_f64x2, f64x2,
6058    [-1.0, 0.0],
6059    [2.0, 1.0],
6060    [-2.0, 0.0] }
6061
6062    test_vec_mul! { test_vec_mul_i64x2, i64x2,
6063    [i64::MAX, -4],
6064    [2, 3],
6065    [i64::MAX.wrapping_mul(2), -12] }
6066
6067    test_vec_mul! { test_vec_mul_u64x2, u64x2,
6068    [u64::MAX, 4],
6069    [2, 3],
6070    [u64::MAX.wrapping_mul(2), 12] }
6071
6072    test_vec_mul! { test_vec_mul_i32x4, i32x4,
6073    [-1, 0, 1, 2],
6074    [2, 1, -1, -2],
6075    [-2, 0, -1, -4] }
6076
6077    test_vec_mul! { test_vec_mul_u32x4, u32x4,
6078    [0, u32::MAX - 1, 1, 2],
6079    [5, 6, 7, 8],
6080    [0, 4294967284, 7, 16] }
6081
6082    test_vec_mul! { test_vec_mul_i16x8, i16x8,
6083    [-1, 0, 1, 2, -1, 0, 1, 2],
6084    [2, 1, -1, -2, 2, 1, -1, -2],
6085    [-2, 0, -1, -4, -2, 0, -1, -4] }
6086
6087    test_vec_mul! { test_vec_mul_u16x8, u16x8,
6088    [0, u16::MAX - 1, 1, 2, 3, 4, 5, 6],
6089    [5, 6, 7, 8, 9, 8, 7, 6],
6090    [0, 65524, 7, 16, 27, 32, 35, 36] }
6091
6092    test_vec_mul! { test_vec_mul_i8x16, i8x16,
6093    [-1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2],
6094    [2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2],
6095    [-2, 0, -1, -4, -2, 0, -1, -4, -2, 0, -1, -4, -2, 0, -1, -4] }
6096
6097    test_vec_mul! { test_vec_mul_u8x16, u8x16,
6098    [0, u8::MAX - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4],
6099    [5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 0, u8::MAX, 1, 2, 3, 4],
6100    [0, 244, 7, 16, 27, 32, 35, 36, 35, 32, 0, 248, 7, 12, 15, 16] }
6101
6102    macro_rules! test_vec_abs {
6103        { $name: ident, $ty: ident, $a: expr, $d: expr } => {
6104            #[simd_test(enable = "vector")]
6105            unsafe fn $name() {
6106                let a: s_t_l!($ty) = vec_splats($a);
6107                let a: s_t_l!($ty) = vec_abs(a);
6108                let d = $ty::splat($d);
6109                assert_eq!(d, transmute(a));
6110            }
6111        }
6112    }
6113
6114    test_vec_abs! { test_vec_abs_i8, i8x16, -42i8, 42i8 }
6115    test_vec_abs! { test_vec_abs_i16, i16x8, -42i16, 42i16 }
6116    test_vec_abs! { test_vec_abs_i32, i32x4, -42i32, 42i32 }
6117    test_vec_abs! { test_vec_abs_i64, i64x2, -42i64, 42i64 }
6118    test_vec_abs! { test_vec_abs_f32, f32x4, -42f32, 42f32 }
6119    test_vec_abs! { test_vec_abs_f64, f64x2, -42f64, 42f64 }
6120
6121    test_vec_1! { test_vec_nabs, vec_nabs, f32x4,
6122    [core::f32::consts::PI, 1.0, 0.0, -1.0],
6123    [-core::f32::consts::PI, -1.0, 0.0, -1.0] }
6124
6125    test_vec_2! { test_vec_andc, vec_andc, i32x4,
6126    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6127    [0b00110011, 0b11110011, 0b00001100, 0b10000000],
6128    [0b11001100, 0b00001100, 0b11000000, 0b01001100] }
6129
6130    test_vec_2! { test_vec_and, vec_and, i32x4,
6131    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6132    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6133    [0b00000000, 0b11000000, 0b00001100, 0b00000000] }
6134
6135    test_vec_2! { test_vec_nand, vec_nand, i32x4,
6136    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6137    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6138    [!0b00000000, !0b11000000, !0b00001100, !0b00000000] }
6139
6140    test_vec_2! { test_vec_orc, vec_orc, u32x4,
6141    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6142    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6143    [0b11001100 | !0b00110011, 0b11001100 | !0b11110011, 0b11001100 | !0b00001100, 0b11001100 | !0b00000000] }
6144
6145    test_vec_2! { test_vec_or, vec_or, i32x4,
6146    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6147    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6148    [0b11111111, 0b11111111, 0b11001100, 0b11001100] }
6149
6150    test_vec_2! { test_vec_nor, vec_nor, i32x4,
6151    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6152    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6153    [!0b11111111, !0b11111111, !0b11001100, !0b11001100] }
6154
6155    test_vec_2! { test_vec_xor, vec_xor, i32x4,
6156    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6157    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6158    [0b11111111, 0b00111111, 0b11000000, 0b11001100] }
6159
6160    test_vec_2! { test_vec_eqv, vec_eqv, i32x4,
6161    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6162    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6163    [!0b11111111, !0b00111111, !0b11000000, !0b11001100] }
6164
6165    test_vec_1! { test_vec_floor_f32, vec_floor, f32x4,
6166        [1.1, 1.9, -0.5, -0.9],
6167        [1.0, 1.0, -1.0, -1.0]
6168    }
6169
6170    test_vec_1! { test_vec_floor_f64_1, vec_floor, f64x2,
6171        [1.1, 1.9],
6172        [1.0, 1.0]
6173    }
6174    test_vec_1! { test_vec_floor_f64_2, vec_floor, f64x2,
6175        [-0.5, -0.9],
6176        [-1.0, -1.0]
6177    }
6178
6179    test_vec_1! { test_vec_ceil_f32, vec_ceil, f32x4,
6180        [0.1, 0.5, 0.6, 0.9],
6181        [1.0, 1.0, 1.0, 1.0]
6182    }
6183    test_vec_1! { test_vec_ceil_f64_1, vec_ceil, f64x2,
6184        [0.1, 0.5],
6185        [1.0, 1.0]
6186    }
6187    test_vec_1! { test_vec_ceil_f64_2, vec_ceil, f64x2,
6188        [0.6, 0.9],
6189        [1.0, 1.0]
6190    }
6191
6192    test_vec_1! { test_vec_round_f32, vec_round, f32x4,
6193        [0.1, 0.5, 0.6, 0.9],
6194        [0.0, 0.0, 1.0, 1.0]
6195    }
6196
6197    test_vec_1! { test_vec_round_f32_even_odd, vec_round, f32x4,
6198        [0.5, 1.5, 2.5, 3.5],
6199        [0.0, 2.0, 2.0, 4.0]
6200    }
6201
6202    test_vec_1! { test_vec_round_f64_1, vec_round, f64x2,
6203        [0.1, 0.5],
6204        [0.0, 0.0]
6205    }
6206    test_vec_1! { test_vec_round_f64_2, vec_round, f64x2,
6207        [0.6, 0.9],
6208        [1.0, 1.0]
6209    }
6210
6211    test_vec_1! { test_vec_roundc_f32, vec_roundc, f32x4,
6212        [0.1, 0.5, 0.6, 0.9],
6213        [0.0, 0.0, 1.0, 1.0]
6214    }
6215
6216    test_vec_1! { test_vec_roundc_f32_even_odd, vec_roundc, f32x4,
6217        [0.5, 1.5, 2.5, 3.5],
6218        [0.0, 2.0, 2.0, 4.0]
6219    }
6220
6221    test_vec_1! { test_vec_roundc_f64_1, vec_roundc, f64x2,
6222        [0.1, 0.5],
6223        [0.0, 0.0]
6224    }
6225    test_vec_1! { test_vec_roundc_f64_2, vec_roundc, f64x2,
6226        [0.6, 0.9],
6227        [1.0, 1.0]
6228    }
6229
6230    test_vec_1! { test_vec_rint_f32, vec_rint, f32x4,
6231        [0.1, 0.5, 0.6, 0.9],
6232        [0.0, 0.0, 1.0, 1.0]
6233    }
6234
6235    test_vec_1! { test_vec_rint_f32_even_odd, vec_rint, f32x4,
6236        [0.5, 1.5, 2.5, 3.5],
6237        [0.0, 2.0, 2.0, 4.0]
6238    }
6239
6240    test_vec_1! { test_vec_rint_f64_1, vec_rint, f64x2,
6241        [0.1, 0.5],
6242        [0.0, 0.0]
6243    }
6244    test_vec_1! { test_vec_rint_f64_2, vec_rint, f64x2,
6245        [0.6, 0.9],
6246        [1.0, 1.0]
6247    }
6248
6249    test_vec_2! { test_vec_sll, vec_sll, i32x4, u8x16 -> i32x4,
6250    [1, 1, 1, 1],
6251    [0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 8],
6252    [1 << 2, 1 << 3, 1 << 4, 1] }
6253
6254    test_vec_2! { test_vec_srl, vec_srl, i32x4, u8x16 -> i32x4,
6255    [0b1000, 0b1000, 0b1000, 0b1000],
6256    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6257    [4, 2, 1, 8] }
6258
6259    test_vec_2! { test_vec_sral_pos, vec_sral, u32x4, u8x16 -> i32x4,
6260    [0b1000, 0b1000, 0b1000, 0b1000],
6261    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6262    [4, 2, 1, 8] }
6263
6264    test_vec_2! { test_vec_sral_neg, vec_sral, i32x4, u8x16 -> i32x4,
6265    [-8, -8, -8, -8],
6266    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6267    [-4, -2, -1, -8] }
6268
6269    test_vec_1! { test_vec_reve_f32, vec_reve, f32x4,
6270        [0.1, 0.5, 0.6, 0.9],
6271        [0.9, 0.6, 0.5, 0.1]
6272    }
6273
6274    test_vec_1! { test_vec_revb_u32, vec_revb, u32x4,
6275        [0xAABBCCDD, 0xEEFF0011, 0x22334455, 0x66778899],
6276        [0xDDCCBBAA, 0x1100FFEE, 0x55443322, 0x99887766]
6277    }
6278
6279    test_vec_2! { test_vec_mergeh_u32, vec_mergeh, u32x4,
6280        [0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD],
6281        [0x00000000, 0x11111111, 0x22222222, 0x33333333],
6282        [0xAAAAAAAA, 0x00000000, 0xBBBBBBBB, 0x11111111]
6283    }
6284
6285    test_vec_2! { test_vec_mergel_u32, vec_mergel, u32x4,
6286        [0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD],
6287        [0x00000000, 0x11111111, 0x22222222, 0x33333333],
6288        [0xCCCCCCCC, 0x22222222, 0xDDDDDDDD, 0x33333333]
6289    }
6290
6291    macro_rules! test_vec_perm {
6292        {$name:ident,
6293         $shorttype:ident, $longtype:ident,
6294         [$($a:expr),+], [$($b:expr),+], [$($c:expr),+], [$($d:expr),+]} => {
6295            #[simd_test(enable = "vector")]
6296            unsafe fn $name() {
6297                let a: $longtype = transmute($shorttype::new($($a),+));
6298                let b: $longtype = transmute($shorttype::new($($b),+));
6299                let c: vector_unsigned_char = transmute(u8x16::new($($c),+));
6300                let d = $shorttype::new($($d),+);
6301
6302                let r: $shorttype = transmute(vec_perm(a, b, c));
6303                assert_eq!(d, r);
6304            }
6305        }
6306    }
6307
6308    test_vec_perm! {test_vec_perm_u8x16,
6309    u8x16, vector_unsigned_char,
6310    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
6311    [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115],
6312    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6313     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6314    [0, 1, 100, 101, 2, 3, 102, 103, 4, 5, 104, 105, 6, 7, 106, 107]}
6315    test_vec_perm! {test_vec_perm_i8x16,
6316    i8x16, vector_signed_char,
6317    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
6318    [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115],
6319    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6320     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6321    [0, 1, 100, 101, 2, 3, 102, 103, 4, 5, 104, 105, 6, 7, 106, 107]}
6322
6323    test_vec_perm! {test_vec_perm_m8x16,
6324    m8x16, vector_bool_char,
6325    [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false],
6326    [true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true],
6327    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6328     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6329    [false, false, true, true, false, false, true, true, false, false, true, true, false, false, true, true]}
6330    test_vec_perm! {test_vec_perm_u16x8,
6331    u16x8, vector_unsigned_short,
6332    [0, 1, 2, 3, 4, 5, 6, 7],
6333    [10, 11, 12, 13, 14, 15, 16, 17],
6334    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6335     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6336    [0, 10, 1, 11, 2, 12, 3, 13]}
6337    test_vec_perm! {test_vec_perm_i16x8,
6338    i16x8, vector_signed_short,
6339    [0, 1, 2, 3, 4, 5, 6, 7],
6340    [10, 11, 12, 13, 14, 15, 16, 17],
6341    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6342     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6343    [0, 10, 1, 11, 2, 12, 3, 13]}
6344    test_vec_perm! {test_vec_perm_m16x8,
6345    m16x8, vector_bool_short,
6346    [false, false, false, false, false, false, false, false],
6347    [true, true, true, true, true, true, true, true],
6348    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6349     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6350    [false, true, false, true, false, true, false, true]}
6351
6352    test_vec_perm! {test_vec_perm_u32x4,
6353    u32x4, vector_unsigned_int,
6354    [0, 1, 2, 3],
6355    [10, 11, 12, 13],
6356    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6357     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6358    [0, 10, 1, 11]}
6359    test_vec_perm! {test_vec_perm_i32x4,
6360    i32x4, vector_signed_int,
6361    [0, 1, 2, 3],
6362    [10, 11, 12, 13],
6363    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6364     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6365    [0, 10, 1, 11]}
6366    test_vec_perm! {test_vec_perm_m32x4,
6367    m32x4, vector_bool_int,
6368    [false, false, false, false],
6369    [true, true, true, true],
6370    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6371     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6372    [false, true, false, true]}
6373    test_vec_perm! {test_vec_perm_f32x4,
6374    f32x4, vector_float,
6375    [0.0, 1.0, 2.0, 3.0],
6376    [1.0, 1.1, 1.2, 1.3],
6377    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6378     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6379    [0.0, 1.0, 1.0, 1.1]}
6380
6381    test_vec_1! { test_vec_sqrt, vec_sqrt, f32x4,
6382    [core::f32::consts::PI, 1.0, 25.0, 2.0],
6383    [core::f32::consts::PI.sqrt(), 1.0, 5.0, core::f32::consts::SQRT_2] }
6384
6385    test_vec_2! { test_vec_find_any_eq, vec_find_any_eq, i32x4, i32x4 -> u32x4,
6386        [1, -2, 3, -4],
6387        [-5, 3, -7, 8],
6388        [0, 0, 0xFFFFFFFF, 0]
6389    }
6390
6391    test_vec_2! { test_vec_find_any_ne, vec_find_any_ne, i32x4, i32x4 -> u32x4,
6392        [1, -2, 3, -4],
6393        [-5, 3, -7, 8],
6394        [0xFFFFFFFF, 0xFFFFFFFF, 0, 0xFFFFFFFF]
6395    }
6396
6397    test_vec_2! { test_vec_find_any_eq_idx_1, vec_find_any_eq_idx, i32x4, i32x4 -> u32x4,
6398        [1, 2, 3, 4],
6399        [5, 3, 7, 8],
6400        [0, 8, 0, 0]
6401    }
6402    test_vec_2! { test_vec_find_any_eq_idx_2, vec_find_any_eq_idx, i32x4, i32x4 -> u32x4,
6403        [1, 2, 3, 4],
6404        [5, 6, 7, 8],
6405        [0, 16, 0, 0]
6406    }
6407
6408    test_vec_2! { test_vec_find_any_ne_idx_1, vec_find_any_ne_idx, i32x4, i32x4 -> u32x4,
6409        [1, 2, 3, 4],
6410        [1, 5, 3, 4],
6411        [0, 4, 0, 0]
6412    }
6413    test_vec_2! { test_vec_find_any_ne_idx_2, vec_find_any_ne_idx, i32x4, i32x4 -> u32x4,
6414        [1, 2, 3, 4],
6415        [1, 2, 3, 4],
6416        [0, 16, 0, 0]
6417    }
6418
6419    test_vec_2! { test_vec_find_any_eq_or_0_idx_1, vec_find_any_eq_or_0_idx, i32x4, i32x4 -> u32x4,
6420        [1, 2, 0, 4],
6421        [5, 6, 7, 8],
6422        [0, 8, 0, 0]
6423    }
6424    test_vec_2! { test_vec_find_any_ne_or_0_idx_1, vec_find_any_ne_or_0_idx, i32x4, i32x4 -> u32x4,
6425        [1, 2, 0, 4],
6426        [1, 2, 3, 4],
6427        [0, 8, 0, 0]
6428    }
6429
6430    #[simd_test(enable = "vector")]
6431    fn test_vec_find_any_eq_cc() {
6432        let a = vector_unsigned_int([1, 2, 3, 4]);
6433        let b = vector_unsigned_int([5, 3, 7, 8]);
6434
6435        let (d, c) = unsafe { vec_find_any_eq_cc(a, b) };
6436        assert_eq!(c, 1);
6437        assert_eq!(d.as_array(), &[0, 0, -1, 0]);
6438
6439        let a = vector_unsigned_int([1, 2, 3, 4]);
6440        let b = vector_unsigned_int([5, 6, 7, 8]);
6441        let (d, c) = unsafe { vec_find_any_eq_cc(a, b) };
6442        assert_eq!(c, 3);
6443        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6444    }
6445
6446    #[simd_test(enable = "vector")]
6447    fn test_vec_find_any_ne_cc() {
6448        let a = vector_unsigned_int([1, 2, 3, 4]);
6449        let b = vector_unsigned_int([5, 3, 7, 8]);
6450
6451        let (d, c) = unsafe { vec_find_any_ne_cc(a, b) };
6452        assert_eq!(c, 1);
6453        assert_eq!(d.as_array(), &[-1, -1, 0, -1]);
6454
6455        let a = vector_unsigned_int([1, 2, 3, 4]);
6456        let b = vector_unsigned_int([1, 2, 3, 4]);
6457        let (d, c) = unsafe { vec_find_any_ne_cc(a, b) };
6458        assert_eq!(c, 3);
6459        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6460    }
6461
6462    #[simd_test(enable = "vector")]
6463    fn test_vec_find_any_eq_idx_cc() {
6464        let a = vector_unsigned_int([1, 2, 3, 4]);
6465        let b = vector_unsigned_int([5, 3, 7, 8]);
6466
6467        let (d, c) = unsafe { vec_find_any_eq_idx_cc(a, b) };
6468        assert_eq!(c, 1);
6469        assert_eq!(d.as_array(), &[0, 8, 0, 0]);
6470
6471        let a = vector_unsigned_int([1, 2, 3, 4]);
6472        let b = vector_unsigned_int([5, 6, 7, 8]);
6473        let (d, c) = unsafe { vec_find_any_eq_idx_cc(a, b) };
6474        assert_eq!(c, 3);
6475        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6476    }
6477
6478    #[simd_test(enable = "vector")]
6479    fn test_vec_find_any_ne_idx_cc() {
6480        let a = vector_unsigned_int([5, 2, 3, 4]);
6481        let b = vector_unsigned_int([5, 3, 7, 8]);
6482
6483        let (d, c) = unsafe { vec_find_any_ne_idx_cc(a, b) };
6484        assert_eq!(c, 1);
6485        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6486
6487        let a = vector_unsigned_int([1, 2, 3, 4]);
6488        let b = vector_unsigned_int([1, 2, 3, 4]);
6489        let (d, c) = unsafe { vec_find_any_ne_idx_cc(a, b) };
6490        assert_eq!(c, 3);
6491        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6492    }
6493
6494    #[simd_test(enable = "vector")]
6495    fn test_vec_find_any_eq_or_0_idx_cc() {
6496        // if no element of a matches any element of b with an equal value, and there is at least one element from a with a value of 0
6497        let a = vector_unsigned_int([0, 1, 2, 3]);
6498        let b = vector_unsigned_int([4, 5, 6, 7]);
6499        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6500        assert_eq!(c, 0);
6501        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6502
6503        // if at least one element of a matches any element of b with an equal value, and no elements of a with a value of 0
6504        let a = vector_unsigned_int([1, 2, 3, 4]);
6505        let b = vector_unsigned_int([5, 2, 3, 4]);
6506        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6507        assert_eq!(c, 1);
6508        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6509
6510        // if at least one element of a matches any element of b with an equal value, and there is at least one element from a has a value of 0
6511        let a = vector_unsigned_int([1, 2, 3, 0]);
6512        let b = vector_unsigned_int([1, 2, 3, 4]);
6513        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6514        assert_eq!(c, 2);
6515        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6516
6517        // if no element of a matches any element of b with an equal value, and there is no element from a with a value of 0.
6518        let a = vector_unsigned_int([1, 2, 3, 4]);
6519        let b = vector_unsigned_int([5, 6, 7, 8]);
6520        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6521        assert_eq!(c, 3);
6522        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6523    }
6524
6525    #[simd_test(enable = "vector")]
6526    fn test_vec_find_any_ne_or_0_idx_cc() {
6527        // if no element of a matches any element of b with a not equal value, and there is at least one element from a with a value of 0.
6528        let a = vector_unsigned_int([0, 1, 2, 3]);
6529        let b = vector_unsigned_int([4, 1, 2, 3]);
6530        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6531        assert_eq!(c, 0);
6532        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6533
6534        // if at least one element of a matches any element of b with a not equal value, and no elements of a with a value of 0.
6535        let a = vector_unsigned_int([4, 2, 3, 4]);
6536        let b = vector_unsigned_int([4, 5, 6, 7]);
6537        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6538        assert_eq!(c, 1);
6539        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6540
6541        // if at least one element of a matches any element of b with a not equal value, and there is at least one element from a has a value of 0.
6542        let a = vector_unsigned_int([1, 0, 1, 1]);
6543        let b = vector_unsigned_int([4, 5, 6, 7]);
6544        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6545        assert_eq!(c, 2);
6546        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6547
6548        // if no element of a matches any element of b with a not equal value, and there is no element from a with a value of 0.
6549        let a = vector_unsigned_int([4, 4, 4, 4]);
6550        let b = vector_unsigned_int([4, 5, 6, 7]);
6551        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6552        assert_eq!(c, 3);
6553        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6554    }
6555
6556    #[simd_test(enable = "vector")]
6557    fn test_vector_load() {
6558        let expected = [0xAAAA_AAAA, 0xBBBB_BBBB, 0xCCCC_CCCC, 0xDDDD_DDDD];
6559
6560        let source: [u32; 8] = [
6561            0xAAAA_AAAA,
6562            0xBBBB_BBBB,
6563            0xCCCC_CCCC,
6564            0xDDDD_DDDD,
6565            0,
6566            0,
6567            0,
6568            0,
6569        ];
6570        assert_eq!(
6571            unsafe { vec_xl::<vector_unsigned_int>(0, source.as_ptr()) }.as_array(),
6572            &expected
6573        );
6574
6575        // offset is in bytes
6576        let source: [u32; 8] = [
6577            0x0000_AAAA,
6578            0xAAAA_BBBB,
6579            0xBBBB_CCCC,
6580            0xCCCC_DDDD,
6581            0xDDDD_0000,
6582            0,
6583            0,
6584            0,
6585        ];
6586        assert_eq!(
6587            unsafe { vec_xl::<vector_unsigned_int>(2, source.as_ptr()) }.as_array(),
6588            &expected
6589        );
6590    }
6591
6592    #[simd_test(enable = "vector")]
6593    fn test_vector_store() {
6594        let vec = vector_unsigned_int([0xAAAA_AAAA, 0xBBBB_BBBB, 0xCCCC_CCCC, 0xDDDD_DDDD]);
6595
6596        let mut dest = [0u32; 8];
6597        unsafe { vec_xst(vec, 0, dest.as_mut_ptr()) };
6598        assert_eq!(
6599            dest,
6600            [
6601                0xAAAA_AAAA,
6602                0xBBBB_BBBB,
6603                0xCCCC_CCCC,
6604                0xDDDD_DDDD,
6605                0,
6606                0,
6607                0,
6608                0
6609            ]
6610        );
6611
6612        // offset is in bytes
6613        let mut dest = [0u32; 8];
6614        unsafe { vec_xst(vec, 2, dest.as_mut_ptr()) };
6615        assert_eq!(
6616            dest,
6617            [
6618                0x0000_AAAA,
6619                0xAAAA_BBBB,
6620                0xBBBB_CCCC,
6621                0xCCCC_DDDD,
6622                0xDDDD_0000,
6623                0,
6624                0,
6625                0,
6626            ]
6627        );
6628    }
6629
6630    #[simd_test(enable = "vector")]
6631    fn test_vector_lcbb() {
6632        #[repr(align(64))]
6633        struct Align64<T>(T);
6634
6635        static ARRAY: Align64<[u8; 128]> = Align64([0; 128]);
6636
6637        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[64..].as_ptr()) }, 16);
6638        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[63..].as_ptr()) }, 1);
6639        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[56..].as_ptr()) }, 8);
6640        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[48..].as_ptr()) }, 16);
6641    }
6642
6643    test_vec_2! { test_vec_pack, vec_pack, i16x8, i16x8 -> i8x16,
6644        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6645        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6646        [0, 1, -1, 42, -1, 0, 48, -48, -1, 0, 57, -57, 0, 1, -1, 42]
6647    }
6648
6649    test_vec_2! { test_vec_packs, vec_packs, i16x8, i16x8 -> i8x16,
6650        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6651        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6652        [0, 1, -1, 42, 127, -128, 127, -128, 127, -128, 127, -128, 0, 1, -1, 42]
6653    }
6654
6655    test_vec_2! { test_vec_packsu_signed, vec_packsu, i16x8, i16x8 -> u8x16,
6656        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6657        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6658        [0, 1, 0, 42, 255, 0, 255, 0, 255, 0, 255, 0, 0, 1, 0, 42]
6659    }
6660
6661    test_vec_2! { test_vec_packsu_unsigned, vec_packsu, u16x8, u16x8 -> u8x16,
6662        [65535, 32768, 1234, 5678, 16, 8, 4, 2],
6663        [30000, 25000, 20000, 15000, 31, 63, 127, 255],
6664        [255, 255, 255, 255, 16, 8, 4, 2, 255, 255, 255, 255, 31, 63, 127, 255]
6665    }
6666
6667    test_vec_2! { test_vec_rl, vec_rl, u32x4,
6668        [0x12345678, 0x9ABCDEF0, 0x0F0F0F0F, 0x12345678],
6669        [4, 8, 12, 68],
6670        [0x23456781, 0xBCDEF09A, 0xF0F0F0F0, 0x23456781]
6671    }
6672
6673    test_vec_1! { test_vec_unpackh_i, vec_unpackh, i16x8 -> i32x4,
6674        [0x1234, -2, 0x0F0F, -32768, 0, 0, 0, 0],
6675        [0x1234, -2, 0x0F0F, -32768]
6676    }
6677
6678    test_vec_1! { test_vec_unpackh_u, vec_unpackh, u16x8 -> u32x4,
6679        [0x1234, 0xFFFF, 0x0F0F, 0x8000, 0, 0, 0, 0],
6680        [0x1234, 0xFFFF, 0x0F0F, 0x8000]
6681    }
6682
6683    test_vec_1! { test_vec_unpackl_i, vec_unpackl, i16x8 -> i32x4,
6684        [0, 0, 0, 0, 0x1234, -2, 0x0F0F, -32768],
6685        [0x1234, -2, 0x0F0F, -32768]
6686    }
6687
6688    test_vec_1! { test_vec_unpackl_u, vec_unpackl, u16x8 -> u32x4,
6689        [0, 0, 0, 0, 0x1234, 0xFFFF, 0x0F0F, 0x8000],
6690        [0x1234, 0xFFFF, 0x0F0F, 0x8000]
6691    }
6692
6693    test_vec_2! { test_vec_avg, vec_avg, u32x4,
6694        [2, 1, u32::MAX, 0],
6695        [4, 2, 2, 0],
6696        [3, (1u32 + 2).div_ceil(2), (u32::MAX as u64 + 2u64).div_ceil(2) as u32, 0]
6697    }
6698
6699    test_vec_2! { test_vec_checksum, vec_checksum, u32x4,
6700        [1, 2, 3, u32::MAX],
6701        [5, 6, 7, 8],
6702        [0, 12, 0, 0]
6703    }
6704
6705    test_vec_2! { test_vec_add_u128, vec_add_u128, u8x16,
6706        [0x01, 0x05, 0x0F, 0x1A, 0x2F, 0x3F, 0x50, 0x65,
6707                              0x7A, 0x8F, 0x9A, 0xAD, 0xB0, 0xC3, 0xD5, 0xE8],
6708        [0xF0, 0xEF, 0xC3, 0xB1, 0x92, 0x71, 0x5A, 0x43,
6709                              0x3B, 0x29, 0x13, 0x04, 0xD7, 0xA1, 0x8C, 0x76],
6710        [0xF1, 0xF4, 0xD2, 0xCB, 0xC1, 0xB0, 0xAA, 0xA8, 0xB5, 0xB8, 0xAD, 0xB2, 0x88, 0x65, 0x62, 0x5E]
6711    }
6712
6713    #[simd_test(enable = "vector")]
6714    fn test_vec_addc_u128() {
6715        unsafe {
6716            let a = u128::MAX;
6717            let b = 1u128;
6718
6719            let d: u128 = transmute(vec_addc_u128(transmute(a), transmute(b)));
6720            assert!(a.checked_add(b).is_none());
6721            assert_eq!(d, 1);
6722
6723            let a = 1u128;
6724            let b = 1u128;
6725
6726            let d: u128 = transmute(vec_addc_u128(transmute(a), transmute(b)));
6727            assert!(a.checked_add(b).is_some());
6728            assert_eq!(d, 0);
6729        }
6730    }
6731
6732    #[simd_test(enable = "vector")]
6733    fn test_vec_subc_u128() {
6734        unsafe {
6735            let a = 0u128;
6736            let b = 1u128;
6737
6738            let d: u128 = transmute(vec_subc_u128(transmute(a), transmute(b)));
6739            assert!(a.checked_sub(b).is_none());
6740            assert_eq!(d, 0);
6741
6742            let a = 1u128;
6743            let b = 1u128;
6744
6745            let d: u128 = transmute(vec_subc_u128(transmute(a), transmute(b)));
6746            assert!(a.checked_sub(b).is_some());
6747            assert_eq!(d, 1);
6748        }
6749    }
6750
6751    test_vec_2! { test_vec_mule_u, vec_mule, u16x8, u16x8 -> u32x4,
6752        [0xFFFF, 0, 2, 0, 2, 0, 1, 0],
6753        [0xFFFF, 0, 4, 0, 0xFFFF, 0, 2, 0],
6754        [0xFFFE_0001, 8, 0x0001_FFFE, 2]
6755    }
6756
6757    test_vec_2! { test_vec_mule_i, vec_mule, i16x8, i16x8 -> i32x4,
6758        [i16::MIN, 0, -2, 0, 2, 0, 1, 0],
6759        [i16::MIN, 0, 4, 0, i16::MAX, 0, 2, 0],
6760        [0x4000_0000, -8, 0xFFFE, 2]
6761    }
6762
6763    test_vec_2! { test_vec_mulo_u, vec_mulo, u16x8, u16x8 -> u32x4,
6764        [0, 0xFFFF, 0, 2, 0, 2, 0, 1],
6765        [0, 0xFFFF, 0, 4, 0, 0xFFFF, 0, 2],
6766        [0xFFFE_0001, 8, 0x0001_FFFE, 2]
6767    }
6768
6769    test_vec_2! { test_vec_mulo_i, vec_mulo, i16x8, i16x8 -> i32x4,
6770        [0, i16::MIN, 0, -2, 0, 2, 0, 1],
6771        [0, i16::MIN, 0, 4, 0, i16::MAX, 0, 2],
6772        [0x4000_0000, -8, 0xFFFE, 2]
6773    }
6774
6775    test_vec_2! { test_vec_mulh_u, vec_mulh, u32x4, u32x4 -> u32x4,
6776        [u32::MAX, 2, 2, 1],
6777        [u32::MAX, 4, u32::MAX, 2],
6778        [u32::MAX - 1, 0, 1, 0]
6779    }
6780
6781    test_vec_2! { test_vec_mulh_i, vec_mulh, i32x4, i32x4 -> i32x4,
6782        [i32::MIN, -2, 2, 1],
6783        [i32::MIN, 4, i32::MAX, 2],
6784        [0x4000_0000, -1, 0, 0]
6785    }
6786
6787    test_vec_2! { test_vec_gfmsum_1, vec_gfmsum, u16x8, u16x8 -> u32x4,
6788        [0x1234, 0x5678, 0x9ABC, 0xDEF0, 0x1357, 0x2468, 0xACE0, 0xBDF0],
6789        [0xFFFF, 0x0001, 0x8000, 0x7FFF, 0xAAAA, 0x5555, 0x1234, 0x5678],
6790        [0xE13A794, 0x68764A50, 0x94AA3E, 0x2C93F300]
6791    }
6792
6793    test_vec_2! { test_vec_gfmsum_2, vec_gfmsum, u16x8, u16x8 -> u32x4,
6794        [0x0000, 0xFFFF, 0xAAAA, 0x5555, 0x1234, 0x5678, 0x9ABC, 0xDEF0],
6795        [0xFFFF, 0x0000, 0x5555, 0xAAAA, 0x0001, 0x8000, 0x7FFF, 0x1357],
6796        [0, 0, 0x2B3C1234, 0x3781D244]
6797    }
6798
6799    #[simd_test(enable = "vector")]
6800    fn test_vec_gfmsum_128() {
6801        let a = vector_unsigned_long_long([1, 2]);
6802        let b = vector_unsigned_long_long([3, 4]);
6803
6804        let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) };
6805        assert_eq!(d, 11);
6806
6807        let a = vector_unsigned_long_long([0x0101010101010101, 0x0202020202020202]);
6808        let b = vector_unsigned_long_long([0x0404040404040404, 0x0505050505050505]);
6809
6810        let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) };
6811        assert_eq!(d, 0xE000E000E000E000E000E000E000E);
6812    }
6813
6814    #[simd_test(enable = "vector-enhancements-1")]
6815    fn test_vec_bperm_u128() {
6816        let a = vector_unsigned_char([65, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]);
6817        let b = vector_unsigned_char([
6818            0, 0, 0, 0, 1, 1, 1, 1, 128, 128, 128, 128, 255, 255, 255, 255,
6819        ]);
6820        let d = unsafe { vec_bperm_u128(a, b) };
6821        assert_eq!(d.as_array(), &[0xF00, 0]);
6822    }
6823
6824    #[simd_test(enable = "vector")]
6825    fn test_vec_sel() {
6826        let a = vector_signed_int([1, 2, 3, 4]);
6827        let b = vector_signed_int([5, 6, 7, 8]);
6828
6829        let e = vector_unsigned_int([9, 10, 11, 12]);
6830        let f = vector_unsigned_int([9, 9, 11, 11]);
6831
6832        let c: vector_bool_int = unsafe { simd_eq(e, f) };
6833        assert_eq!(c.as_array(), &[!0, 0, !0, 0]);
6834        let d: vector_signed_int = unsafe { vec_sel(a, b, c) };
6835        assert_eq!(d.as_array(), &[5, 2, 7, 4]);
6836    }
6837
6838    #[simd_test(enable = "vector")]
6839    fn test_vec_gather_element() {
6840        let a1: [u32; 10] = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19];
6841        let a2: [u32; 10] = [20, 21, 22, 23, 24, 25, 26, 27, 28, 29];
6842
6843        let v1 = vector_unsigned_int([1, 2, 3, 4]);
6844        let v2 = vector_unsigned_int([1, 2, 3, 4]);
6845
6846        let sizeof_int = core::mem::size_of::<u32>() as u32;
6847        let v3 = vector_unsigned_int([
6848            5 * sizeof_int,
6849            8 * sizeof_int,
6850            9 * sizeof_int,
6851            6 * sizeof_int,
6852        ]);
6853
6854        unsafe {
6855            let d1 = vec_gather_element::<_, 0>(v1, v3, a1.as_ptr());
6856            assert_eq!(d1.as_array(), &[15, 2, 3, 4]);
6857            let d2 = vec_gather_element::<_, 0>(v2, v3, a2.as_ptr());
6858            assert_eq!(d2.as_array(), &[25, 2, 3, 4]);
6859        }
6860    }
6861
6862    #[simd_test(enable = "vector")]
6863    fn test_vec_fp_test_data_class() {
6864        let mut cc = 42;
6865
6866        let v1 = vector_double([0.0, f64::NAN]);
6867        let v2 = vector_double([f64::INFINITY, 1.0]);
6868        let v3 = vector_double([1.0, 2.0]);
6869
6870        unsafe {
6871            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_ZERO>(v1, &mut cc);
6872            assert_eq!(cc, 1);
6873            assert_eq!(d.as_array(), &[!0, 0]);
6874
6875            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NAN>(v1, &mut cc);
6876            assert_eq!(cc, 1);
6877            assert_eq!(d.as_array(), &[0, !0]);
6878
6879            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY>(v2, &mut cc);
6880            assert_eq!(cc, 1);
6881            assert_eq!(d.as_array(), &[!0, 0]);
6882
6883            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY_N>(v2, &mut cc);
6884            assert_eq!(cc, 3);
6885            assert_eq!(d.as_array(), &[0, 0]);
6886
6887            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v2, &mut cc);
6888            assert_eq!(cc, 1);
6889            assert_eq!(d.as_array(), &[0, !0]);
6890
6891            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v3, &mut cc);
6892            assert_eq!(cc, 0);
6893            assert_eq!(d.as_array(), &[!0, !0]);
6894        }
6895    }
6896
6897    #[simd_test(enable = "vector")]
6898    fn test_vec_fp_any_all_nan_numeric() {
6899        unsafe {
6900            assert_eq!(
6901                vec_all_nan(vector_double([f64::NAN, f64::NAN])),
6902                i32::from(true)
6903            );
6904            assert_eq!(
6905                vec_all_nan(vector_double([f64::NAN, 1.0])),
6906                i32::from(false)
6907            );
6908            assert_eq!(vec_all_nan(vector_double([0.0, 1.0])), i32::from(false));
6909
6910            assert_eq!(
6911                vec_any_nan(vector_double([f64::NAN, f64::NAN])),
6912                i32::from(true)
6913            );
6914            assert_eq!(vec_any_nan(vector_double([f64::NAN, 1.0])), i32::from(true));
6915            assert_eq!(vec_any_nan(vector_double([0.0, 1.0])), i32::from(false));
6916
6917            assert_eq!(
6918                vec_all_numeric(vector_double([f64::NAN, f64::NAN])),
6919                i32::from(false)
6920            );
6921            assert_eq!(
6922                vec_all_numeric(vector_double([f64::NAN, 1.0])),
6923                i32::from(false)
6924            );
6925            assert_eq!(vec_all_numeric(vector_double([0.0, 1.0])), i32::from(true));
6926
6927            assert_eq!(
6928                vec_any_numeric(vector_double([f64::NAN, f64::NAN])),
6929                i32::from(false)
6930            );
6931            assert_eq!(
6932                vec_any_numeric(vector_double([f64::NAN, 1.0])),
6933                i32::from(true)
6934            );
6935            assert_eq!(vec_any_numeric(vector_double([0.0, 1.0])), i32::from(true));
6936
6937            // "numeric" means "not NaN". infinities are numeric
6938            assert_eq!(
6939                vec_all_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])),
6940                i32::from(true)
6941            );
6942            assert_eq!(
6943                vec_any_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])),
6944                i32::from(true)
6945            );
6946        }
6947    }
6948
6949    #[simd_test(enable = "vector")]
6950    fn test_vec_test_mask() {
6951        unsafe {
6952            let v = vector_unsigned_long_long([0xFF00FF00FF00FF00; 2]);
6953            let m = vector_unsigned_long_long([0x0000FF000000FF00; 2]);
6954            assert_eq!(vec_test_mask(v, m), 3);
6955
6956            let v = vector_unsigned_long_long([u64::MAX; 2]);
6957            let m = vector_unsigned_long_long([0; 2]);
6958            assert_eq!(vec_test_mask(v, m), 0);
6959
6960            let v = vector_unsigned_long_long([0; 2]);
6961            let m = vector_unsigned_long_long([u64::MAX; 2]);
6962            assert_eq!(vec_test_mask(v, m), 0);
6963
6964            let v = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]);
6965            let m = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]);
6966            assert_eq!(vec_test_mask(v, m), 3);
6967        }
6968    }
6969
6970    #[simd_test(enable = "vector-enhancements-2")]
6971    fn test_vec_search_string_cc() {
6972        unsafe {
6973            let b = vector_unsigned_char(*b"ABCD------------");
6974            let c = vector_unsigned_char([4; 16]);
6975
6976            let haystack = vector_unsigned_char(*b"__ABCD__________");
6977            let (result, d) = vec_search_string_cc(haystack, b, c);
6978            assert_eq!(result.as_array()[7], 2);
6979            assert_eq!(d, 2);
6980
6981            let haystack = vector_unsigned_char(*b"___ABCD_________");
6982            let (result, d) = vec_search_string_cc(haystack, b, c);
6983            assert_eq!(result.as_array()[7], 3);
6984            assert_eq!(d, 2);
6985
6986            let haystack = vector_unsigned_char(*b"________________");
6987            let (result, d) = vec_search_string_cc(haystack, b, c);
6988            assert_eq!(result.as_array()[7], 16);
6989            assert_eq!(d, 0);
6990
6991            let haystack = vector_unsigned_char(*b"______\0_________");
6992            let (result, d) = vec_search_string_cc(haystack, b, c);
6993            assert_eq!(result.as_array()[7], 16);
6994            assert_eq!(d, 0);
6995
6996            let haystack = vector_unsigned_char(*b"______\0__ABCD___");
6997            let (result, d) = vec_search_string_cc(haystack, b, c);
6998            assert_eq!(result.as_array()[7], 9);
6999            assert_eq!(d, 2);
7000        }
7001    }
7002
7003    #[simd_test(enable = "vector-enhancements-2")]
7004    fn test_vec_search_string_until_zero_cc() {
7005        unsafe {
7006            let b = vector_unsigned_char(*b"ABCD\0\0\0\0\0\0\0\0\0\0\0\0");
7007            let c = vector_unsigned_char([16; 16]);
7008
7009            let haystack = vector_unsigned_char(*b"__ABCD__________");
7010            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
7011            assert_eq!(result.as_array()[7], 2);
7012            assert_eq!(d, 2);
7013
7014            let haystack = vector_unsigned_char(*b"___ABCD_________");
7015            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
7016            assert_eq!(result.as_array()[7], 3);
7017            assert_eq!(d, 2);
7018
7019            let haystack = vector_unsigned_char(*b"________________");
7020            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
7021            assert_eq!(result.as_array()[7], 16);
7022            assert_eq!(d, 0);
7023
7024            let haystack = vector_unsigned_char(*b"______\0_________");
7025            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
7026            assert_eq!(result.as_array()[7], 16);
7027            assert_eq!(d, 1);
7028
7029            let haystack = vector_unsigned_char(*b"______\0__ABCD___");
7030            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
7031            assert_eq!(result.as_array()[7], 16);
7032            assert_eq!(d, 1);
7033        }
7034    }
7035
7036    #[simd_test(enable = "vector")]
7037    fn test_vec_doublee() {
7038        unsafe {
7039            let v = vector_float([1.0, 2.0, 3.0, 4.0]);
7040            assert_eq!(vec_doublee(v).as_array(), &[1.0, 3.0]);
7041
7042            let v = vector_float([f32::NAN, 2.0, f32::INFINITY, 4.0]);
7043            let d = vec_doublee(v);
7044            assert!(d.as_array()[0].is_nan());
7045            assert_eq!(d.as_array()[1], f64::INFINITY);
7046        }
7047    }
7048
7049    #[simd_test(enable = "vector")]
7050    fn test_vec_floate() {
7051        // NOTE: indices 1 and 3 can have an arbitrary value. With the C version
7052        // these are poison values, our version initializes the memory but its
7053        // value still should not be relied upon by application code.
7054        unsafe {
7055            let v = vector_double([1.0, 2.0]);
7056            let d = vec_floate(v);
7057            assert_eq!(d.as_array()[0], 1.0);
7058            assert_eq!(d.as_array()[2], 2.0);
7059
7060            let v = vector_double([f64::NAN, f64::INFINITY]);
7061            let d = vec_floate(v);
7062            assert!(d.as_array()[0].is_nan());
7063            assert_eq!(d.as_array()[2], f32::INFINITY);
7064
7065            let v = vector_double([f64::MIN, f64::MAX]);
7066            let d = vec_floate(v);
7067            assert_eq!(d.as_array()[0], f64::MIN as f32);
7068            assert_eq!(d.as_array()[2], f64::MAX as f32);
7069        }
7070    }
7071
7072    #[simd_test(enable = "vector")]
7073    fn test_vec_extend_s64() {
7074        unsafe {
7075            let v = vector_signed_char([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
7076            assert_eq!(vec_extend_s64(v).as_array(), &[7, 15]);
7077
7078            let v = vector_signed_short([0, 1, 2, 3, 4, 5, 6, 7]);
7079            assert_eq!(vec_extend_s64(v).as_array(), &[3, 7]);
7080
7081            let v = vector_signed_int([0, 1, 2, 3]);
7082            assert_eq!(vec_extend_s64(v).as_array(), &[1, 3]);
7083        }
7084    }
7085
7086    #[simd_test(enable = "vector")]
7087    fn test_vec_signed() {
7088        unsafe {
7089            let v = vector_float([1.0, 2.5, -2.5, -0.0]);
7090            assert_eq!(vec_signed(v).as_array(), &[1, 2, -2, 0]);
7091
7092            let v = vector_double([2.5, -2.5]);
7093            assert_eq!(vec_signed(v).as_array(), &[2, -2]);
7094        }
7095    }
7096
7097    #[simd_test(enable = "vector")]
7098    fn test_vec_unsigned() {
7099        // NOTE: converting a negative floating point value is UB!
7100        unsafe {
7101            let v = vector_float([1.0, 2.5, 3.5, 0.0]);
7102            assert_eq!(vec_unsigned(v).as_array(), &[1, 2, 3, 0]);
7103
7104            let v = vector_double([2.5, 3.5]);
7105            assert_eq!(vec_unsigned(v).as_array(), &[2, 3]);
7106        }
7107    }
7108
7109    #[simd_test(enable = "vector")]
7110    fn test_vec_cp_until_zero() {
7111        unsafe {
7112            let v = vector_signed_int([1, 2, 3, 4]);
7113            let d = vec_cp_until_zero(v);
7114            assert_eq!(d.as_array(), &[1, 2, 3, 4]);
7115
7116            let v = vector_signed_int([1, 2, 0, 4]);
7117            let d = vec_cp_until_zero(v);
7118            assert_eq!(d.as_array(), &[1, 2, 0, 0]);
7119        }
7120    }
7121
7122    #[simd_test(enable = "vector")]
7123    fn test_vec_cp_until_zero_cc() {
7124        unsafe {
7125            let v = vector_signed_int([1, 2, 3, 4]);
7126            let (d, cc) = vec_cp_until_zero_cc(v);
7127            assert_eq!(d.as_array(), &[1, 2, 3, 4]);
7128            assert_eq!(cc, 3);
7129
7130            let v = vector_signed_int([1, 2, 0, 4]);
7131            let (d, cc) = vec_cp_until_zero_cc(v);
7132            assert_eq!(d.as_array(), &[1, 2, 0, 0]);
7133            assert_eq!(cc, 0);
7134        }
7135    }
7136
7137    #[simd_test(enable = "vector-enhancements-1")]
7138    fn test_vec_msum_u128() {
7139        let a = vector_unsigned_long_long([1, 2]);
7140        let b = vector_unsigned_long_long([3, 4]);
7141
7142        unsafe {
7143            let c: vector_unsigned_char = transmute(100u128);
7144
7145            let d: u128 = transmute(vec_msum_u128::<0>(a, b, c));
7146            assert_eq!(d, (1 * 3) + (2 * 4) + 100);
7147
7148            let d: u128 = transmute(vec_msum_u128::<4>(a, b, c));
7149            assert_eq!(d, (1 * 3) + (2 * 4) * 2 + 100);
7150
7151            let d: u128 = transmute(vec_msum_u128::<8>(a, b, c));
7152            assert_eq!(d, (1 * 3) * 2 + (2 * 4) + 100);
7153
7154            let d: u128 = transmute(vec_msum_u128::<12>(a, b, c));
7155            assert_eq!(d, (1 * 3) * 2 + (2 * 4) * 2 + 100);
7156        }
7157    }
7158
7159    #[simd_test(enable = "vector")]
7160    fn test_vec_sld() {
7161        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7162        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7163
7164        unsafe {
7165            let d = vec_sld::<_, 4>(a, b);
7166            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]);
7167        }
7168    }
7169
7170    #[simd_test(enable = "vector")]
7171    fn test_vec_sldw() {
7172        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7173        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7174
7175        unsafe {
7176            let d = vec_sldw::<_, 1>(a, b);
7177            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]);
7178        }
7179    }
7180
7181    #[simd_test(enable = "vector-enhancements-2")]
7182    fn test_vec_sldb() {
7183        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7184        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7185
7186        unsafe {
7187            let d = vec_sldb::<_, 4>(a, b);
7188            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAB]);
7189        }
7190    }
7191
7192    #[simd_test(enable = "vector-enhancements-2")]
7193    fn test_vec_srdb() {
7194        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7195        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7196
7197        unsafe {
7198            let d = vec_srdb::<_, 4>(a, b);
7199            assert_eq!(d.as_array(), &[0xABBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7200        }
7201    }
7202
7203    const GT: u32 = 0x20000000;
7204    const LT: u32 = 0x40000000;
7205    const EQ: u32 = 0x80000000;
7206
7207    #[simd_test(enable = "vector")]
7208    fn test_vec_cmprg() {
7209        let a = vector_unsigned_int([11, 22, 33, 44]);
7210        let b = vector_unsigned_int([10, 20, 30, 40]);
7211
7212        let c = vector_unsigned_int([GT, LT, GT, LT]);
7213        let d = unsafe { vec_cmprg(a, b, c) };
7214        assert_eq!(d.as_array(), &[!0, 0, !0, 0]);
7215
7216        let c = vector_unsigned_int([GT, LT, 0, 0]);
7217        let d = unsafe { vec_cmprg(a, b, c) };
7218        assert_eq!(d.as_array(), &[!0, 0, 0, 0]);
7219
7220        let a = vector_unsigned_int([11, 22, 33, 30]);
7221        let b = vector_unsigned_int([10, 20, 30, 30]);
7222
7223        let c = vector_unsigned_int([GT, LT, EQ, EQ]);
7224        let d = unsafe { vec_cmprg(a, b, c) };
7225        assert_eq!(d.as_array(), &[!0, 0, 0, !0]);
7226    }
7227
7228    #[simd_test(enable = "vector")]
7229    fn test_vec_cmpnrg() {
7230        let a = vector_unsigned_int([11, 22, 33, 44]);
7231        let b = vector_unsigned_int([10, 20, 30, 40]);
7232
7233        let c = vector_unsigned_int([GT, LT, GT, LT]);
7234        let d = unsafe { vec_cmpnrg(a, b, c) };
7235        assert_eq!(d.as_array(), &[0, !0, 0, !0]);
7236
7237        let c = vector_unsigned_int([GT, LT, 0, 0]);
7238        let d = unsafe { vec_cmpnrg(a, b, c) };
7239        assert_eq!(d.as_array(), &[0, !0, !0, !0]);
7240
7241        let a = vector_unsigned_int([11, 22, 33, 30]);
7242        let b = vector_unsigned_int([10, 20, 30, 30]);
7243
7244        let c = vector_unsigned_int([GT, LT, EQ, EQ]);
7245        let d = unsafe { vec_cmpnrg(a, b, c) };
7246        assert_eq!(d.as_array(), &[0, !0, !0, 0]);
7247    }
7248
7249    #[simd_test(enable = "vector")]
7250    fn test_vec_cmprg_idx() {
7251        let a = vector_unsigned_int([1, 11, 22, 33]);
7252        let b = vector_unsigned_int([10, 20, 30, 40]);
7253
7254        let c = vector_unsigned_int([GT, LT, GT, LT]);
7255        let d = unsafe { vec_cmprg_idx(a, b, c) };
7256        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
7257    }
7258
7259    #[simd_test(enable = "vector")]
7260    fn test_vec_cmpnrg_idx() {
7261        let a = vector_unsigned_int([1, 11, 22, 33]);
7262        let b = vector_unsigned_int([10, 20, 30, 40]);
7263
7264        let c = vector_unsigned_int([GT, LT, GT, LT]);
7265        let d = unsafe { vec_cmpnrg_idx(a, b, c) };
7266        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
7267    }
7268
7269    #[simd_test(enable = "vector")]
7270    fn test_vec_cmprg_or_0_idx() {
7271        let a = vector_unsigned_int([1, 0, 22, 33]);
7272        let b = vector_unsigned_int([10, 20, 30, 40]);
7273
7274        let c = vector_unsigned_int([GT, LT, GT, LT]);
7275        let d = unsafe { vec_cmprg_or_0_idx(a, b, c) };
7276        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
7277    }
7278
7279    #[simd_test(enable = "vector")]
7280    fn test_vec_cmpnrg_or_0_idx() {
7281        let a = vector_unsigned_int([11, 33, 0, 22]);
7282        let b = vector_unsigned_int([10, 20, 30, 40]);
7283
7284        let c = vector_unsigned_int([GT, LT, GT, LT]);
7285        let d = unsafe { vec_cmpnrg_or_0_idx(a, b, c) };
7286        assert_eq!(d.as_array(), &[0, 8, 0, 0]);
7287    }
7288
7289    test_vec_2! { test_vec_cmpgt, vec_cmpgt, f32x4, f32x4 -> i32x4,
7290        [1.0, f32::NAN, f32::NAN, 3.14],
7291        [2.0, f32::NAN, 5.0, 2.0],
7292        [0, 0, 0, !0]
7293    }
7294
7295    test_vec_2! { test_vec_cmpge, vec_cmpge, f32x4, f32x4 -> i32x4,
7296        [1.0, f32::NAN, f32::NAN, 3.14],
7297        [1.0, f32::NAN, 5.0, 2.0],
7298        [!0, 0, 0, !0]
7299    }
7300
7301    test_vec_2! { test_vec_cmplt, vec_cmplt, f32x4, f32x4 -> i32x4,
7302        [1.0, f32::NAN, f32::NAN, 2.0],
7303        [2.0, f32::NAN, 5.0, 2.0],
7304        [!0, 0, 0, 0]
7305    }
7306
7307    test_vec_2! { test_vec_cmple, vec_cmple, f32x4, f32x4 -> i32x4,
7308        [1.0, f32::NAN, f32::NAN, 2.0],
7309        [1.0, f32::NAN, 5.0, 3.14],
7310        [!0, 0, 0, !0]
7311    }
7312
7313    test_vec_2! { test_vec_cmpeq, vec_cmpeq, f32x4, f32x4 -> i32x4,
7314        [1.0, f32::NAN, f32::NAN, 2.0],
7315        [1.0, f32::NAN, 5.0, 3.14],
7316        [!0, 0, 0, 0]
7317    }
7318
7319    test_vec_2! { test_vec_cmpne, vec_cmpne, f32x4, f32x4 -> i32x4,
7320        [1.0, f32::NAN, f32::NAN, 2.0],
7321        [1.0, f32::NAN, 5.0, 3.14],
7322        [0, !0, !0, !0]
7323    }
7324
7325    #[simd_test(enable = "vector")]
7326    fn test_vec_meadd() {
7327        let a = vector_unsigned_short([1, 0, 2, 0, 3, 0, 4, 0]);
7328        let b = vector_unsigned_short([5, 0, 6, 0, 7, 0, 8, 0]);
7329        let c = vector_unsigned_int([2, 2, 2, 2]);
7330
7331        let d = unsafe { vec_meadd(a, b, c) };
7332        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7333
7334        let a = vector_signed_short([1, 0, 2, 0, 3, 0, 4, 0]);
7335        let b = vector_signed_short([5, 0, 6, 0, 7, 0, 8, 0]);
7336        let c = vector_signed_int([2, -2, 2, -2]);
7337
7338        let d = unsafe { vec_meadd(a, b, c) };
7339        assert_eq!(d.as_array(), &[7, 10, 23, 30]);
7340    }
7341
7342    #[simd_test(enable = "vector")]
7343    fn test_vec_moadd() {
7344        let a = vector_unsigned_short([0, 1, 0, 2, 0, 3, 0, 4]);
7345        let b = vector_unsigned_short([0, 5, 0, 6, 0, 7, 0, 8]);
7346        let c = vector_unsigned_int([2, 2, 2, 2]);
7347
7348        let d = unsafe { vec_moadd(a, b, c) };
7349        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7350
7351        let a = vector_signed_short([0, 1, 0, 2, 0, 3, 0, 4]);
7352        let b = vector_signed_short([0, 5, 0, 6, 0, 7, 0, 8]);
7353        let c = vector_signed_int([2, -2, 2, -2]);
7354
7355        let d = unsafe { vec_moadd(a, b, c) };
7356        assert_eq!(d.as_array(), &[7, 10, 23, 30]);
7357    }
7358
7359    #[simd_test(enable = "vector")]
7360    fn test_vec_mhadd() {
7361        let a = vector_unsigned_int([1, 2, 3, 4]);
7362        let b = vector_unsigned_int([5, 6, 7, 8]);
7363        let c = vector_unsigned_int([u32::MAX; 4]);
7364
7365        let d = unsafe { vec_mhadd(a, b, c) };
7366        assert_eq!(d.as_array(), &[1, 1, 1, 1]);
7367
7368        let a = vector_signed_int([-1, -2, -3, -4]);
7369        let b = vector_signed_int([5, 6, 7, 8]);
7370        let c = vector_signed_int([i32::MIN; 4]);
7371
7372        let d = unsafe { vec_mhadd(a, b, c) };
7373        assert_eq!(d.as_array(), &[-1, -1, -1, -1]);
7374    }
7375
7376    #[simd_test(enable = "vector")]
7377    fn test_vec_mladd() {
7378        let a = vector_unsigned_int([1, 2, 3, 4]);
7379        let b = vector_unsigned_int([5, 6, 7, 8]);
7380        let c = vector_unsigned_int([2, 2, 2, 2]);
7381
7382        let d = unsafe { vec_mladd(a, b, c) };
7383        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7384
7385        let a = vector_signed_int([-1, -2, -3, -4]);
7386        let b = vector_signed_int([5, 6, 7, 8]);
7387        let c = vector_signed_int([2, 2, 2, 2]);
7388
7389        let d = unsafe { vec_mladd(a, b, c) };
7390        assert_eq!(d.as_array(), &[-3, -10, -19, -30]);
7391    }
7392
7393    #[simd_test(enable = "vector")]
7394    fn test_vec_extract() {
7395        let v = vector_unsigned_int([1, 2, 3, 4]);
7396
7397        assert_eq!(unsafe { vec_extract(v, 1) }, 2);
7398        assert_eq!(unsafe { vec_extract(v, 4 + 2) }, 3);
7399    }
7400
7401    #[simd_test(enable = "vector")]
7402    fn test_vec_insert() {
7403        let mut v = vector_unsigned_int([1, 2, 3, 4]);
7404
7405        v = unsafe { vec_insert(42, v, 1) };
7406        assert_eq!(v.as_array(), &[1, 42, 3, 4]);
7407
7408        v = unsafe { vec_insert(64, v, 6) };
7409        assert_eq!(v.as_array(), &[1, 42, 64, 4]);
7410    }
7411
7412    #[simd_test(enable = "vector")]
7413    fn test_vec_promote() {
7414        let v: vector_unsigned_int = unsafe { vec_promote(42, 1).assume_init() };
7415        assert_eq!(v.as_array(), &[0, 42, 0, 0]);
7416    }
7417
7418    #[simd_test(enable = "vector")]
7419    fn test_vec_insert_and_zero() {
7420        let v = unsafe { vec_insert_and_zero::<vector_unsigned_int>(&42u32) };
7421        assert_eq!(v.as_array(), vector_unsigned_int([0, 42, 0, 0]).as_array());
7422    }
7423}