core/stdarch/crates/core_arch/src/s390x/
vector.rs

1//! s390x vector intrinsics.
2//!
3//! For more info see the [Reference Summary] or the online [IBM docs].
4//!
5//! [Reference Summary]: https://www.ibm.com/support/pages/sites/default/files/2021-05/SA22-7871-10.pdf
6//! [IBM docs]: https://www.ibm.com/docs/en/zos/2.4.0?topic=support-vector-built-in-functions
7
8#![allow(non_camel_case_types)]
9
10use crate::{core_arch::simd::*, intrinsics::simd::*, mem::MaybeUninit, mem::transmute};
11
12#[cfg(test)]
13use stdarch_test::assert_instr;
14
15use super::macros::*;
16
17types! {
18    #![unstable(feature = "stdarch_s390x", issue = "135681")]
19
20    /// s390x-specific 128-bit wide vector of sixteen packed `i8`
21    pub struct vector_signed_char(16 x i8);
22    /// s390x-specific 128-bit wide vector of sixteen packed `u8`
23    pub struct vector_unsigned_char(16 x u8);
24    /// s390x-specific 128-bit wide vector mask of sixteen packed elements
25    pub struct vector_bool_char(16 x i8);
26
27    /// s390x-specific 128-bit wide vector of eight packed `i16`
28    pub struct vector_signed_short(8 x i16);
29    /// s390x-specific 128-bit wide vector of eight packed `u16`
30    pub struct vector_unsigned_short(8 x u16);
31    /// s390x-specific 128-bit wide vector mask of eight packed elements
32    pub struct vector_bool_short(8 x i16);
33
34    /// s390x-specific 128-bit wide vector of four packed `i32`
35    pub struct vector_signed_int(4 x i32);
36    /// s390x-specific 128-bit wide vector of four packed `u32`
37    pub struct vector_unsigned_int(4 x u32);
38    /// s390x-specific 128-bit wide vector mask of four packed elements
39    pub struct vector_bool_int(4 x i32);
40
41    /// s390x-specific 128-bit wide vector of two packed `i64`
42    pub struct vector_signed_long_long(2 x i64);
43    /// s390x-specific 128-bit wide vector of two packed `u64`
44    pub struct vector_unsigned_long_long(2 x u64);
45    /// s390x-specific 128-bit wide vector mask of two packed elements
46    pub struct vector_bool_long_long(2 x i64);
47
48    /// s390x-specific 128-bit wide vector of four packed `f32`
49    pub struct vector_float(4 x f32);
50    /// s390x-specific 128-bit wide vector of two packed `f64`
51    pub struct vector_double(2 x f64);
52}
53
54#[repr(C, packed)]
55struct PackedTuple<T, U> {
56    x: T,
57    y: U,
58}
59
60#[allow(improper_ctypes)]
61#[rustfmt::skip]
62unsafe extern "unadjusted" {
63    #[link_name = "llvm.smax.v16i8"] fn vmxb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
64    #[link_name = "llvm.smax.v8i16"] fn vmxh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
65    #[link_name = "llvm.smax.v4i32"] fn vmxf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
66    #[link_name = "llvm.smax.v2i64"] fn vmxg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
67
68    #[link_name = "llvm.umax.v16i8"] fn vmxlb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
69    #[link_name = "llvm.umax.v8i16"] fn vmxlh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
70    #[link_name = "llvm.umax.v4i32"] fn vmxlf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
71    #[link_name = "llvm.umax.v2i64"] fn vmxlg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
72
73    #[link_name = "llvm.smin.v16i8"] fn vmnb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
74    #[link_name = "llvm.smin.v8i16"] fn vmnh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
75    #[link_name = "llvm.smin.v4i32"] fn vmnf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
76    #[link_name = "llvm.smin.v2i64"] fn vmng(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
77
78    #[link_name = "llvm.umin.v16i8"] fn vmnlb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
79    #[link_name = "llvm.umin.v8i16"] fn vmnlh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
80    #[link_name = "llvm.umin.v4i32"] fn vmnlf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
81    #[link_name = "llvm.umin.v2i64"] fn vmnlg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
82
83    #[link_name = "llvm.nearbyint.v4f32"] fn nearbyint_v4f32(a: vector_float) -> vector_float;
84    #[link_name = "llvm.nearbyint.v2f64"] fn nearbyint_v2f64(a: vector_double) -> vector_double;
85
86    #[link_name = "llvm.roundeven.v4f32"] fn roundeven_v4f32(a: vector_float) -> vector_float;
87    #[link_name = "llvm.roundeven.v2f64"] fn roundeven_v2f64(a: vector_double) -> vector_double;
88
89    #[link_name = "llvm.s390.vsra"] fn vsra(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
90    #[link_name = "llvm.s390.vsrl"] fn vsrl(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
91    #[link_name = "llvm.s390.vsl"] fn vsl(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
92
93    #[link_name = "llvm.s390.vsrab"] fn vsrab(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
94    #[link_name = "llvm.s390.vsrlb"] fn vsrlb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
95    #[link_name = "llvm.s390.vslb"] fn vslb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
96
97    #[link_name = "llvm.s390.vsldb"] fn vsldb(a: i8x16, b: i8x16, c: u32) -> i8x16;
98    #[link_name = "llvm.s390.vsld"] fn vsld(a: i8x16, b: i8x16, c: u32) -> i8x16;
99    #[link_name = "llvm.s390.vsrd"] fn vsrd(a: i8x16, b: i8x16, c: u32) -> i8x16;
100
101    #[link_name = "llvm.s390.verimb"] fn verimb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char, d: i32) -> vector_signed_char;
102    #[link_name = "llvm.s390.verimh"] fn verimh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short, d: i32) -> vector_signed_short;
103    #[link_name = "llvm.s390.verimf"] fn verimf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int, d: i32) -> vector_signed_int;
104    #[link_name = "llvm.s390.verimg"] fn verimg(a: vector_signed_long_long, b: vector_signed_long_long, c: vector_signed_long_long, d: i32) -> vector_signed_long_long;
105
106    #[link_name = "llvm.s390.vperm"] fn vperm(a: vector_signed_char, b: vector_signed_char, c: vector_unsigned_char) -> vector_signed_char;
107
108    #[link_name = "llvm.s390.vsumb"] fn vsumb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_int;
109    #[link_name = "llvm.s390.vsumh"] fn vsumh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
110
111    #[link_name = "llvm.s390.vsumgh"] fn vsumgh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_long_long;
112    #[link_name = "llvm.s390.vsumgf"] fn vsumgf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
113
114    #[link_name = "llvm.s390.vsumqf"] fn vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128;
115    #[link_name = "llvm.s390.vsumqg"] fn vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128;
116
117    #[link_name = "llvm.s390.vscbiq"] fn vscbiq(a: u128, b: u128) -> u128;
118    #[link_name = "llvm.s390.vsbiq"] fn vsbiq(a: u128, b: u128, c: u128) -> u128;
119    #[link_name = "llvm.s390.vsbcbiq"] fn vsbcbiq(a: u128, b: u128, c: u128) -> u128;
120
121    #[link_name = "llvm.s390.vacq"] fn vacq(a: u128, b: u128, c: u128) -> u128;
122
123    #[link_name = "llvm.s390.vscbib"] fn vscbib(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
124    #[link_name = "llvm.s390.vscbih"] fn vscbih(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
125    #[link_name = "llvm.s390.vscbif"] fn vscbif(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
126    #[link_name = "llvm.s390.vscbig"] fn vscbig(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
127
128    #[link_name = "llvm.s390.vfaeb"] fn vfaeb(a: vector_signed_char, b: vector_signed_char, c: i32) -> vector_signed_char;
129    #[link_name = "llvm.s390.vfaeh"] fn vfaeh(a: vector_signed_short, b: vector_signed_short, c: i32) -> vector_signed_short;
130    #[link_name = "llvm.s390.vfaef"] fn vfaef(a: vector_signed_int, b: vector_signed_int, c: i32) -> vector_signed_int;
131
132    #[link_name = "llvm.s390.vfaezb"] fn vfaezb(a: vector_signed_char, b: vector_signed_char, c: i32) -> vector_signed_char;
133    #[link_name = "llvm.s390.vfaezh"] fn vfaezh(a: vector_signed_short, b: vector_signed_short, c: i32) -> vector_signed_short;
134    #[link_name = "llvm.s390.vfaezf"] fn vfaezf(a: vector_signed_int, b: vector_signed_int, c: i32) -> vector_signed_int;
135
136    #[link_name = "llvm.s390.vfaebs"] fn vfaebs(a: vector_signed_char, b: vector_signed_char, c: i32) -> PackedTuple<vector_signed_char, i32>;
137    #[link_name = "llvm.s390.vfaehs"] fn vfaehs(a: vector_signed_short, b: vector_signed_short, c: i32) -> PackedTuple<vector_signed_short, i32>;
138    #[link_name = "llvm.s390.vfaefs"] fn vfaefs(a: vector_signed_int, b: vector_signed_int, c: i32) -> PackedTuple<vector_signed_int, i32>;
139
140    #[link_name = "llvm.s390.vfaezbs"] fn vfaezbs(a: vector_signed_char, b: vector_signed_char, c: i32) -> PackedTuple<vector_signed_char, i32>;
141    #[link_name = "llvm.s390.vfaezhs"] fn vfaezhs(a: vector_signed_short, b: vector_signed_short, c: i32) -> PackedTuple<vector_signed_short, i32>;
142    #[link_name = "llvm.s390.vfaezfs"] fn vfaezfs(a: vector_signed_int, b: vector_signed_int, c: i32) -> PackedTuple<vector_signed_int, i32>;
143
144    #[link_name = "llvm.s390.vll"] fn vll(a: u32, b: *const u8) -> vector_signed_char;
145    #[link_name = "llvm.s390.vstl"] fn vstl(a: vector_signed_char, b: u32, c: *mut u8);
146
147    #[link_name = "llvm.s390.vlrl"] fn vlrl(a: u32, b: *const u8) -> vector_unsigned_char;
148    #[link_name = "llvm.s390.vstrl"] fn vstrl(a: vector_unsigned_char, b: u32, c: *mut u8);
149
150    #[link_name = "llvm.s390.lcbb"] fn lcbb(a: *const u8, b: u32) -> u32;
151    #[link_name = "llvm.s390.vlbb"] fn vlbb(a: *const u8, b: u32) -> MaybeUninit<vector_signed_char>;
152
153    #[link_name = "llvm.s390.vpksh"] fn vpksh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_char;
154    #[link_name = "llvm.s390.vpksf"] fn vpksf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_short;
155    #[link_name = "llvm.s390.vpksg"] fn vpksg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_int;
156
157    #[link_name = "llvm.s390.vpklsh"] fn vpklsh(a: vector_signed_short, b: vector_signed_short) -> vector_unsigned_char;
158    #[link_name = "llvm.s390.vpklsf"] fn vpklsf(a: vector_signed_int, b: vector_signed_int) -> vector_unsigned_short;
159    #[link_name = "llvm.s390.vpklsg"] fn vpklsg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_unsigned_int;
160
161    #[link_name = "llvm.s390.vpkshs"] fn vpkshs(a: vector_signed_short, b: vector_signed_short) -> PackedTuple<vector_signed_char, i32>;
162    #[link_name = "llvm.s390.vpksfs"] fn vpksfs(a: vector_signed_int, b: vector_signed_int) -> PackedTuple<vector_signed_short, i32>;
163    #[link_name = "llvm.s390.vpksgs"] fn vpksgs(a: vector_signed_long_long, b: vector_signed_long_long) -> PackedTuple<vector_signed_int, i32>;
164
165    #[link_name = "llvm.s390.vpklshs"] fn vpklshs(a: vector_unsigned_short, b: vector_unsigned_short) -> PackedTuple<vector_unsigned_char, i32>;
166    #[link_name = "llvm.s390.vpklsfs"] fn vpklsfs(a: vector_unsigned_int, b: vector_unsigned_int) -> PackedTuple<vector_unsigned_short, i32>;
167    #[link_name = "llvm.s390.vpklsgs"] fn vpklsgs(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> PackedTuple<vector_unsigned_int, i32>;
168
169    #[link_name = "llvm.s390.vuplb"] fn vuplb (a: vector_signed_char) -> vector_signed_short;
170    #[link_name = "llvm.s390.vuplhw"] fn vuplhw (a: vector_signed_short) -> vector_signed_int;
171    #[link_name = "llvm.s390.vuplf"] fn vuplf (a: vector_signed_int) -> vector_signed_long_long;
172    #[link_name = "llvm.s390.vupllb"] fn vupllb (a: vector_unsigned_char) -> vector_unsigned_short;
173    #[link_name = "llvm.s390.vupllh"] fn vupllh (a: vector_unsigned_short) -> vector_unsigned_int;
174    #[link_name = "llvm.s390.vupllf"] fn vupllf (a: vector_unsigned_int) -> vector_unsigned_long_long;
175
176    #[link_name = "llvm.s390.vavgb"] fn vavgb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
177    #[link_name = "llvm.s390.vavgh"] fn vavgh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
178    #[link_name = "llvm.s390.vavgf"] fn vavgf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
179    #[link_name = "llvm.s390.vavgg"] fn vavgg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
180
181    #[link_name = "llvm.s390.vavglb"] fn vavglb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
182    #[link_name = "llvm.s390.vavglh"] fn vavglh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
183    #[link_name = "llvm.s390.vavglf"] fn vavglf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
184    #[link_name = "llvm.s390.vavglg"] fn vavglg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
185
186    #[link_name = "llvm.s390.vcksm"] fn vcksm(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
187
188    #[link_name = "llvm.s390.vmeb"] fn vmeb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short;
189    #[link_name = "llvm.s390.vmeh"] fn vmeh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int;
190    #[link_name = "llvm.s390.vmef"] fn vmef(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long;
191
192    #[link_name = "llvm.s390.vmleb"] fn vmleb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
193    #[link_name = "llvm.s390.vmleh"] fn vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
194    #[link_name = "llvm.s390.vmlef"] fn vmlef(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
195
196    #[link_name = "llvm.s390.vmob"] fn vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short;
197    #[link_name = "llvm.s390.vmoh"] fn vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int;
198    #[link_name = "llvm.s390.vmof"] fn vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long;
199
200    #[link_name = "llvm.s390.vmlob"] fn vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
201    #[link_name = "llvm.s390.vmloh"] fn vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
202    #[link_name = "llvm.s390.vmlof"] fn vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
203
204    #[link_name = "llvm.s390.vmhb"] fn vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
205    #[link_name = "llvm.s390.vmhh"] fn vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
206    #[link_name = "llvm.s390.vmhf"] fn vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
207
208    #[link_name = "llvm.s390.vmlhb"] fn vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
209    #[link_name = "llvm.s390.vmlhh"] fn vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
210    #[link_name = "llvm.s390.vmlhf"] fn vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
211
212    #[link_name = "llvm.s390.vmaeb"] fn vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short;
213    #[link_name = "llvm.s390.vmaeh"] fn vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int;
214    #[link_name = "llvm.s390.vmaef"] fn vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long;
215
216    #[link_name = "llvm.s390.vmaleb"] fn vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
217    #[link_name = "llvm.s390.vmaleh"] fn vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
218    #[link_name = "llvm.s390.vmalef"] fn vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
219
220    #[link_name = "llvm.s390.vmaob"] fn vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short;
221    #[link_name = "llvm.s390.vmaoh"] fn vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int;
222    #[link_name = "llvm.s390.vmaof"] fn vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long;
223
224    #[link_name = "llvm.s390.vmalob"] fn vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
225    #[link_name = "llvm.s390.vmaloh"] fn vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
226    #[link_name = "llvm.s390.vmalof"] fn vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
227
228    #[link_name = "llvm.s390.vmahb"] fn vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char;
229    #[link_name = "llvm.s390.vmahh"] fn vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short;
230    #[link_name = "llvm.s390.vmahf"] fn vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int;
231
232    #[link_name = "llvm.s390.vmalhb"] fn vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char;
233    #[link_name = "llvm.s390.vmalhh"] fn vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short;
234    #[link_name = "llvm.s390.vmalhf"] fn vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int;
235
236    #[link_name = "llvm.s390.vmalb"] fn vmalb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char;
237    #[link_name = "llvm.s390.vmalh"] fn vmalh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short;
238    #[link_name = "llvm.s390.vmalf"] fn vmalf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int;
239
240    #[link_name = "llvm.s390.vmallb"] fn vmallb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char;
241    #[link_name = "llvm.s390.vmallh"] fn vmallh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short;
242    #[link_name = "llvm.s390.vmallf"] fn vmallf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int;
243
244    #[link_name = "llvm.s390.vgfmb"] fn vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
245    #[link_name = "llvm.s390.vgfmh"] fn vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
246    #[link_name = "llvm.s390.vgfmf"] fn vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
247    #[link_name = "llvm.s390.vgfmg"] fn vgfmg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128;
248
249    #[link_name = "llvm.s390.vgfmab"] fn vgfmab(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
250    #[link_name = "llvm.s390.vgfmah"] fn vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
251    #[link_name = "llvm.s390.vgfmaf"] fn vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
252    #[link_name = "llvm.s390.vgfmag"] fn vgfmag(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128) -> u128;
253
254    #[link_name = "llvm.s390.vbperm"] fn vbperm(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_long_long;
255
256    #[link_name = "llvm.s390.vftcisb"] fn vftcisb(a: vector_float, b: u32) -> PackedTuple<vector_bool_int, i32>;
257    #[link_name = "llvm.s390.vftcidb"] fn vftcidb(a: vector_double, b: u32) -> PackedTuple<vector_bool_long_long, i32>;
258
259    #[link_name = "llvm.s390.vtm"] fn vtm(a: i8x16, b: i8x16) -> i32;
260
261    #[link_name = "llvm.s390.vstrsb"] fn vstrsb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
262    #[link_name = "llvm.s390.vstrsh"] fn vstrsh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
263    #[link_name = "llvm.s390.vstrsf"] fn vstrsf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
264
265    #[link_name = "llvm.s390.vstrszb"] fn vstrszb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
266    #[link_name = "llvm.s390.vstrszh"] fn vstrszh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
267    #[link_name = "llvm.s390.vstrszf"] fn vstrszf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
268
269    #[link_name = "llvm.s390.vistrb"] fn vistrb(a: vector_unsigned_char) -> vector_unsigned_char;
270    #[link_name = "llvm.s390.vistrh"] fn vistrh(a: vector_unsigned_short) -> vector_unsigned_short;
271    #[link_name = "llvm.s390.vistrf"] fn vistrf(a: vector_unsigned_int) -> vector_unsigned_int;
272
273    #[link_name = "llvm.s390.vistrbs"] fn vistrbs(a: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
274    #[link_name = "llvm.s390.vistrhs"] fn vistrhs(a: vector_unsigned_short) -> PackedTuple<vector_unsigned_short, i32>;
275    #[link_name = "llvm.s390.vistrfs"] fn vistrfs(a: vector_unsigned_int) -> PackedTuple<vector_unsigned_int, i32>;
276
277    #[link_name = "llvm.s390.vmslg"] fn vmslg(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128, d: u32) -> u128;
278
279    #[link_name = "llvm.s390.vstrcb"] fn vstrcb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char;
280    #[link_name = "llvm.s390.vstrch"] fn vstrch(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short;
281    #[link_name = "llvm.s390.vstrcf"] fn vstrcf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int;
282
283    #[link_name = "llvm.s390.vstrcbs"] fn vstrcbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple<vector_bool_char, i32>;
284    #[link_name = "llvm.s390.vstrchs"] fn vstrchs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple<vector_bool_short, i32>;
285    #[link_name = "llvm.s390.vstrcfs"] fn vstrcfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple<vector_bool_int, i32>;
286
287    #[link_name = "llvm.s390.vstrczb"] fn vstrczb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char;
288    #[link_name = "llvm.s390.vstrczh"] fn vstrczh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short;
289    #[link_name = "llvm.s390.vstrczf"] fn vstrczf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int;
290
291    #[link_name = "llvm.s390.vstrczbs"] fn vstrczbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple<vector_bool_char, i32>;
292    #[link_name = "llvm.s390.vstrczhs"] fn vstrczhs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple<vector_bool_short, i32>;
293    #[link_name = "llvm.s390.vstrczfs"] fn vstrczfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple<vector_bool_int, i32>;
294
295    #[link_name = "llvm.s390.vfeeb"] fn vfeeb(a: i8x16, b: i8x16) -> i8x16;
296    #[link_name = "llvm.s390.vfeeh"] fn vfeeh(a: i16x8, b: i16x8) -> i16x8;
297    #[link_name = "llvm.s390.vfeef"] fn vfeef(a: i32x4, b: i32x4) -> i32x4;
298
299    #[link_name = "llvm.s390.vfeezb"] fn vfeezb(a: i8x16, b: i8x16) -> i8x16;
300    #[link_name = "llvm.s390.vfeezh"] fn vfeezh(a: i16x8, b: i16x8) -> i16x8;
301    #[link_name = "llvm.s390.vfeezf"] fn vfeezf(a: i32x4, b: i32x4) -> i32x4;
302
303    #[link_name = "llvm.s390.vfeebs"] fn vfeebs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
304    #[link_name = "llvm.s390.vfeehs"] fn vfeehs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
305    #[link_name = "llvm.s390.vfeefs"] fn vfeefs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
306
307    #[link_name = "llvm.s390.vfeezbs"] fn vfeezbs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
308    #[link_name = "llvm.s390.vfeezhs"] fn vfeezhs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
309    #[link_name = "llvm.s390.vfeezfs"] fn vfeezfs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
310
311    #[link_name = "llvm.s390.vfeneb"] fn vfeneb(a: i8x16, b: i8x16) -> i8x16;
312    #[link_name = "llvm.s390.vfeneh"] fn vfeneh(a: i16x8, b: i16x8) -> i16x8;
313    #[link_name = "llvm.s390.vfenef"] fn vfenef(a: i32x4, b: i32x4) -> i32x4;
314
315    #[link_name = "llvm.s390.vfenezb"] fn vfenezb(a: i8x16, b: i8x16) -> i8x16;
316    #[link_name = "llvm.s390.vfenezh"] fn vfenezh(a: i16x8, b: i16x8) -> i16x8;
317    #[link_name = "llvm.s390.vfenezf"] fn vfenezf(a: i32x4, b: i32x4) -> i32x4;
318
319    #[link_name = "llvm.s390.vfenebs"] fn vfenebs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
320    #[link_name = "llvm.s390.vfenehs"] fn vfenehs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
321    #[link_name = "llvm.s390.vfenefs"] fn vfenefs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
322
323    #[link_name = "llvm.s390.vfenezbs"] fn vfenezbs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
324    #[link_name = "llvm.s390.vfenezhs"] fn vfenezhs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
325    #[link_name = "llvm.s390.vfenezfs"] fn vfenezfs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
326}
327
328impl_from! { i8x16, u8x16,  i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 }
329
330impl_neg! { i8x16 : 0 }
331impl_neg! { i16x8 : 0 }
332impl_neg! { i32x4 : 0 }
333impl_neg! { i64x2 : 0 }
334impl_neg! { f32x4 : 0f32 }
335impl_neg! { f64x2 : 0f64 }
336
337#[repr(simd)]
338struct ShuffleMask<const N: usize>([u32; N]);
339
340impl<const N: usize> ShuffleMask<N> {
341    const fn reverse() -> Self {
342        let mut index = [0; N];
343        let mut i = 0;
344        while i < N {
345            index[i] = (N - i - 1) as u32;
346            i += 1;
347        }
348        ShuffleMask(index)
349    }
350
351    const fn merge_low() -> Self {
352        let mut mask = [0; N];
353        let mut i = N / 2;
354        let mut index = 0;
355        while index < N {
356            mask[index] = i as u32;
357            mask[index + 1] = (i + N) as u32;
358
359            i += 1;
360            index += 2;
361        }
362        ShuffleMask(mask)
363    }
364
365    const fn merge_high() -> Self {
366        let mut mask = [0; N];
367        let mut i = 0;
368        let mut index = 0;
369        while index < N {
370            mask[index] = i as u32;
371            mask[index + 1] = (i + N) as u32;
372
373            i += 1;
374            index += 2;
375        }
376        ShuffleMask(mask)
377    }
378
379    const fn pack() -> Self {
380        let mut mask = [0; N];
381        let mut i = 1;
382        let mut index = 0;
383        while index < N {
384            mask[index] = i as u32;
385
386            i += 2;
387            index += 1;
388        }
389        ShuffleMask(mask)
390    }
391
392    const fn unpack_low() -> Self {
393        let mut mask = [0; N];
394        let mut i = 0;
395        while i < N {
396            mask[i] = (N + i) as u32;
397            i += 1;
398        }
399        ShuffleMask(mask)
400    }
401
402    const fn unpack_high() -> Self {
403        let mut mask = [0; N];
404        let mut i = 0;
405        while i < N {
406            mask[i] = i as u32;
407            i += 1;
408        }
409        ShuffleMask(mask)
410    }
411}
412
413const fn genmask<const MASK: u16>() -> [u8; 16] {
414    let mut bits = MASK;
415    let mut elements = [0u8; 16];
416
417    let mut i = 0;
418    while i < 16 {
419        elements[i] = match bits & (1u16 << 15) {
420            0 => 0,
421            _ => 0xFF,
422        };
423
424        bits <<= 1;
425        i += 1;
426    }
427
428    elements
429}
430
431const fn genmasks(bit_width: u32, a: u8, b: u8) -> u64 {
432    let bit_width = bit_width as u8;
433    let a = a % bit_width;
434    let mut b = b % bit_width;
435    if a > b {
436        b = bit_width - 1;
437    }
438
439    // of course these indices start from the left
440    let a = (bit_width - 1) - a;
441    let b = (bit_width - 1) - b;
442
443    ((1u64.wrapping_shl(a as u32 + 1)) - 1) & !((1u64.wrapping_shl(b as u32)) - 1)
444}
445
446const fn validate_block_boundary(block_boundary: u16) -> u32 {
447    assert!(
448        block_boundary.is_power_of_two() && block_boundary >= 64 && block_boundary <= 4096,
449        "block boundary must be a constant power of 2 from 64 to 4096",
450    );
451
452    // so that 64 is encoded as 0, 128 as 1, ect.
453    block_boundary as u32 >> 7
454}
455
456enum FindImm {
457    Eq = 4,
458    Ne = 12,
459    EqIdx = 0,
460    NeIdx = 8,
461}
462
463#[macro_use]
464mod sealed {
465    use super::*;
466
467    #[unstable(feature = "stdarch_s390x", issue = "135681")]
468    pub trait VectorAdd<Other> {
469        type Result;
470        unsafe fn vec_add(self, other: Other) -> Self::Result;
471    }
472
473    macro_rules! impl_add {
474        ($name:ident, $a:ty, $instr:ident) => {
475            impl_add!($name, $a, $a, $a, $instr);
476        };
477        ($name:ident, $a:ty, $b:ty, $c:ty, $instr:ident) => {
478            #[inline]
479            #[target_feature(enable = "vector")]
480            #[cfg_attr(test, assert_instr($instr))]
481            pub unsafe fn $name(a: $a, b: $b) -> $c {
482                transmute(simd_add(transmute(a), b))
483            }
484
485            #[unstable(feature = "stdarch_s390x", issue = "135681")]
486            impl VectorAdd<$b> for $a {
487                type Result = $c;
488
489                #[inline]
490                #[target_feature(enable = "vector")]
491                unsafe fn vec_add(self, other: $b) -> Self::Result {
492                    $name(self, other)
493                }
494            }
495        };
496    }
497
498    #[rustfmt::skip]
499    mod impl_add {
500        use super::*;
501
502        impl_add!(va_sc, vector_signed_char, vab);
503        impl_add!(va_uc, vector_unsigned_char, vab);
504        impl_add!(va_sh, vector_signed_short, vah);
505        impl_add!(va_uh, vector_unsigned_short, vah);
506        impl_add!(va_sf, vector_signed_int, vaf);
507        impl_add!(va_uf, vector_unsigned_int, vaf);
508        impl_add!(va_sg, vector_signed_long_long, vag);
509        impl_add!(va_ug, vector_unsigned_long_long, vag);
510
511        impl_add!(va_sc_bc, vector_signed_char, vector_bool_char, vector_signed_char, vab);
512        impl_add!(va_uc_bc, vector_unsigned_char, vector_bool_char, vector_unsigned_char, vab);
513        impl_add!(va_sh_bh, vector_signed_short, vector_bool_short, vector_signed_short, vah);
514        impl_add!(va_uh_bh, vector_unsigned_short, vector_bool_short, vector_unsigned_short, vah);
515        impl_add!(va_sf_bf, vector_signed_int, vector_bool_int, vector_signed_int, vaf);
516        impl_add!(va_uf_bf, vector_unsigned_int, vector_bool_int, vector_unsigned_int, vaf);
517        impl_add!(va_sg_bg, vector_signed_long_long, vector_bool_long_long, vector_signed_long_long, vag);
518        impl_add!(va_ug_bg, vector_unsigned_long_long, vector_bool_long_long, vector_unsigned_long_long, vag);
519
520        impl_add!(va_bc_sc, vector_bool_char, vector_signed_char, vector_signed_char, vab);
521        impl_add!(va_bc_uc, vector_bool_char, vector_unsigned_char, vector_unsigned_char, vab);
522        impl_add!(va_bh_sh, vector_bool_short, vector_signed_short, vector_signed_short, vah);
523        impl_add!(va_bh_uh, vector_bool_short, vector_unsigned_short, vector_unsigned_short, vah);
524        impl_add!(va_bf_sf, vector_bool_int, vector_signed_int, vector_signed_int, vaf);
525        impl_add!(va_bf_uf, vector_bool_int, vector_unsigned_int, vector_unsigned_int, vaf);
526        impl_add!(va_bg_sg, vector_bool_long_long, vector_signed_long_long, vector_signed_long_long, vag);
527        impl_add!(va_bg_ug, vector_bool_long_long, vector_unsigned_long_long, vector_unsigned_long_long, vag);
528
529        impl_add!(va_double, vector_double, vfadb);
530
531        #[inline]
532        #[target_feature(enable = "vector")]
533        #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfasb))]
534        pub unsafe fn va_float(a: vector_float, b: vector_float) -> vector_float {
535            transmute(simd_add(a, b))
536        }
537
538        #[unstable(feature = "stdarch_s390x", issue = "135681")]
539        impl VectorAdd<Self> for vector_float {
540            type Result = Self;
541
542            #[inline]
543            #[target_feature(enable = "vector")]
544            unsafe fn vec_add(self, other: Self) -> Self::Result {
545                va_float(self, other)
546            }
547        }
548    }
549
550    #[unstable(feature = "stdarch_s390x", issue = "135681")]
551    pub trait VectorSub<Other> {
552        type Result;
553        unsafe fn vec_sub(self, other: Other) -> Self::Result;
554    }
555
556    macro_rules! impl_sub {
557        ($name:ident, $a:ty, $instr:ident) => {
558            impl_sub!($name, $a, $a, $a, $instr);
559        };
560        ($name:ident, $a:ty, $b:ty, $c:ty, $instr:ident) => {
561            #[inline]
562            #[target_feature(enable = "vector")]
563            #[cfg_attr(test, assert_instr($instr))]
564            pub unsafe fn $name(a: $a, b: $b) -> $c {
565                transmute(simd_sub(transmute(a), b))
566            }
567
568            #[unstable(feature = "stdarch_s390x", issue = "135681")]
569            impl VectorSub<$b> for $a {
570                type Result = $c;
571
572                #[inline]
573                #[target_feature(enable = "vector")]
574                unsafe fn vec_sub(self, other: $b) -> Self::Result {
575                    $name(self, other)
576                }
577            }
578        };
579    }
580
581    #[rustfmt::skip]
582    mod impl_sub {
583        use super::*;
584
585        impl_sub!(vs_sc, vector_signed_char, vsb);
586        impl_sub!(vs_uc, vector_unsigned_char, vsb);
587        impl_sub!(vs_sh, vector_signed_short, vsh);
588        impl_sub!(vs_uh, vector_unsigned_short, vsh);
589        impl_sub!(vs_sf, vector_signed_int, vsf);
590        impl_sub!(vs_uf, vector_unsigned_int, vsf);
591        impl_sub!(vs_sg, vector_signed_long_long, vsg);
592        impl_sub!(vs_ug, vector_unsigned_long_long, vsg);
593
594        impl_sub!(vs_sc_bc, vector_signed_char, vector_bool_char, vector_signed_char, vsb);
595        impl_sub!(vs_uc_bc, vector_unsigned_char, vector_bool_char, vector_unsigned_char, vsb);
596        impl_sub!(vs_sh_bh, vector_signed_short, vector_bool_short, vector_signed_short, vsh);
597        impl_sub!(vs_uh_bh, vector_unsigned_short, vector_bool_short, vector_unsigned_short, vsh);
598        impl_sub!(vs_sf_bf, vector_signed_int, vector_bool_int, vector_signed_int, vsf);
599        impl_sub!(vs_uf_bf, vector_unsigned_int, vector_bool_int, vector_unsigned_int, vsf);
600        impl_sub!(vs_sg_bg, vector_signed_long_long, vector_bool_long_long, vector_signed_long_long, vsg);
601        impl_sub!(vs_ug_bg, vector_unsigned_long_long, vector_bool_long_long, vector_unsigned_long_long, vsg);
602
603        impl_sub!(vs_bc_sc, vector_bool_char, vector_signed_char, vector_signed_char, vsb);
604        impl_sub!(vs_bc_uc, vector_bool_char, vector_unsigned_char, vector_unsigned_char, vsb);
605        impl_sub!(vs_bh_sh, vector_bool_short, vector_signed_short, vector_signed_short, vsh);
606        impl_sub!(vs_bh_uh, vector_bool_short, vector_unsigned_short, vector_unsigned_short, vsh);
607        impl_sub!(vs_bf_sf, vector_bool_int, vector_signed_int, vector_signed_int, vsf);
608        impl_sub!(vs_bf_uf, vector_bool_int, vector_unsigned_int, vector_unsigned_int, vsf);
609        impl_sub!(vs_bg_sg, vector_bool_long_long, vector_signed_long_long, vector_signed_long_long, vsg);
610        impl_sub!(vs_bg_ug, vector_bool_long_long, vector_unsigned_long_long, vector_unsigned_long_long, vsg);
611
612        impl_sub!(vs_double, vector_double, vfsdb);
613
614        #[inline]
615        #[target_feature(enable = "vector")]
616        #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfssb))]
617        pub unsafe fn vs_float(a: vector_float, b: vector_float) -> vector_float {
618            transmute(simd_sub(a, b))
619        }
620
621        #[unstable(feature = "stdarch_s390x", issue = "135681")]
622        impl VectorSub<Self> for vector_float {
623            type Result = Self;
624
625            #[inline]
626            #[target_feature(enable = "vector")]
627            unsafe fn vec_sub(self, other: Self) -> Self::Result {
628                vs_float(self, other)
629            }
630        }
631    }
632
633    #[unstable(feature = "stdarch_s390x", issue = "135681")]
634    pub trait VectorMul {
635        unsafe fn vec_mul(self, b: Self) -> Self;
636    }
637
638    macro_rules! impl_mul {
639        ($name:ident, $a:ty, std_simd) => {
640            #[unstable(feature = "stdarch_s390x", issue = "135681")]
641            impl VectorMul for $a {
642                #[inline]
643                #[target_feature(enable = "vector")]
644                unsafe fn vec_mul(self, other: Self) -> Self {
645                    transmute(simd_mul(transmute(self), other))
646                }
647            }
648        };
649        ($name:ident, $a:ty, $instr:ident) => {
650            #[inline]
651            #[target_feature(enable = "vector")]
652            #[cfg_attr(test, assert_instr($instr))]
653            pub unsafe fn $name(a: $a, b: $a) -> $a {
654                transmute(simd_mul(transmute(a), b))
655            }
656
657            #[unstable(feature = "stdarch_s390x", issue = "135681")]
658            impl VectorMul for $a {
659                #[inline]
660                #[target_feature(enable = "vector")]
661                unsafe fn vec_mul(self, other: Self) -> Self {
662                    $name(self, other)
663                }
664            }
665        };
666    }
667
668    #[rustfmt::skip]
669    mod impl_mul {
670        use super::*;
671
672        impl_mul!(vml_sc, vector_signed_char, vmlb);
673        impl_mul!(vml_uc, vector_unsigned_char, vmlb);
674        impl_mul!(vml_sh, vector_signed_short, vmlhw);
675        impl_mul!(vml_uh, vector_unsigned_short, vmlhw);
676        impl_mul!(vml_sf, vector_signed_int, vmlf);
677        impl_mul!(vml_uf, vector_unsigned_int, vmlf);
678        impl_mul!(vml_sg, vector_signed_long_long, std_simd);
679        impl_mul!(vml_ug, vector_unsigned_long_long, std_simd);
680
681        impl_mul!(vml_float, vector_float, std_simd);
682        impl_mul!(vml_double, vector_double, vfmdb);
683    }
684
685    #[unstable(feature = "stdarch_s390x", issue = "135681")]
686    pub trait VectorMax<Other> {
687        type Result;
688        unsafe fn vec_max(self, b: Other) -> Self::Result;
689    }
690
691    test_impl! { vec_vmxsb (a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [vmxb, vmxb] }
692    test_impl! { vec_vmxsh (a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [vmxh, vmxh] }
693    test_impl! { vec_vmxsf (a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [vmxf, vmxf] }
694    test_impl! { vec_vmxsg (a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [vmxg, vmxg] }
695
696    test_impl! { vec_vmxslb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vmxlb, vmxlb] }
697    test_impl! { vec_vmxslh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vmxlh, vmxlh] }
698    test_impl! { vec_vmxslf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vmxlf, vmxlf] }
699    test_impl! { vec_vmxslg (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vmxlg, vmxlg] }
700
701    impl_vec_trait! { [VectorMax vec_max] ~(vmxlb, vmxb, vmxlh, vmxh, vmxlf, vmxf, vmxlg, vmxg) }
702
703    test_impl! { vec_vfmaxsb (a: vector_float, b: vector_float) -> vector_float [simd_fmax, "vector-enhancements-1" vfmaxsb ] }
704    test_impl! { vec_vfmaxdb (a: vector_double, b: vector_double) -> vector_double [simd_fmax, "vector-enhancements-1" vfmaxdb] }
705
706    impl_vec_trait!([VectorMax vec_max] vec_vfmaxsb (vector_float, vector_float) -> vector_float);
707    impl_vec_trait!([VectorMax vec_max] vec_vfmaxdb (vector_double, vector_double) -> vector_double);
708
709    #[unstable(feature = "stdarch_s390x", issue = "135681")]
710    pub trait VectorMin<Other> {
711        type Result;
712        unsafe fn vec_min(self, b: Other) -> Self::Result;
713    }
714
715    test_impl! { vec_vmnsb (a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [vmnb, vmnb] }
716    test_impl! { vec_vmnsh (a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [vmnh, vmnh] }
717    test_impl! { vec_vmnsf (a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [vmnf, vmnf] }
718    test_impl! { vec_vmnsg (a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [vmng, vmng] }
719
720    test_impl! { vec_vmnslb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vmnlb, vmnlb] }
721    test_impl! { vec_vmnslh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vmnlh, vmnlh] }
722    test_impl! { vec_vmnslf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vmnlf, vmnlf] }
723    test_impl! { vec_vmnslg (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vmnlg, vmnlg] }
724
725    impl_vec_trait! { [VectorMin vec_min] ~(vmxlb, vmxb, vmxlh, vmxh, vmxlf, vmxf, vmxlg, vmxg) }
726
727    test_impl! { vec_vfminsb (a: vector_float, b: vector_float) -> vector_float [simd_fmin, "vector-enhancements-1" vfminsb]  }
728    test_impl! { vec_vfmindb (a: vector_double, b: vector_double) -> vector_double [simd_fmin, "vector-enhancements-1" vfmindb]  }
729
730    impl_vec_trait!([VectorMin vec_min] vec_vfminsb (vector_float, vector_float) -> vector_float);
731    impl_vec_trait!([VectorMin vec_min] vec_vfmindb (vector_double, vector_double) -> vector_double);
732
733    #[unstable(feature = "stdarch_s390x", issue = "135681")]
734    pub trait VectorAbs {
735        unsafe fn vec_abs(self) -> Self;
736    }
737
738    macro_rules! impl_abs {
739        ($name:ident, $ty:ident) => {
740            #[inline]
741            #[target_feature(enable = "vector")]
742            unsafe fn $name(v: s_t_l!($ty)) -> s_t_l!($ty) {
743                v.vec_max(-v)
744            }
745
746            impl_vec_trait! { [VectorAbs vec_abs] $name (s_t_l!($ty)) }
747        };
748    }
749
750    impl_abs! { vec_abs_i8, i8x16 }
751    impl_abs! { vec_abs_i16, i16x8 }
752    impl_abs! { vec_abs_i32, i32x4 }
753    impl_abs! { vec_abs_i64, i64x2 }
754
755    test_impl! { vec_abs_f32 (v: vector_float) -> vector_float [ simd_fabs, "vector-enhancements-1" vflpsb ] }
756    test_impl! { vec_abs_f64 (v: vector_double) -> vector_double [ simd_fabs, vflpdb ] }
757
758    impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f32 (vector_float) }
759    impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f64 (vector_double) }
760
761    #[unstable(feature = "stdarch_s390x", issue = "135681")]
762    pub trait VectorNabs {
763        unsafe fn vec_nabs(self) -> Self;
764    }
765
766    #[inline]
767    #[target_feature(enable = "vector")]
768    #[cfg_attr(
769        all(test, target_feature = "vector-enhancements-1"),
770        assert_instr(vflnsb)
771    )]
772    unsafe fn vec_nabs_f32(a: vector_float) -> vector_float {
773        simd_neg(simd_fabs(a))
774    }
775
776    #[inline]
777    #[target_feature(enable = "vector")]
778    #[cfg_attr(test, assert_instr(vflndb))]
779    unsafe fn vec_nabs_f64(a: vector_double) -> vector_double {
780        simd_neg(simd_fabs(a))
781    }
782
783    impl_vec_trait! { [VectorNabs vec_nabs] vec_nabs_f32 (vector_float) }
784    impl_vec_trait! { [VectorNabs vec_nabs] vec_nabs_f64 (vector_double) }
785
786    #[unstable(feature = "stdarch_s390x", issue = "135681")]
787    pub trait VectorNmsub {
788        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self;
789    }
790
791    #[inline]
792    #[target_feature(enable = "vector")]
793    #[cfg_attr(
794        all(test, target_feature = "vector-enhancements-2"),
795        assert_instr(vfnmssb)
796    )]
797    unsafe fn vec_nmsub_f32(a: vector_float, b: vector_float, c: vector_float) -> vector_float {
798        simd_neg(simd_fma(a, b, simd_neg(c)))
799    }
800
801    #[unstable(feature = "stdarch_s390x", issue = "135681")]
802    impl VectorNmsub for vector_float {
803        #[target_feature(enable = "vector")]
804        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self {
805            vec_nmsub_f32(self, b, c)
806        }
807    }
808
809    #[inline]
810    #[target_feature(enable = "vector")]
811    #[cfg_attr(
812        all(test, target_feature = "vector-enhancements-2"),
813        assert_instr(vfnmsdb)
814    )]
815    unsafe fn vec_nmsub_f64(a: vector_double, b: vector_double, c: vector_double) -> vector_double {
816        simd_neg(simd_fma(a, b, simd_neg(c)))
817    }
818
819    #[unstable(feature = "stdarch_s390x", issue = "135681")]
820    impl VectorNmsub for vector_double {
821        #[target_feature(enable = "vector")]
822        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self {
823            vec_nmsub_f64(self, b, c)
824        }
825    }
826
827    #[unstable(feature = "stdarch_s390x", issue = "135681")]
828    pub trait VectorNmadd {
829        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self;
830    }
831
832    #[inline]
833    #[target_feature(enable = "vector")]
834    #[cfg_attr(
835        all(test, target_feature = "vector-enhancements-2"),
836        assert_instr(vfnmasb)
837    )]
838    unsafe fn vec_nmadd_f32(a: vector_float, b: vector_float, c: vector_float) -> vector_float {
839        simd_neg(simd_fma(a, b, c))
840    }
841
842    #[unstable(feature = "stdarch_s390x", issue = "135681")]
843    impl VectorNmadd for vector_float {
844        #[target_feature(enable = "vector")]
845        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self {
846            vec_nmadd_f32(self, b, c)
847        }
848    }
849
850    #[inline]
851    #[target_feature(enable = "vector")]
852    #[cfg_attr(
853        all(test, target_feature = "vector-enhancements-2"),
854        assert_instr(vfnmadb)
855    )]
856    unsafe fn vec_nmadd_f64(a: vector_double, b: vector_double, c: vector_double) -> vector_double {
857        simd_neg(simd_fma(a, b, c))
858    }
859
860    #[unstable(feature = "stdarch_s390x", issue = "135681")]
861    impl VectorNmadd for vector_double {
862        #[target_feature(enable = "vector")]
863        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self {
864            vec_nmadd_f64(self, b, c)
865        }
866    }
867
868    #[unstable(feature = "stdarch_s390x", issue = "135681")]
869    pub trait VectorSplat {
870        unsafe fn vec_splat<const IMM: u32>(self) -> Self;
871    }
872
873    #[inline]
874    #[target_feature(enable = "vector")]
875    #[cfg_attr(test, assert_instr(vrepb, IMM2 = 1))]
876    unsafe fn vrepb<const IMM2: u32>(a: vector_signed_char) -> vector_signed_char {
877        static_assert_uimm_bits!(IMM2, 4);
878        simd_shuffle(a, a, const { u32x16::from_array([IMM2; 16]) })
879    }
880
881    #[inline]
882    #[target_feature(enable = "vector")]
883    #[cfg_attr(test, assert_instr(vreph, IMM2 = 1))]
884    unsafe fn vreph<const IMM2: u32>(a: vector_signed_short) -> vector_signed_short {
885        static_assert_uimm_bits!(IMM2, 3);
886        simd_shuffle(a, a, const { u32x8::from_array([IMM2; 8]) })
887    }
888
889    #[inline]
890    #[target_feature(enable = "vector")]
891    #[cfg_attr(test, assert_instr(vrepf, IMM2 = 1))]
892    unsafe fn vrepf<const IMM2: u32>(a: vector_signed_int) -> vector_signed_int {
893        static_assert_uimm_bits!(IMM2, 2);
894        simd_shuffle(a, a, const { u32x4::from_array([IMM2; 4]) })
895    }
896
897    #[inline]
898    #[target_feature(enable = "vector")]
899    #[cfg_attr(test, assert_instr(vrepg, IMM2 = 1))]
900    unsafe fn vrepg<const IMM2: u32>(a: vector_signed_long_long) -> vector_signed_long_long {
901        static_assert_uimm_bits!(IMM2, 1);
902        simd_shuffle(a, a, const { u32x2::from_array([IMM2; 2]) })
903    }
904
905    macro_rules! impl_vec_splat {
906        ($ty:ty, $fun:ident) => {
907            #[unstable(feature = "stdarch_s390x", issue = "135681")]
908            impl VectorSplat for $ty {
909                #[inline]
910                #[target_feature(enable = "vector")]
911                unsafe fn vec_splat<const IMM: u32>(self) -> Self {
912                    transmute($fun::<IMM>(transmute(self)))
913                }
914            }
915        };
916    }
917
918    impl_vec_splat! { vector_signed_char, vrepb }
919    impl_vec_splat! { vector_unsigned_char, vrepb }
920    impl_vec_splat! { vector_bool_char, vrepb }
921    impl_vec_splat! { vector_signed_short, vreph }
922    impl_vec_splat! { vector_unsigned_short, vreph }
923    impl_vec_splat! { vector_bool_short, vreph }
924    impl_vec_splat! { vector_signed_int, vrepf }
925    impl_vec_splat! { vector_unsigned_int, vrepf }
926    impl_vec_splat! { vector_bool_int, vrepf }
927    impl_vec_splat! { vector_signed_long_long, vrepg }
928    impl_vec_splat! { vector_unsigned_long_long, vrepg }
929    impl_vec_splat! { vector_bool_long_long, vrepg }
930
931    impl_vec_splat! { vector_float, vrepf }
932    impl_vec_splat! { vector_double, vrepg }
933
934    #[unstable(feature = "stdarch_s390x", issue = "135681")]
935    pub trait VectorSplats<Output> {
936        unsafe fn vec_splats(self) -> Output;
937    }
938
939    macro_rules! impl_vec_splats {
940        ($(($fn:ident ($ty:ty, $shortty:tt) $instr:ident)),*) => {
941            $(
942                #[inline]
943                #[target_feature(enable = "vector")]
944                #[cfg_attr(test, assert_instr($instr))]
945                pub unsafe fn $fn(v: $ty) -> s_t_l!($shortty) {
946                    transmute($shortty::splat(v))
947                }
948
949                #[unstable(feature = "stdarch_s390x", issue = "135681")]
950                impl VectorSplats<s_t_l!($shortty)> for $ty {
951                    #[inline]
952                    #[target_feature(enable = "vector")]
953                    unsafe fn vec_splats(self) -> s_t_l!($shortty) {
954                        $fn (self)
955                    }
956                }
957            )*
958        }
959    }
960
961    impl_vec_splats! {
962        (vec_splats_u8 (u8, u8x16) vrepb),
963        (vec_splats_i8 (i8, i8x16) vrepb),
964        (vec_splats_u16 (u16, u16x8) vreph),
965        (vec_splats_i16 (i16, i16x8) vreph),
966        (vec_splats_u32 (u32, u32x4) vrepf),
967        (vec_splats_i32 (i32, i32x4) vrepf),
968        (vec_splats_u64 (u64, u64x2) vlvgp),
969        (vec_splats_i64 (i64, i64x2) vlvgp),
970        (vec_splats_f32 (f32, f32x4) vrepf),
971        (vec_splats_f64 (f64, f64x2) vrepg)
972    }
973
974    macro_rules! impl_bool_vec_splats {
975        ($(($ty:ty, $shortty:tt, $boolty:ty)),*) => {
976            $(
977                #[unstable(feature = "stdarch_s390x", issue = "135681")]
978                impl VectorSplats<$boolty> for $ty {
979                    #[inline]
980                    #[target_feature(enable = "vector")]
981                    unsafe fn vec_splats(self) -> $boolty {
982                        transmute($shortty::splat(self))
983                    }
984                }
985            )*
986        }
987    }
988
989    impl_bool_vec_splats! {
990        (u8, u8x16, vector_bool_char),
991        (i8, i8x16, vector_bool_char),
992        (u16, u16x8, vector_bool_short),
993        (i16, i16x8, vector_bool_short),
994        (u32, u32x4, vector_bool_int),
995        (i32, i32x4, vector_bool_int),
996        (u64, u64x2, vector_bool_long_long),
997        (i64, i64x2, vector_bool_long_long)
998    }
999
1000    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1001    pub trait CountBits {
1002        type Result;
1003
1004        unsafe fn vec_cntlz(self) -> Self::Result;
1005        unsafe fn vec_cnttz(self) -> Self::Result;
1006        unsafe fn vec_popcnt(self) -> Self::Result;
1007    }
1008
1009    macro_rules! impl_count_bits {
1010        ($ty:tt) => {
1011            #[unstable(feature = "stdarch_s390x", issue = "135681")]
1012            impl CountBits for $ty {
1013                type Result = t_u!($ty);
1014
1015                #[inline]
1016                #[target_feature(enable = "vector")]
1017                unsafe fn vec_cntlz(self) -> Self::Result {
1018                    transmute(simd_ctlz(self))
1019                }
1020
1021                #[inline]
1022                #[target_feature(enable = "vector")]
1023                unsafe fn vec_cnttz(self) -> Self::Result {
1024                    transmute(simd_cttz(self))
1025                }
1026
1027                #[inline]
1028                #[target_feature(enable = "vector")]
1029                unsafe fn vec_popcnt(self) -> Self::Result {
1030                    transmute(simd_ctpop(self))
1031                }
1032            }
1033        };
1034    }
1035
1036    impl_count_bits!(vector_signed_char);
1037    impl_count_bits!(vector_unsigned_char);
1038    impl_count_bits!(vector_signed_short);
1039    impl_count_bits!(vector_unsigned_short);
1040    impl_count_bits!(vector_signed_int);
1041    impl_count_bits!(vector_unsigned_int);
1042    impl_count_bits!(vector_signed_long_long);
1043    impl_count_bits!(vector_unsigned_long_long);
1044
1045    test_impl! { vec_clzb_signed +(a: vector_signed_char) -> vector_unsigned_char [simd_ctlz, vclzb] }
1046    test_impl! { vec_clzh_signed +(a: vector_signed_short) -> vector_unsigned_short [simd_ctlz, vclzh] }
1047    test_impl! { vec_clzf_signed +(a: vector_signed_int) -> vector_unsigned_int [simd_ctlz, vclzf] }
1048    test_impl! { vec_clzg_signed +(a: vector_signed_long_long) -> vector_unsigned_long_long [simd_ctlz, vclzg] }
1049
1050    test_impl! { vec_clzb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctlz, vclzb] }
1051    test_impl! { vec_clzh_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_ctlz, vclzh] }
1052    test_impl! { vec_clzf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_ctlz, vclzf] }
1053    test_impl! { vec_clzg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_ctlz, vclzg] }
1054
1055    test_impl! { vec_ctzb_signed +(a: vector_signed_char) -> vector_unsigned_char [simd_cttz, vctzb] }
1056    test_impl! { vec_ctzh_signed +(a: vector_signed_short) -> vector_unsigned_short [simd_cttz, vctzh] }
1057    test_impl! { vec_ctzf_signed +(a: vector_signed_int) -> vector_unsigned_int [simd_cttz, vctzf] }
1058    test_impl! { vec_ctzg_signed +(a: vector_signed_long_long) -> vector_unsigned_long_long [simd_cttz, vctzg] }
1059
1060    test_impl! { vec_ctzb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_cttz, vctzb] }
1061    test_impl! { vec_ctzh_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_cttz, vctzh] }
1062    test_impl! { vec_ctzf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_cttz, vctzf] }
1063    test_impl! { vec_ctzg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_cttz, vctzg] }
1064
1065    test_impl! { vec_vpopctb_signed +(a: vector_signed_char) -> vector_signed_char [simd_ctpop, vpopctb] }
1066    test_impl! { vec_vpopcth_signed +(a: vector_signed_short) -> vector_signed_short [simd_ctpop, "vector-enhancements-1" vpopcth] }
1067    test_impl! { vec_vpopctf_signed +(a: vector_signed_int) -> vector_signed_int [simd_ctpop, "vector-enhancements-1" vpopctf] }
1068    test_impl! { vec_vpopctg_signed +(a: vector_signed_long_long) -> vector_signed_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] }
1069
1070    test_impl! { vec_vpopctb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctpop, vpopctb] }
1071    test_impl! { vec_vpopcth_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_ctpop, "vector-enhancements-1" vpopcth] }
1072    test_impl! { vec_vpopctf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_ctpop, "vector-enhancements-1" vpopctf] }
1073    test_impl! { vec_vpopctg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] }
1074
1075    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1076    pub trait VectorAnd<Other> {
1077        type Result;
1078        unsafe fn vec_and(self, b: Other) -> Self::Result;
1079    }
1080
1081    impl_vec_trait! { [VectorAnd vec_and] ~(simd_and) }
1082
1083    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1084    pub trait VectorOr<Other> {
1085        type Result;
1086        unsafe fn vec_or(self, b: Other) -> Self::Result;
1087    }
1088
1089    impl_vec_trait! { [VectorOr vec_or] ~(simd_or) }
1090
1091    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1092    pub trait VectorXor<Other> {
1093        type Result;
1094        unsafe fn vec_xor(self, b: Other) -> Self::Result;
1095    }
1096
1097    impl_vec_trait! { [VectorXor vec_xor] ~(simd_xor) }
1098
1099    #[inline]
1100    #[target_feature(enable = "vector")]
1101    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vno))]
1102    unsafe fn nor(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1103        let a: u8x16 = transmute(a);
1104        let b: u8x16 = transmute(b);
1105        transmute(simd_xor(simd_or(a, b), u8x16::splat(0xff)))
1106    }
1107
1108    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1109    pub trait VectorNor<Other> {
1110        type Result;
1111        unsafe fn vec_nor(self, b: Other) -> Self::Result;
1112    }
1113
1114    impl_vec_trait! { [VectorNor vec_nor]+ 2c (nor) }
1115
1116    #[inline]
1117    #[target_feature(enable = "vector")]
1118    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnn))]
1119    unsafe fn nand(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1120        let a: u8x16 = transmute(a);
1121        let b: u8x16 = transmute(b);
1122        transmute(simd_xor(simd_and(a, b), u8x16::splat(0xff)))
1123    }
1124
1125    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1126    pub trait VectorNand<Other> {
1127        type Result;
1128        unsafe fn vec_nand(self, b: Other) -> Self::Result;
1129    }
1130
1131    impl_vec_trait! { [VectorNand vec_nand]+ 2c (nand) }
1132
1133    #[inline]
1134    #[target_feature(enable = "vector")]
1135    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnx))]
1136    unsafe fn eqv(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1137        let a: u8x16 = transmute(a);
1138        let b: u8x16 = transmute(b);
1139        transmute(simd_xor(simd_xor(a, b), u8x16::splat(0xff)))
1140    }
1141
1142    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1143    pub trait VectorEqv<Other> {
1144        type Result;
1145        unsafe fn vec_eqv(self, b: Other) -> Self::Result;
1146    }
1147
1148    impl_vec_trait! { [VectorEqv vec_eqv]+ 2c (eqv) }
1149
1150    #[inline]
1151    #[target_feature(enable = "vector")]
1152    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnc))]
1153    unsafe fn andc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1154        let a = transmute(a);
1155        let b = transmute(b);
1156        transmute(simd_and(simd_xor(u8x16::splat(0xff), b), a))
1157    }
1158
1159    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1160    pub trait VectorAndc<Other> {
1161        type Result;
1162        unsafe fn vec_andc(self, b: Other) -> Self::Result;
1163    }
1164
1165    impl_vec_trait! { [VectorAndc vec_andc]+ 2c (andc) }
1166
1167    #[inline]
1168    #[target_feature(enable = "vector")]
1169    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(voc))]
1170    unsafe fn orc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1171        let a = transmute(a);
1172        let b = transmute(b);
1173        transmute(simd_or(simd_xor(u8x16::splat(0xff), b), a))
1174    }
1175
1176    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1177    pub trait VectorOrc<Other> {
1178        type Result;
1179        unsafe fn vec_orc(self, b: Other) -> Self::Result;
1180    }
1181
1182    impl_vec_trait! { [VectorOrc vec_orc]+ 2c (orc) }
1183
1184    // Z vector intrinsic      C23 math.h  LLVM IR         ISO/IEC 60559 operation        inexact  vfidb parameters
1185    //
1186    // vec_rint                rint        llvm.rint       roundToIntegralExact           yes      0, 0
1187    // vec_roundc              nearbyint   llvm.nearbyint  n/a                            no       4, 0
1188    // vec_floor / vec_roundm  floor       llvm.floor      roundToIntegralTowardNegative  no       4, 7
1189    // vec_ceil / vec_roundp   ceil        llvm.ceil       roundToIntegralTowardPositive  no       4, 6
1190    // vec_trunc / vec_roundz  trunc       llvm.trunc      roundToIntegralTowardZero      no       4, 5
1191    // vec_round               roundeven   llvm.roundeven  roundToIntegralTiesToEven      no       4, 4
1192    // n/a                     round       llvm.round      roundToIntegralTiesAway        no       4, 1
1193
1194    // `simd_round_ties_even` is implemented as `llvm.rint`.
1195    test_impl! { vec_rint_f32 (a: vector_float) -> vector_float [simd_round_ties_even, "vector-enhancements-1" vfisb] }
1196    test_impl! { vec_rint_f64 (a: vector_double) -> vector_double [simd_round_ties_even, vfidb] }
1197
1198    test_impl! { vec_roundc_f32 (a: vector_float) -> vector_float [nearbyint_v4f32,  "vector-enhancements-1" vfisb] }
1199    test_impl! { vec_roundc_f64 (a: vector_double) -> vector_double [nearbyint_v2f64, vfidb] }
1200
1201    // FIXME(llvm) llvm trunk already lowers roundeven to vfidb, but rust does not use it yet
1202    // use https://godbolt.org/z/cWq95fexe to check, and enable the instruction test when it works
1203    test_impl! { vec_round_f32 (a: vector_float) -> vector_float [roundeven_v4f32, _] }
1204    test_impl! { vec_round_f64 (a: vector_double) -> vector_double [roundeven_v2f64, _] }
1205
1206    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1207    pub trait VectorRoundc {
1208        unsafe fn vec_roundc(self) -> Self;
1209    }
1210
1211    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1212    pub trait VectorRound {
1213        unsafe fn vec_round(self) -> Self;
1214    }
1215
1216    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1217    pub trait VectorRint {
1218        unsafe fn vec_rint(self) -> Self;
1219    }
1220
1221    impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f32 (vector_float) }
1222    impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f64 (vector_double) }
1223
1224    impl_vec_trait! { [VectorRound vec_round] vec_round_f32 (vector_float) }
1225    impl_vec_trait! { [VectorRound vec_round] vec_round_f64 (vector_double) }
1226
1227    impl_vec_trait! { [VectorRint vec_rint] simd_round_ties_even (vector_float) }
1228    impl_vec_trait! { [VectorRint vec_rint] simd_round_ties_even (vector_double) }
1229
1230    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1231    pub trait VectorTrunc {
1232        // same as vec_roundz
1233        unsafe fn vec_trunc(self) -> Self;
1234    }
1235
1236    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1237    pub trait VectorCeil {
1238        // same as vec_roundp
1239        unsafe fn vec_ceil(self) -> Self;
1240    }
1241
1242    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1243    pub trait VectorFloor {
1244        // same as vec_roundm
1245        unsafe fn vec_floor(self) -> Self;
1246    }
1247
1248    impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_float) }
1249    impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_double) }
1250
1251    impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_float) }
1252    impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_double) }
1253
1254    impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_float) }
1255    impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_double) }
1256
1257    macro_rules! impl_vec_shift {
1258        ([$Trait:ident $m:ident] ($b:ident, $h:ident, $w:ident, $g:ident)) => {
1259            impl_vec_trait!{ [$Trait $m]+ $b (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1260            impl_vec_trait!{ [$Trait $m]+ $b (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1261            impl_vec_trait!{ [$Trait $m]+ $h (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1262            impl_vec_trait!{ [$Trait $m]+ $h (vector_signed_short, vector_unsigned_short) -> vector_signed_short }
1263            impl_vec_trait!{ [$Trait $m]+ $w (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1264            impl_vec_trait!{ [$Trait $m]+ $w (vector_signed_int, vector_unsigned_int) -> vector_signed_int }
1265            impl_vec_trait!{ [$Trait $m]+ $g (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1266            impl_vec_trait!{ [$Trait $m]+ $g (vector_signed_long_long, vector_unsigned_long_long) -> vector_signed_long_long }
1267        };
1268    }
1269
1270    macro_rules! impl_shift {
1271        ($fun:ident $intr:ident $ty:ident) => {
1272            #[inline]
1273            #[target_feature(enable = "vector")]
1274            #[cfg_attr(test, assert_instr($fun))]
1275            unsafe fn $fun(a: t_t_l!($ty), b: t_t_l!($ty)) -> t_t_l!($ty) {
1276                let a = transmute(a);
1277                // use the remainder of b by the width of a's elements to prevent UB
1278                let b = simd_rem(transmute(b), <t_t_s!($ty)>::splat($ty::BITS as $ty));
1279
1280                transmute($intr(a, b))
1281            }
1282        };
1283    }
1284
1285    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1286    pub trait VectorSl<Other> {
1287        type Result;
1288        unsafe fn vec_sl(self, b: Other) -> Self::Result;
1289    }
1290
1291    impl_shift! { veslvb simd_shl u8 }
1292    impl_shift! { veslvh simd_shl u16 }
1293    impl_shift! { veslvf simd_shl u32 }
1294    impl_shift! { veslvg simd_shl u64 }
1295
1296    impl_vec_shift! { [VectorSl vec_sl] (veslvb, veslvh, veslvf, veslvg) }
1297
1298    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1299    pub trait VectorSr<Other> {
1300        type Result;
1301        unsafe fn vec_sr(self, b: Other) -> Self::Result;
1302    }
1303
1304    impl_shift! { vesrlvb simd_shr u8 }
1305    impl_shift! { vesrlvh simd_shr u16 }
1306    impl_shift! { vesrlvf simd_shr u32 }
1307    impl_shift! { vesrlvg simd_shr u64 }
1308
1309    impl_vec_shift! { [VectorSr vec_sr] (vesrlvb, vesrlvh, vesrlvf, vesrlvg) }
1310
1311    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1312    pub trait VectorSra<Other> {
1313        type Result;
1314        unsafe fn vec_sra(self, b: Other) -> Self::Result;
1315    }
1316
1317    impl_shift! { vesravb simd_shr i8 }
1318    impl_shift! { vesravh simd_shr i16 }
1319    impl_shift! { vesravf simd_shr i32 }
1320    impl_shift! { vesravg simd_shr i64 }
1321
1322    impl_vec_shift! { [VectorSra vec_sra] (vesravb, vesravh, vesravf, vesravg) }
1323
1324    macro_rules! impl_vec_shift_byte {
1325        ([$trait:ident $m:ident] ($f:ident)) => {
1326            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_signed_char) -> vector_unsigned_char }
1327            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1328            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_signed_char) -> vector_signed_char }
1329            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1330            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_signed_short) -> vector_unsigned_short }
1331            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1332            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_signed_short) -> vector_signed_short }
1333            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_unsigned_short) -> vector_signed_short }
1334            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_signed_int) -> vector_unsigned_int }
1335            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1336            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_signed_int) -> vector_signed_int }
1337            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_unsigned_int) -> vector_signed_int }
1338            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_signed_long_long) -> vector_unsigned_long_long }
1339            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1340            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_signed_long_long) -> vector_signed_long_long }
1341            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_unsigned_long_long) -> vector_signed_long_long }
1342            impl_vec_trait!{ [$trait $m]+ $f (vector_float, vector_signed_int) -> vector_float }
1343            impl_vec_trait!{ [$trait $m]+ $f (vector_float, vector_unsigned_int) -> vector_float }
1344            impl_vec_trait!{ [$trait $m]+ $f (vector_double, vector_signed_long_long) -> vector_double }
1345            impl_vec_trait!{ [$trait $m]+ $f (vector_double, vector_unsigned_long_long) -> vector_double }
1346        };
1347    }
1348
1349    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1350    pub trait VectorSlb<Other> {
1351        type Result;
1352        unsafe fn vec_slb(self, b: Other) -> Self::Result;
1353    }
1354
1355    impl_vec_shift_byte! { [VectorSlb vec_slb] (vslb) }
1356
1357    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1358    pub trait VectorSrab<Other> {
1359        type Result;
1360        unsafe fn vec_srab(self, b: Other) -> Self::Result;
1361    }
1362
1363    impl_vec_shift_byte! { [VectorSrab vec_srab] (vsrab) }
1364
1365    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1366    pub trait VectorSrb<Other> {
1367        type Result;
1368        unsafe fn vec_srb(self, b: Other) -> Self::Result;
1369    }
1370
1371    impl_vec_shift_byte! { [VectorSrb vec_srb] (vsrlb) }
1372
1373    macro_rules! impl_vec_shift_long {
1374        ([$trait:ident $m:ident] ($f:ident)) => {
1375            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1376            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1377            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_unsigned_char) -> vector_unsigned_short }
1378            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_unsigned_char) -> vector_signed_short }
1379            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_unsigned_char) -> vector_unsigned_int }
1380            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_unsigned_char) -> vector_signed_int }
1381            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_unsigned_char) -> vector_unsigned_long_long }
1382            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_unsigned_char) -> vector_signed_long_long }
1383        };
1384    }
1385
1386    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1387    pub trait VectorSrl<Other> {
1388        type Result;
1389        unsafe fn vec_srl(self, b: Other) -> Self::Result;
1390    }
1391
1392    impl_vec_shift_long! { [VectorSrl vec_srl] (vsrl) }
1393
1394    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1395    pub trait VectorSral<Other> {
1396        type Result;
1397        unsafe fn vec_sral(self, b: Other) -> Self::Result;
1398    }
1399
1400    impl_vec_shift_long! { [VectorSral vec_sral] (vsra) }
1401
1402    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1403    pub trait VectorSll<Other> {
1404        type Result;
1405        unsafe fn vec_sll(self, b: Other) -> Self::Result;
1406    }
1407
1408    impl_vec_shift_long! { [VectorSll vec_sll] (vsl) }
1409
1410    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1411    pub trait VectorRl<Other> {
1412        type Result;
1413        unsafe fn vec_rl(self, b: Other) -> Self::Result;
1414    }
1415
1416    macro_rules! impl_rot {
1417        ($fun:ident $ty:ident) => {
1418            #[inline]
1419            #[target_feature(enable = "vector")]
1420            #[cfg_attr(test, assert_instr($fun))]
1421            unsafe fn $fun(a: t_t_l!($ty), b: t_t_l!($ty)) -> t_t_l!($ty) {
1422                simd_funnel_shl(a, a, b)
1423            }
1424        };
1425    }
1426
1427    impl_rot! { verllvb u8 }
1428    impl_rot! { verllvh u16 }
1429    impl_rot! { verllvf u32 }
1430    impl_rot! { verllvg u64 }
1431
1432    impl_vec_shift! { [VectorRl vec_rl] (verllvb, verllvh, verllvf, verllvg) }
1433
1434    macro_rules! test_rot_imm {
1435        ($fun:ident $instr:ident $ty:ident) => {
1436            #[inline]
1437            #[target_feature(enable = "vector")]
1438            #[cfg_attr(test, assert_instr($instr))]
1439            unsafe fn $fun(a: t_t_l!($ty), bits: core::ffi::c_ulong) -> t_t_l!($ty) {
1440                // mod by the number of bits in a's element type to prevent UB
1441                let bits = (bits % $ty::BITS as core::ffi::c_ulong) as $ty;
1442                let b = <t_t_s!($ty)>::splat(bits);
1443
1444                simd_funnel_shl(a, a, transmute(b))
1445            }
1446        };
1447    }
1448
1449    test_rot_imm! { verllvb_imm verllb u8 }
1450    test_rot_imm! { verllvh_imm verllh u16 }
1451    test_rot_imm! { verllvf_imm verllf u32 }
1452    test_rot_imm! { verllvg_imm verllg u64 }
1453
1454    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1455    pub trait VectorRli {
1456        unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self;
1457    }
1458
1459    macro_rules! impl_rot_imm {
1460        ($($ty:ident, $intr:ident),*) => {
1461            $(
1462                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1463                impl VectorRli for $ty {
1464                    #[inline]
1465                    #[target_feature(enable = "vector")]
1466                    unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self {
1467                        transmute($intr(transmute(self), bits))
1468                    }
1469                }
1470
1471                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1472                impl VectorRli for t_u!($ty) {
1473                    #[inline]
1474                    #[target_feature(enable = "vector")]
1475                    unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self {
1476                        $intr(self, bits)
1477                    }
1478                }
1479            )*
1480        }
1481    }
1482
1483    impl_rot_imm! {
1484        vector_signed_char, verllvb_imm,
1485        vector_signed_short, verllvh_imm,
1486        vector_signed_int, verllvf_imm,
1487        vector_signed_long_long, verllvg_imm
1488    }
1489
1490    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1491    pub trait VectorRlMask<Other> {
1492        unsafe fn vec_rl_mask<const IMM8: u8>(self, other: Other) -> Self;
1493    }
1494
1495    macro_rules! impl_rl_mask {
1496        ($($ty:ident, $intr:ident, $fun:ident),*) => {
1497            $(
1498                #[inline]
1499                #[target_feature(enable = "vector")]
1500                #[cfg_attr(test, assert_instr($intr, IMM8 = 6))]
1501                unsafe fn $fun<const IMM8: u8>(a: $ty, b: t_u!($ty)) -> $ty {
1502                    // mod by the number of bits in a's element type to prevent UB
1503                    $intr(a, a, transmute(b), const { (IMM8 % <l_t_t!($ty)>::BITS as u8) as i32 })
1504                }
1505
1506                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1507                impl VectorRlMask<t_u!($ty)> for $ty {
1508                    #[inline]
1509                    #[target_feature(enable = "vector")]
1510                    unsafe fn vec_rl_mask<const IMM8: u8>(self, other: t_u!($ty)) -> Self {
1511                        $fun::<IMM8>(self, other)
1512                    }
1513                }
1514
1515                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1516                impl VectorRlMask<t_u!($ty)> for t_u!($ty) {
1517                    #[inline]
1518                    #[target_feature(enable = "vector")]
1519                    unsafe fn vec_rl_mask<const IMM8: u8>(self, other: t_u!($ty)) -> Self {
1520                        transmute($fun::<IMM8>(transmute(self), transmute(other)))
1521                    }
1522                }
1523            )*
1524        }
1525    }
1526
1527    impl_rl_mask! {
1528        vector_signed_char, verimb, test_verimb,
1529        vector_signed_short, verimh, test_verimh,
1530        vector_signed_int, verimf, test_verimf,
1531        vector_signed_long_long, verimg, test_verimg
1532    }
1533
1534    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1535    pub trait VectorReve {
1536        unsafe fn vec_reve(self) -> Self;
1537    }
1538
1539    #[repr(simd)]
1540    struct ReverseMask<const N: usize>([u32; N]);
1541
1542    impl<const N: usize> ReverseMask<N> {
1543        const fn new() -> Self {
1544            let mut index = [0; N];
1545            let mut i = 0;
1546            while i < N {
1547                index[i] = (N - i - 1) as u32;
1548                i += 1;
1549            }
1550            ReverseMask(index)
1551        }
1552    }
1553
1554    macro_rules! impl_reve {
1555        ($($ty:ident, $fun:ident, $instr:ident),*) => {
1556            $(
1557                #[inline]
1558                #[target_feature(enable = "vector")]
1559                #[cfg_attr(test, assert_instr($instr))]
1560                unsafe fn $fun(a: $ty) -> $ty {
1561                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1562                    simd_shuffle(a, a, const { ShuffleMask::<N>::reverse() })
1563                }
1564
1565                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1566                impl VectorReve for $ty {
1567                    #[inline]
1568                    #[target_feature(enable = "vector")]
1569                    unsafe fn vec_reve(self) -> Self {
1570                        $fun(self)
1571                    }
1572                }
1573
1574                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1575                impl VectorReve for t_u!($ty) {
1576                    #[inline]
1577                    #[target_feature(enable = "vector")]
1578                    unsafe fn vec_reve(self) -> Self {
1579                        transmute($fun(transmute(self)))
1580                    }
1581                }
1582
1583                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1584                impl VectorReve for t_b!($ty) {
1585                    #[inline]
1586                    #[target_feature(enable = "vector")]
1587                    unsafe fn vec_reve(self) -> Self {
1588                        transmute($fun(transmute(self)))
1589                    }
1590                }
1591            )*
1592        }
1593    }
1594
1595    impl_reve! {
1596        vector_signed_char, reveb, vperm,
1597        vector_signed_short, reveh, vperm,
1598        vector_signed_int, revef, vperm,
1599        vector_signed_long_long, reveg, vpdi
1600    }
1601
1602    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1603    impl VectorReve for vector_float {
1604        #[inline]
1605        #[target_feature(enable = "vector")]
1606        unsafe fn vec_reve(self) -> Self {
1607            transmute(transmute::<_, vector_signed_int>(self).vec_reve())
1608        }
1609    }
1610
1611    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1612    impl VectorReve for vector_double {
1613        #[inline]
1614        #[target_feature(enable = "vector")]
1615        unsafe fn vec_reve(self) -> Self {
1616            transmute(transmute::<_, vector_signed_long_long>(self).vec_reve())
1617        }
1618    }
1619
1620    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1621    pub trait VectorRevb {
1622        unsafe fn vec_revb(self) -> Self;
1623    }
1624
1625    test_impl! { bswapb (a: vector_signed_char) -> vector_signed_char [simd_bswap, _] }
1626    test_impl! { bswaph (a: vector_signed_short) -> vector_signed_short [simd_bswap, vperm] }
1627    test_impl! { bswapf (a: vector_signed_int) -> vector_signed_int [simd_bswap, vperm] }
1628    test_impl! { bswapg (a: vector_signed_long_long) -> vector_signed_long_long [simd_bswap, vperm] }
1629
1630    impl_vec_trait! { [VectorRevb vec_revb]+ bswapb (vector_unsigned_char) }
1631    impl_vec_trait! { [VectorRevb vec_revb]+ bswapb (vector_signed_char) }
1632    impl_vec_trait! { [VectorRevb vec_revb]+ bswaph (vector_unsigned_short) }
1633    impl_vec_trait! { [VectorRevb vec_revb]+ bswaph (vector_signed_short) }
1634    impl_vec_trait! { [VectorRevb vec_revb]+ bswapf (vector_unsigned_int) }
1635    impl_vec_trait! { [VectorRevb vec_revb]+ bswapf (vector_signed_int) }
1636    impl_vec_trait! { [VectorRevb vec_revb]+ bswapg (vector_unsigned_long_long) }
1637    impl_vec_trait! { [VectorRevb vec_revb]+ bswapg (vector_signed_long_long) }
1638
1639    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1640    impl VectorRevb for vector_float {
1641        #[inline]
1642        #[target_feature(enable = "vector")]
1643        unsafe fn vec_revb(self) -> Self {
1644            transmute(transmute::<_, vector_signed_int>(self).vec_revb())
1645        }
1646    }
1647
1648    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1649    impl VectorRevb for vector_double {
1650        #[inline]
1651        #[target_feature(enable = "vector")]
1652        unsafe fn vec_revb(self) -> Self {
1653            transmute(transmute::<_, vector_signed_long_long>(self).vec_revb())
1654        }
1655    }
1656
1657    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1658    pub trait VectorMergel {
1659        unsafe fn vec_mergel(self, other: Self) -> Self;
1660    }
1661
1662    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1663    pub trait VectorMergeh {
1664        unsafe fn vec_mergeh(self, other: Self) -> Self;
1665    }
1666
1667    macro_rules! impl_merge {
1668        ($($ty:ident, $mergel:ident, $mergeh:ident),*) => {
1669            $(
1670                #[inline]
1671                #[target_feature(enable = "vector")]
1672                #[cfg_attr(test, assert_instr($mergel))]
1673                unsafe fn $mergel(a: $ty, b: $ty) -> $ty {
1674                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1675                    simd_shuffle(a, b, const { ShuffleMask::<N>::merge_low() })
1676                }
1677
1678                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1679                impl VectorMergel for $ty {
1680                    #[inline]
1681                    #[target_feature(enable = "vector")]
1682                    unsafe fn vec_mergel(self, other: Self) -> Self {
1683                        $mergel(self, other)
1684                    }
1685                }
1686
1687                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1688                impl VectorMergel for t_u!($ty) {
1689                    #[inline]
1690                    #[target_feature(enable = "vector")]
1691                    unsafe fn vec_mergel(self, other: Self) -> Self {
1692                        transmute($mergel(transmute(self), transmute(other)))
1693                    }
1694                }
1695
1696                #[inline]
1697                #[target_feature(enable = "vector")]
1698                #[cfg_attr(test, assert_instr($mergeh))]
1699                unsafe fn $mergeh(a: $ty, b: $ty) -> $ty {
1700                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1701                    simd_shuffle(a, b, const { ShuffleMask::<N>::merge_high() })
1702                }
1703
1704                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1705                impl VectorMergeh for $ty {
1706                    #[inline]
1707                    #[target_feature(enable = "vector")]
1708                    unsafe fn vec_mergeh(self, other: Self) -> Self {
1709                        $mergeh(self, other)
1710                    }
1711                }
1712
1713                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1714                impl VectorMergeh for t_u!($ty) {
1715                    #[inline]
1716                    #[target_feature(enable = "vector")]
1717                    unsafe fn vec_mergeh(self, other: Self) -> Self {
1718                        transmute($mergeh(transmute(self), transmute(other)))
1719                    }
1720                }
1721            )*
1722        }
1723    }
1724
1725    impl_merge! {
1726        vector_signed_char, vmrlb, vmrhb,
1727        vector_signed_short, vmrlh, vmrhh,
1728        vector_signed_int, vmrlf, vmrhf,
1729        vector_signed_long_long, vmrlg, vmrhg
1730    }
1731
1732    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1733    pub trait VectorPerm {
1734        unsafe fn vec_perm(self, other: Self, c: vector_unsigned_char) -> Self;
1735    }
1736
1737    macro_rules! impl_merge {
1738        ($($ty:ident),*) => {
1739            $(
1740                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1741                impl VectorPerm for $ty {
1742                    #[inline]
1743                    #[target_feature(enable = "vector")]
1744                    unsafe fn vec_perm(self, other: Self, c: vector_unsigned_char) -> Self {
1745                        transmute(vperm(transmute(self), transmute(other), c))
1746                    }
1747                }
1748            )*
1749        }
1750    }
1751
1752    impl_merge! {
1753        vector_signed_char,
1754        vector_signed_short,
1755        vector_signed_int,
1756        vector_signed_long_long,
1757        vector_unsigned_char,
1758        vector_unsigned_short,
1759        vector_unsigned_int,
1760        vector_unsigned_long_long,
1761        vector_bool_char,
1762        vector_bool_short,
1763        vector_bool_int,
1764        vector_bool_long_long,
1765        vector_float,
1766        vector_double
1767    }
1768
1769    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1770    pub trait VectorSumU128 {
1771        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char;
1772    }
1773
1774    #[inline]
1775    #[target_feature(enable = "vector")]
1776    #[cfg_attr(test, assert_instr(vsumqf))]
1777    pub unsafe fn vec_vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128 {
1778        transmute(vsumqf(a, b))
1779    }
1780
1781    #[inline]
1782    #[target_feature(enable = "vector")]
1783    #[cfg_attr(test, assert_instr(vsumqg))]
1784    pub unsafe fn vec_vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128 {
1785        transmute(vsumqg(a, b))
1786    }
1787
1788    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1789    impl VectorSumU128 for vector_unsigned_int {
1790        #[inline]
1791        #[target_feature(enable = "vector")]
1792        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char {
1793            transmute(vec_vsumqf(self, other))
1794        }
1795    }
1796
1797    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1798    impl VectorSumU128 for vector_unsigned_long_long {
1799        #[inline]
1800        #[target_feature(enable = "vector")]
1801        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char {
1802            transmute(vec_vsumqg(self, other))
1803        }
1804    }
1805
1806    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1807    pub trait VectorSum2 {
1808        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long;
1809    }
1810
1811    test_impl! { vec_vsumgh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_long_long [vsumgh, vsumgh] }
1812    test_impl! { vec_vsumgf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [vsumgf, vsumgf] }
1813
1814    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1815    impl VectorSum2 for vector_unsigned_short {
1816        #[inline]
1817        #[target_feature(enable = "vector")]
1818        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long {
1819            vec_vsumgh(self, other)
1820        }
1821    }
1822
1823    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1824    impl VectorSum2 for vector_unsigned_int {
1825        #[inline]
1826        #[target_feature(enable = "vector")]
1827        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long {
1828            vec_vsumgf(self, other)
1829        }
1830    }
1831
1832    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1833    pub trait VectorSum4 {
1834        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int;
1835    }
1836
1837    test_impl! { vec_vsumb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_int [vsumb, vsumb] }
1838    test_impl! { vec_vsumh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int [vsumh, vsumh] }
1839
1840    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1841    impl VectorSum4 for vector_unsigned_char {
1842        #[inline]
1843        #[target_feature(enable = "vector")]
1844        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int {
1845            vec_vsumb(self, other)
1846        }
1847    }
1848
1849    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1850    impl VectorSum4 for vector_unsigned_short {
1851        #[inline]
1852        #[target_feature(enable = "vector")]
1853        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int {
1854            vec_vsumh(self, other)
1855        }
1856    }
1857
1858    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1859    pub trait VectorSubc<Other> {
1860        type Result;
1861        unsafe fn vec_subc(self, b: Other) -> Self::Result;
1862    }
1863
1864    test_impl! { vec_vscbib (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vscbib, vscbib] }
1865    test_impl! { vec_vscbih (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vscbih, vscbih] }
1866    test_impl! { vec_vscbif (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vscbif, vscbif] }
1867    test_impl! { vec_vscbig (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vscbig, vscbig] }
1868
1869    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbib (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1870    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbih (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1871    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbif (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1872    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbig (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1873
1874    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1875    pub trait VectorSqrt {
1876        unsafe fn vec_sqrt(self) -> Self;
1877    }
1878
1879    test_impl! { vec_sqrt_f32 (v: vector_float) -> vector_float [ simd_fsqrt, "vector-enhancements-1" vfsqsb ] }
1880    test_impl! { vec_sqrt_f64 (v: vector_double) -> vector_double [ simd_fsqrt, vfsqdb ] }
1881
1882    impl_vec_trait! { [VectorSqrt vec_sqrt] vec_sqrt_f32 (vector_float) }
1883    impl_vec_trait! { [VectorSqrt vec_sqrt] vec_sqrt_f64 (vector_double) }
1884
1885    macro_rules! vfae_wrapper {
1886        ($($name:ident $ty:ident)*) => {
1887            $(
1888                #[inline]
1889                #[target_feature(enable = "vector")]
1890                #[cfg_attr(test, assert_instr($name, IMM = 0))]
1891                unsafe fn $name<const IMM: i32>(
1892                    a: $ty,
1893                    b: $ty,
1894                ) -> $ty {
1895                    super::$name(a, b, IMM)
1896                }
1897            )*
1898        }
1899     }
1900
1901    vfae_wrapper! {
1902       vfaeb vector_signed_char
1903       vfaeh vector_signed_short
1904       vfaef vector_signed_int
1905
1906       vfaezb vector_signed_char
1907       vfaezh vector_signed_short
1908       vfaezf vector_signed_int
1909    }
1910
1911    macro_rules! impl_vfae {
1912        ([idx_cc $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1913            impl_vfae! { [idx_cc $Trait $m] $imm
1914                $b vector_signed_char vector_signed_char
1915                $b vector_unsigned_char vector_unsigned_char
1916                $b vector_bool_char vector_unsigned_char
1917
1918                $h vector_signed_short vector_signed_short
1919                $h vector_unsigned_short vector_unsigned_short
1920                $h vector_bool_short vector_unsigned_short
1921
1922                $f vector_signed_int vector_signed_int
1923                $f vector_unsigned_int vector_unsigned_int
1924                $f vector_bool_int vector_unsigned_int
1925            }
1926        };
1927        ([idx_cc $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident $r:ident)*) => {
1928            $(
1929                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1930                impl $Trait<Self> for $ty {
1931                    type Result = $r;
1932                    #[inline]
1933                    #[target_feature(enable = "vector")]
1934                    unsafe fn $m(self, b: Self) -> (Self::Result, i32) {
1935                        let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b));
1936                        (transmute(x), y)
1937                    }
1938                }
1939            )*
1940        };
1941        ([cc $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1942            impl_vfae! { [cc $Trait $m] $imm
1943                $b vector_signed_char
1944                $b vector_unsigned_char
1945                $b vector_bool_char
1946
1947                $h vector_signed_short
1948                $h vector_unsigned_short
1949                $h vector_bool_short
1950
1951                $f vector_signed_int
1952                $f vector_unsigned_int
1953                $f vector_bool_int
1954            }
1955        };
1956        ([cc $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident)*) => {
1957            $(
1958                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1959                impl $Trait<Self> for $ty {
1960                    type Result = t_b!($ty);
1961                    #[inline]
1962                    #[target_feature(enable = "vector")]
1963                    unsafe fn $m(self, b: Self) -> (Self::Result, i32) {
1964                        let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b));
1965                        (transmute(x), y)
1966                    }
1967                }
1968            )*
1969        };
1970        ([idx $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1971            impl_vfae! { [idx $Trait $m] $imm
1972                $b vector_signed_char vector_signed_char
1973                $b vector_unsigned_char vector_unsigned_char
1974                $b vector_bool_char vector_unsigned_char
1975
1976                $h vector_signed_short vector_signed_short
1977                $h vector_unsigned_short vector_unsigned_short
1978                $h vector_bool_short vector_unsigned_short
1979
1980                $f vector_signed_int vector_signed_int
1981                $f vector_unsigned_int vector_unsigned_int
1982                $f vector_bool_int vector_unsigned_int
1983            }
1984        };
1985        ([idx $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident $r:ident)*) => {
1986            $(
1987                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1988                impl $Trait<Self> for $ty {
1989                    type Result = $r;
1990                    #[inline]
1991                    #[target_feature(enable = "vector")]
1992                    unsafe fn $m(self, b: Self) -> Self::Result {
1993                        transmute($fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)))
1994                    }
1995                }
1996            )*
1997        };
1998        ([$Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1999            impl_vfae! { [$Trait $m] $imm
2000                $b vector_signed_char
2001                $b vector_unsigned_char
2002                $b vector_bool_char
2003
2004                $h vector_signed_short
2005                $h vector_unsigned_short
2006                $h vector_bool_short
2007
2008                $f vector_signed_int
2009                $f vector_unsigned_int
2010                $f vector_bool_int
2011            }
2012        };
2013        ([$Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident)*) => {
2014            $(
2015                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2016                impl $Trait<Self> for $ty {
2017                    type Result = t_b!($ty);
2018                    #[inline]
2019                    #[target_feature(enable = "vector")]
2020                    unsafe fn $m(self, b: Self) -> Self::Result {
2021                        transmute($fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)))
2022                    }
2023                }
2024            )*
2025        };
2026    }
2027
2028    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2029    pub trait VectorFindAnyEq<Other> {
2030        type Result;
2031        unsafe fn vec_find_any_eq(self, other: Other) -> Self::Result;
2032    }
2033
2034    impl_vfae! { [VectorFindAnyEq vec_find_any_eq] Eq vfaeb vfaeh vfaef }
2035
2036    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2037    pub trait VectorFindAnyNe<Other> {
2038        type Result;
2039        unsafe fn vec_find_any_ne(self, other: Other) -> Self::Result;
2040    }
2041
2042    impl_vfae! { [VectorFindAnyNe vec_find_any_ne] Ne vfaeb vfaeh vfaef }
2043
2044    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2045    pub trait VectorFindAnyEqOrZeroIdx<Other> {
2046        type Result;
2047        unsafe fn vec_find_any_eq_or_0_idx(self, other: Other) -> Self::Result;
2048    }
2049
2050    impl_vfae! { [idx VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx] EqIdx
2051        vfaezb vector_signed_char vector_signed_char
2052        vfaezb vector_unsigned_char vector_unsigned_char
2053        vfaezb vector_bool_char vector_unsigned_char
2054
2055        vfaezh vector_signed_short vector_signed_short
2056        vfaezh vector_unsigned_short vector_unsigned_short
2057        vfaezh vector_bool_short vector_unsigned_short
2058
2059        vfaezf vector_signed_int vector_signed_int
2060        vfaezf vector_unsigned_int vector_unsigned_int
2061        vfaezf vector_bool_int vector_unsigned_int
2062    }
2063
2064    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2065    pub trait VectorFindAnyNeOrZeroIdx<Other> {
2066        type Result;
2067        unsafe fn vec_find_any_ne_or_0_idx(self, other: Other) -> Self::Result;
2068    }
2069
2070    impl_vfae! { [idx VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx] NeIdx
2071        vfaezb vector_signed_char vector_signed_char
2072        vfaezb vector_unsigned_char vector_unsigned_char
2073        vfaezb vector_bool_char vector_unsigned_char
2074
2075        vfaezh vector_signed_short vector_signed_short
2076        vfaezh vector_unsigned_short vector_unsigned_short
2077        vfaezh vector_bool_short vector_unsigned_short
2078
2079        vfaezf vector_signed_int vector_signed_int
2080        vfaezf vector_unsigned_int vector_unsigned_int
2081        vfaezf vector_bool_int vector_unsigned_int
2082    }
2083
2084    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2085    pub trait VectorFindAnyEqIdx<Other> {
2086        type Result;
2087        unsafe fn vec_find_any_eq_idx(self, other: Other) -> Self::Result;
2088    }
2089
2090    impl_vfae! { [idx VectorFindAnyEqIdx vec_find_any_eq_idx] EqIdx vfaeb vfaeh vfaef }
2091
2092    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2093    pub trait VectorFindAnyNeIdx<Other> {
2094        type Result;
2095        unsafe fn vec_find_any_ne_idx(self, other: Other) -> Self::Result;
2096    }
2097
2098    impl_vfae! { [idx VectorFindAnyNeIdx vec_find_any_ne_idx] NeIdx vfaeb vfaeh vfaef }
2099
2100    macro_rules! vfaes_wrapper {
2101        ($($name:ident $ty:ident)*) => {
2102            $(
2103                #[inline]
2104                #[target_feature(enable = "vector")]
2105                #[cfg_attr(test, assert_instr($name, IMM = 0))]
2106                unsafe fn $name<const IMM: i32>(
2107                    a: $ty,
2108                    b: $ty,
2109                ) -> PackedTuple<$ty, i32> {
2110                    super::$name(a, b, IMM)
2111                }
2112            )*
2113        }
2114     }
2115
2116    vfaes_wrapper! {
2117        vfaebs vector_signed_char
2118        vfaehs vector_signed_short
2119        vfaefs vector_signed_int
2120
2121        vfaezbs vector_signed_char
2122        vfaezhs vector_signed_short
2123        vfaezfs vector_signed_int
2124    }
2125
2126    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2127    pub trait VectorFindAnyEqCC<Other> {
2128        type Result;
2129        unsafe fn vec_find_any_eq_cc(self, other: Other) -> (Self::Result, i32);
2130    }
2131
2132    impl_vfae! { [cc VectorFindAnyEqCC vec_find_any_eq_cc] Eq vfaebs vfaehs vfaefs }
2133
2134    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2135    pub trait VectorFindAnyNeCC<Other> {
2136        type Result;
2137        unsafe fn vec_find_any_ne_cc(self, other: Other) -> (Self::Result, i32);
2138    }
2139
2140    impl_vfae! { [cc VectorFindAnyNeCC vec_find_any_ne_cc] Ne vfaebs vfaehs vfaefs }
2141
2142    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2143    pub trait VectorFindAnyEqIdxCC<Other> {
2144        type Result;
2145        unsafe fn vec_find_any_eq_idx_cc(self, other: Other) -> (Self::Result, i32);
2146    }
2147
2148    impl_vfae! { [idx_cc VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc] EqIdx vfaebs vfaehs vfaefs }
2149
2150    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2151    pub trait VectorFindAnyNeIdxCC<Other> {
2152        type Result;
2153        unsafe fn vec_find_any_ne_idx_cc(self, other: Other) -> (Self::Result, i32);
2154    }
2155
2156    impl_vfae! { [idx_cc VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc] NeIdx vfaebs vfaehs vfaefs }
2157
2158    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2159    pub trait VectorFindAnyEqOrZeroIdxCC<Other> {
2160        type Result;
2161        unsafe fn vec_find_any_eq_or_0_idx_cc(self, other: Other) -> (Self::Result, i32);
2162    }
2163
2164    impl_vfae! { [idx_cc VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc] EqIdx vfaezbs vfaezhs vfaezfs }
2165
2166    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2167    pub trait VectorFindAnyNeOrZeroIdxCC<Other> {
2168        type Result;
2169        unsafe fn vec_find_any_ne_or_0_idx_cc(self, other: Other) -> (Self::Result, i32);
2170    }
2171
2172    impl_vfae! { [idx_cc VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc] NeIdx vfaezbs vfaezhs vfaezfs }
2173
2174    #[inline]
2175    #[target_feature(enable = "vector")]
2176    #[cfg_attr(test, assert_instr(vl))]
2177    unsafe fn test_vector_load(offset: isize, ptr: *const i32) -> vector_signed_int {
2178        ptr.byte_offset(offset)
2179            .cast::<vector_signed_int>()
2180            .read_unaligned()
2181    }
2182
2183    #[inline]
2184    #[target_feature(enable = "vector")]
2185    #[cfg_attr(test, assert_instr(vst))]
2186    unsafe fn test_vector_store(vector: vector_signed_int, offset: isize, ptr: *mut i32) {
2187        ptr.byte_offset(offset)
2188            .cast::<vector_signed_int>()
2189            .write_unaligned(vector)
2190    }
2191
2192    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2193    pub trait VectorLoad: Sized {
2194        type ElementType;
2195
2196        #[inline]
2197        #[target_feature(enable = "vector")]
2198        unsafe fn vec_xl(offset: isize, ptr: *const Self::ElementType) -> Self {
2199            ptr.byte_offset(offset).cast::<Self>().read_unaligned()
2200        }
2201
2202        unsafe fn vec_load_len(ptr: *const Self::ElementType, byte_count: u32) -> Self;
2203
2204        unsafe fn vec_load_bndry<const BLOCK_BOUNDARY: u16>(
2205            ptr: *const Self::ElementType,
2206        ) -> MaybeUninit<Self>;
2207    }
2208
2209    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2210    pub trait VectorStore: Sized {
2211        type ElementType;
2212
2213        #[inline]
2214        #[target_feature(enable = "vector")]
2215        unsafe fn vec_xst(self, offset: isize, ptr: *mut Self::ElementType) {
2216            ptr.byte_offset(offset).cast::<Self>().write_unaligned(self)
2217        }
2218
2219        unsafe fn vec_store_len(self, ptr: *mut Self::ElementType, byte_count: u32);
2220    }
2221
2222    macro_rules! impl_load_store {
2223        ($($ty:ident)*) => {
2224            $(
2225                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2226                impl VectorLoad for t_t_l!($ty) {
2227                    type ElementType = $ty;
2228
2229                    #[inline]
2230                    #[target_feature(enable = "vector")]
2231                    unsafe fn vec_load_len(ptr: *const Self::ElementType, byte_count: u32) -> Self {
2232                        transmute(vll( byte_count, ptr.cast(),))
2233                    }
2234
2235                    #[inline]
2236                    #[target_feature(enable = "vector")]
2237                    unsafe fn vec_load_bndry<const BLOCK_BOUNDARY: u16>(ptr: *const Self::ElementType) -> MaybeUninit<Self> {
2238                        transmute(vlbb(ptr.cast(), const { validate_block_boundary(BLOCK_BOUNDARY) }))
2239                    }
2240
2241                }
2242
2243                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2244                impl VectorStore for t_t_l!($ty) {
2245                    type ElementType = $ty;
2246
2247                    #[inline]
2248                    #[target_feature(enable = "vector")]
2249                    unsafe fn vec_store_len(self, ptr: *mut Self::ElementType, byte_count: u32) {
2250                        vstl(transmute(self), byte_count, ptr.cast())
2251                    }
2252                }
2253            )*
2254        }
2255    }
2256
2257    impl_load_store! { i8 u8 i16 u16 i32 u32 i64 u64 f32 f64 }
2258
2259    #[inline]
2260    #[target_feature(enable = "vector")]
2261    #[cfg_attr(test, assert_instr(vll))]
2262    unsafe fn test_vec_load_len(ptr: *const i32, byte_count: u32) -> vector_signed_int {
2263        vector_signed_int::vec_load_len(ptr, byte_count)
2264    }
2265
2266    #[inline]
2267    #[target_feature(enable = "vector")]
2268    #[cfg_attr(test, assert_instr(vlbb))]
2269    unsafe fn test_vec_load_bndry(ptr: *const i32) -> MaybeUninit<vector_signed_int> {
2270        vector_signed_int::vec_load_bndry::<512>(ptr)
2271    }
2272
2273    #[inline]
2274    #[target_feature(enable = "vector")]
2275    #[cfg_attr(test, assert_instr(vstl))]
2276    unsafe fn test_vec_store_len(vector: vector_signed_int, ptr: *mut i32, byte_count: u32) {
2277        vector.vec_store_len(ptr, byte_count)
2278    }
2279
2280    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2281    pub trait VectorLoadPair: Sized {
2282        type ElementType;
2283
2284        unsafe fn vec_load_pair(a: Self::ElementType, b: Self::ElementType) -> Self;
2285    }
2286
2287    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2288    impl VectorLoadPair for vector_signed_long_long {
2289        type ElementType = i64;
2290
2291        #[inline]
2292        #[target_feature(enable = "vector")]
2293        unsafe fn vec_load_pair(a: i64, b: i64) -> Self {
2294            vector_signed_long_long([a, b])
2295        }
2296    }
2297
2298    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2299    impl VectorLoadPair for vector_unsigned_long_long {
2300        type ElementType = u64;
2301
2302        #[inline]
2303        #[target_feature(enable = "vector")]
2304        unsafe fn vec_load_pair(a: u64, b: u64) -> Self {
2305            vector_unsigned_long_long([a, b])
2306        }
2307    }
2308
2309    #[inline]
2310    #[target_feature(enable = "vector")]
2311    unsafe fn pack<T, const N: usize>(a: T, b: T) -> T {
2312        simd_shuffle(a, b, const { ShuffleMask::<N>::pack() })
2313    }
2314
2315    #[inline]
2316    #[target_feature(enable = "vector")]
2317    #[cfg_attr(test, assert_instr(vpkh))]
2318    unsafe fn vpkh(a: i16x8, b: i16x8) -> i8x16 {
2319        let a: i8x16 = transmute(a);
2320        let b: i8x16 = transmute(b);
2321        simd_shuffle(a, b, const { ShuffleMask::<16>::pack() })
2322    }
2323    #[inline]
2324    #[target_feature(enable = "vector")]
2325    #[cfg_attr(test, assert_instr(vpkf))]
2326    unsafe fn vpkf(a: i32x4, b: i32x4) -> i16x8 {
2327        let a: i16x8 = transmute(a);
2328        let b: i16x8 = transmute(b);
2329        simd_shuffle(a, b, const { ShuffleMask::<8>::pack() })
2330    }
2331    #[inline]
2332    #[target_feature(enable = "vector")]
2333    #[cfg_attr(test, assert_instr(vpkg))]
2334    unsafe fn vpkg(a: i64x2, b: i64x2) -> i32x4 {
2335        let a: i32x4 = transmute(a);
2336        let b: i32x4 = transmute(b);
2337        simd_shuffle(a, b, const { ShuffleMask::<4>::pack() })
2338    }
2339
2340    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2341    pub trait VectorPack<Other> {
2342        type Result;
2343        unsafe fn vec_pack(self, b: Other) -> Self::Result;
2344    }
2345
2346    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_signed_short, vector_signed_short) -> vector_signed_char }
2347    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2348    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_bool_short, vector_bool_short) -> vector_bool_char }
2349    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_signed_int, vector_signed_int) -> vector_signed_short }
2350    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2351    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_bool_int, vector_bool_int) -> vector_bool_short }
2352    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_signed_long_long, vector_signed_long_long) -> vector_signed_int }
2353    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2354    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_bool_long_long, vector_bool_long_long) -> vector_bool_int }
2355
2356    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2357    pub trait VectorPacks<Other> {
2358        type Result;
2359        unsafe fn vec_packs(self, b: Other) -> Self::Result;
2360    }
2361
2362    impl_vec_trait! { [VectorPacks vec_packs] vpksh (vector_signed_short, vector_signed_short) -> vector_signed_char }
2363    impl_vec_trait! { [VectorPacks vec_packs] vpklsh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2364    impl_vec_trait! { [VectorPacks vec_packs] vpksf (vector_signed_int, vector_signed_int) -> vector_signed_short }
2365    impl_vec_trait! { [VectorPacks vec_packs] vpklsf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2366    impl_vec_trait! { [VectorPacks vec_packs] vpksg (vector_signed_long_long, vector_signed_long_long) -> vector_signed_int }
2367    impl_vec_trait! { [VectorPacks vec_packs] vpklsg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2368
2369    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2370    pub trait VectorPacksu<Other> {
2371        type Result;
2372        unsafe fn vec_packsu(self, b: Other) -> Self::Result;
2373    }
2374
2375    unsafe fn simd_smax<T: Copy>(a: T, b: T) -> T {
2376        simd_select::<T, T>(simd_gt::<T, T>(a, b), a, b)
2377    }
2378
2379    #[inline]
2380    #[target_feature(enable = "vector")]
2381    #[cfg_attr(test, assert_instr(vpklsh))]
2382    unsafe fn vpacksuh(a: vector_signed_short, b: vector_signed_short) -> vector_unsigned_char {
2383        vpklsh(
2384            simd_smax(a, vector_signed_short([0; 8])),
2385            simd_smax(b, vector_signed_short([0; 8])),
2386        )
2387    }
2388    #[inline]
2389    #[target_feature(enable = "vector")]
2390    #[cfg_attr(test, assert_instr(vpklsf))]
2391    unsafe fn vpacksuf(a: vector_signed_int, b: vector_signed_int) -> vector_unsigned_short {
2392        vpklsf(
2393            simd_smax(a, vector_signed_int([0; 4])),
2394            simd_smax(b, vector_signed_int([0; 4])),
2395        )
2396    }
2397    #[inline]
2398    #[target_feature(enable = "vector")]
2399    #[cfg_attr(test, assert_instr(vpklsg))]
2400    unsafe fn vpacksug(
2401        a: vector_signed_long_long,
2402        b: vector_signed_long_long,
2403    ) -> vector_unsigned_int {
2404        vpklsg(
2405            simd_smax(a, vector_signed_long_long([0; 2])),
2406            simd_smax(b, vector_signed_long_long([0; 2])),
2407        )
2408    }
2409
2410    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksuh (vector_signed_short, vector_signed_short) -> vector_unsigned_char }
2411    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2412    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksuf (vector_signed_int, vector_signed_int) -> vector_unsigned_short }
2413    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2414    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksug (vector_signed_long_long, vector_signed_long_long) -> vector_unsigned_int }
2415    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2416
2417    macro_rules! impl_vector_packs_cc {
2418        ($($intr:ident $ty:ident $outty:ident)*) => {
2419            $(
2420                #[inline]
2421                #[target_feature(enable = "vector")]
2422                #[cfg_attr(test, assert_instr($intr))]
2423                unsafe fn $intr(
2424                    a: $ty,
2425                    b: $ty,
2426                ) -> ($outty, i32) {
2427                    let PackedTuple { x, y } = super::$intr(a, b);
2428                    (x, y)
2429                }
2430
2431                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2432                impl VectorPacksCC for $ty {
2433                    type Result = $outty;
2434
2435                    #[inline]
2436                    #[target_feature(enable = "vector")]
2437                    unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32) {
2438                        $intr(self, b)
2439                    }
2440                }
2441            )*
2442        }
2443    }
2444
2445    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2446    pub trait VectorPacksCC {
2447        type Result;
2448        unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32);
2449    }
2450
2451    impl_vector_packs_cc! {
2452        vpkshs vector_signed_short vector_signed_char
2453        vpklshs vector_unsigned_short vector_unsigned_char
2454        vpksfs vector_signed_int vector_signed_short
2455        vpklsfs vector_unsigned_int vector_unsigned_short
2456        vpksgs vector_signed_long_long vector_signed_int
2457        vpklsgs vector_unsigned_long_long vector_unsigned_int
2458    }
2459
2460    macro_rules! impl_vector_packsu_cc {
2461        ($($intr:ident $ty:ident $outty:ident)*) => {
2462            $(
2463                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2464                impl VectorPacksuCC for $ty {
2465                    type Result = $outty;
2466
2467                    #[inline]
2468                    #[target_feature(enable = "vector")]
2469                    unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32) {
2470                        $intr(self, b)
2471                    }
2472                }
2473            )*
2474        }
2475    }
2476
2477    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2478    pub trait VectorPacksuCC {
2479        type Result;
2480        unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32);
2481    }
2482
2483    impl_vector_packsu_cc! {
2484        vpklshs vector_unsigned_short vector_unsigned_char
2485        vpklsfs vector_unsigned_int vector_unsigned_short
2486        vpklsgs vector_unsigned_long_long vector_unsigned_int
2487    }
2488
2489    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2490    pub trait VectorMadd {
2491        unsafe fn vec_madd(self, b: Self, c: Self) -> Self;
2492        unsafe fn vec_msub(self, b: Self, c: Self) -> Self;
2493    }
2494
2495    test_impl! { vfmasb (a: vector_float, b: vector_float, c: vector_float) -> vector_float [simd_fma, "vector-enhancements-1" vfmasb] }
2496    test_impl! { vfmadb (a: vector_double, b: vector_double, c: vector_double) -> vector_double [simd_fma, vfmadb] }
2497
2498    #[inline]
2499    unsafe fn simd_fms<T>(a: T, b: T, c: T) -> T {
2500        simd_fma(a, b, simd_neg(c))
2501    }
2502
2503    test_impl! { vfmssb (a: vector_float, b: vector_float, c: vector_float) -> vector_float [simd_fms, "vector-enhancements-1" vfmssb] }
2504    test_impl! { vfmsdb (a: vector_double, b: vector_double, c: vector_double) -> vector_double [simd_fms, vfmsdb] }
2505
2506    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2507    impl VectorMadd for vector_float {
2508        #[inline]
2509        #[target_feature(enable = "vector")]
2510        unsafe fn vec_madd(self, b: Self, c: Self) -> Self {
2511            vfmasb(self, b, c)
2512        }
2513
2514        #[inline]
2515        #[target_feature(enable = "vector")]
2516        unsafe fn vec_msub(self, b: Self, c: Self) -> Self {
2517            vfmssb(self, b, c)
2518        }
2519    }
2520
2521    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2522    impl VectorMadd for vector_double {
2523        #[inline]
2524        #[target_feature(enable = "vector")]
2525        unsafe fn vec_madd(self, b: Self, c: Self) -> Self {
2526            vfmadb(self, b, c)
2527        }
2528
2529        #[inline]
2530        #[target_feature(enable = "vector")]
2531        unsafe fn vec_msub(self, b: Self, c: Self) -> Self {
2532            vfmsdb(self, b, c)
2533        }
2534    }
2535
2536    macro_rules! impl_vec_unpack {
2537        ($mask:ident $instr:ident $src:ident $shuffled:ident $dst:ident $width:literal) => {
2538            #[inline]
2539            #[target_feature(enable = "vector")]
2540            #[cfg_attr(test, assert_instr($instr))]
2541            unsafe fn $instr(a: $src) -> $dst {
2542                simd_as(simd_shuffle::<_, _, $shuffled>(
2543                    a,
2544                    a,
2545                    const { ShuffleMask::<$width>::$mask() },
2546                ))
2547            }
2548        };
2549    }
2550
2551    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2552    pub trait VectorUnpackh {
2553        type Result;
2554        unsafe fn vec_unpackh(self) -> Self::Result;
2555    }
2556
2557    impl_vec_unpack!(unpack_high vuphb vector_signed_char i8x8 vector_signed_short 8);
2558    impl_vec_unpack!(unpack_high vuphh vector_signed_short i16x4 vector_signed_int 4);
2559    impl_vec_unpack!(unpack_high vuphf vector_signed_int i32x2 vector_signed_long_long 2);
2560
2561    impl_vec_unpack!(unpack_high vuplhb vector_unsigned_char u8x8 vector_unsigned_short 8);
2562    impl_vec_unpack!(unpack_high vuplhh vector_unsigned_short u16x4 vector_unsigned_int 4);
2563    impl_vec_unpack!(unpack_high vuplhf vector_unsigned_int u32x2 vector_unsigned_long_long 2);
2564
2565    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphb (vector_signed_char) -> vector_signed_short}
2566    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphh (vector_signed_short) -> vector_signed_int}
2567    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphf (vector_signed_int) -> vector_signed_long_long}
2568
2569    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhb (vector_unsigned_char) -> vector_unsigned_short}
2570    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhh (vector_unsigned_short) -> vector_unsigned_int}
2571    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhf (vector_unsigned_int) -> vector_unsigned_long_long}
2572
2573    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhb (vector_bool_char) -> vector_bool_short}
2574    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhh (vector_bool_short) -> vector_bool_int}
2575    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhf (vector_bool_int) -> vector_bool_long_long}
2576
2577    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2578    pub trait VectorUnpackl {
2579        type Result;
2580        unsafe fn vec_unpackl(self) -> Self::Result;
2581    }
2582
2583    // FIXME(llvm): a shuffle + simd_as does not currently optimize into a single instruction like
2584    // unpachk above. Tracked in https://github.com/llvm/llvm-project/issues/129576.
2585
2586    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplb (vector_signed_char) -> vector_signed_short}
2587    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplhw (vector_signed_short) -> vector_signed_int}
2588    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplf (vector_signed_int) -> vector_signed_long_long}
2589
2590    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllb (vector_unsigned_char) -> vector_unsigned_short}
2591    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllh (vector_unsigned_short) -> vector_unsigned_int}
2592    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllf (vector_unsigned_int) -> vector_unsigned_long_long}
2593
2594    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllb (vector_bool_char) -> vector_bool_short}
2595    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllh (vector_bool_short) -> vector_bool_int}
2596    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllf (vector_bool_int) -> vector_bool_long_long}
2597
2598    test_impl! { vec_vavgb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vavgb, vavgb ] }
2599    test_impl! { vec_vavgh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vavgh, vavgh ] }
2600    test_impl! { vec_vavgf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vavgf, vavgf ] }
2601    test_impl! { vec_vavgg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [ vavgg, vavgg ] }
2602
2603    test_impl! { vec_vavglb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vavglb, vavglb ] }
2604    test_impl! { vec_vavglh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vavglh, vavglh ] }
2605    test_impl! { vec_vavglf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vavglf, vavglf ] }
2606    test_impl! { vec_vavglg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [ vavglg, vavglg ] }
2607
2608    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2609    pub trait VectorAvg<Other> {
2610        type Result;
2611        unsafe fn vec_avg(self, b: Other) -> Self::Result;
2612    }
2613
2614    impl_vec_trait! { [VectorAvg vec_avg] 2 (vec_vavglb, vec_vavgb, vec_vavglh, vec_vavgh, vec_vavglf, vec_vavgf, vec_vavglg, vec_vavgg) }
2615
2616    macro_rules! impl_mul {
2617        ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty) -> $r:ty) => {
2618            #[unstable(feature = "stdarch_s390x", issue = "135681")]
2619            impl $Trait<$r> for $a {
2620                #[inline]
2621                #[target_feature(enable = "vector")]
2622                unsafe fn $m(self, b: $b) -> $r {
2623                    $fun(transmute(self), transmute(b))
2624                }
2625            }
2626        };
2627        ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty, $c:ty) -> $r:ty) => {
2628            #[unstable(feature = "stdarch_s390x", issue = "135681")]
2629            impl $Trait for $a {
2630                type Result = $r;
2631                #[inline]
2632                #[target_feature(enable = "vector")]
2633                unsafe fn $m(self, b: $b, c: $c) -> $r {
2634                    $fun(self, b, c)
2635                }
2636            }
2637        };
2638    }
2639
2640    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2641    pub trait VectorMule<Result> {
2642        unsafe fn vec_mule(self, b: Self) -> Result;
2643    }
2644
2645    // FIXME(llvm) sadly this does not yet work https://github.com/llvm/llvm-project/issues/129705
2646    //    #[target_feature(enable = "vector")]
2647    //    #[cfg_attr(test, assert_instr(vmleh))]
2648    //    unsafe fn vec_vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int {
2649    //        let even_a: vector_unsigned_int = simd_as(simd_shuffle::<_, _, u16x4>(
2650    //            a,
2651    //            a,
2652    //            const { ShuffleMask([0, 2, 4, 6]) },
2653    //        ));
2654    //
2655    //        let even_b: vector_unsigned_int = simd_as(simd_shuffle::<_, _, u16x4>(
2656    //            b,
2657    //            b,
2658    //            const { ShuffleMask([0, 2, 4, 6]) },
2659    //        ));
2660    //
2661    //        simd_mul(even_a, even_b)
2662    //    }
2663
2664    test_impl! { vec_vmeb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short [ vmeb, vmeb ] }
2665    test_impl! { vec_vmeh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int[ vmeh, vmeh ] }
2666    test_impl! { vec_vmef(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long [ vmef, vmef ] }
2667
2668    test_impl! { vec_vmleb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vmleb, vmleb ] }
2669    test_impl! { vec_vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vmleh, vmleh ] }
2670    test_impl! { vec_vmlef(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vmlef, vmlef ] }
2671
2672    impl_mul!([VectorMule vec_mule] vec_vmeb (vector_signed_char, vector_signed_char) -> vector_signed_short );
2673    impl_mul!([VectorMule vec_mule] vec_vmeh (vector_signed_short, vector_signed_short) -> vector_signed_int);
2674    impl_mul!([VectorMule vec_mule] vec_vmef (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
2675
2676    impl_mul!([VectorMule vec_mule] vec_vmleb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2677    impl_mul!([VectorMule vec_mule] vec_vmleh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2678    impl_mul!([VectorMule vec_mule] vec_vmlef (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2679
2680    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2681    pub trait VectorMulo<Result> {
2682        unsafe fn vec_mulo(self, b: Self) -> Result;
2683    }
2684
2685    test_impl! { vec_vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short [ vmob, vmob ] }
2686    test_impl! { vec_vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int[ vmoh, vmoh ] }
2687    test_impl! { vec_vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long [ vmof, vmof ] }
2688
2689    test_impl! { vec_vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vmlob, vmlob ] }
2690    test_impl! { vec_vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vmloh, vmloh ] }
2691    test_impl! { vec_vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vmlof, vmlof ] }
2692
2693    impl_mul!([VectorMulo vec_mulo] vec_vmob (vector_signed_char, vector_signed_char) -> vector_signed_short );
2694    impl_mul!([VectorMulo vec_mulo] vec_vmoh (vector_signed_short, vector_signed_short) -> vector_signed_int);
2695    impl_mul!([VectorMulo vec_mulo] vec_vmof (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
2696
2697    impl_mul!([VectorMulo vec_mulo] vec_vmlob (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2698    impl_mul!([VectorMulo vec_mulo] vec_vmloh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2699    impl_mul!([VectorMulo vec_mulo] vec_vmlof (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2700
2701    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2702    pub trait VectorMulh<Result> {
2703        unsafe fn vec_mulh(self, b: Self) -> Result;
2704    }
2705
2706    test_impl! { vec_vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vmhb, vmhb ] }
2707    test_impl! { vec_vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vmhh, vmhh ] }
2708    test_impl! { vec_vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vmhf, vmhf ] }
2709
2710    test_impl! { vec_vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vmlhb, vmlhb ] }
2711    test_impl! { vec_vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vmlhh, vmlhh ] }
2712    test_impl! { vec_vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vmlhf, vmlhf ] }
2713
2714    impl_mul!([VectorMulh vec_mulh] vec_vmhb (vector_signed_char, vector_signed_char) -> vector_signed_char);
2715    impl_mul!([VectorMulh vec_mulh] vec_vmhh (vector_signed_short, vector_signed_short) -> vector_signed_short);
2716    impl_mul!([VectorMulh vec_mulh] vec_vmhf (vector_signed_int, vector_signed_int) -> vector_signed_int);
2717
2718    impl_mul!([VectorMulh vec_mulh] vec_vmlhb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char);
2719    impl_mul!([VectorMulh vec_mulh] vec_vmlhh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2720    impl_mul!([VectorMulh vec_mulh] vec_vmlhf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int);
2721
2722    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2723    pub trait VectorMeadd {
2724        type Result;
2725        unsafe fn vec_meadd(self, b: Self, c: Self::Result) -> Self::Result;
2726    }
2727
2728    test_impl! { vec_vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaeb, vmaeb ] }
2729    test_impl! { vec_vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaeh, vmaeh ] }
2730    test_impl! { vec_vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaef, vmaef ] }
2731
2732    test_impl! { vec_vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmaleb, vmaleb ] }
2733    test_impl! { vec_vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaleh, vmaleh ] }
2734    test_impl! { vec_vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalef, vmalef ] }
2735
2736    impl_mul!([VectorMeadd vec_meadd] vec_vmaeb (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short );
2737    impl_mul!([VectorMeadd vec_meadd] vec_vmaeh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int);
2738    impl_mul!([VectorMeadd vec_meadd] vec_vmaef (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long );
2739
2740    impl_mul!([VectorMeadd vec_meadd] vec_vmaleb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short );
2741    impl_mul!([VectorMeadd vec_meadd] vec_vmaleh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int);
2742    impl_mul!([VectorMeadd vec_meadd] vec_vmalef (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long );
2743
2744    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2745    pub trait VectorMoadd {
2746        type Result;
2747        unsafe fn vec_moadd(self, b: Self, c: Self::Result) -> Self::Result;
2748    }
2749
2750    test_impl! { vec_vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaob, vmaob ] }
2751    test_impl! { vec_vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaoh, vmaoh ] }
2752    test_impl! { vec_vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaof, vmaof ] }
2753
2754    test_impl! { vec_vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmalob, vmalob ] }
2755    test_impl! { vec_vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaloh, vmaloh ] }
2756    test_impl! { vec_vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalof, vmalof ] }
2757
2758    impl_mul!([VectorMoadd vec_moadd] vec_vmaob (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short );
2759    impl_mul!([VectorMoadd vec_moadd] vec_vmaoh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int);
2760    impl_mul!([VectorMoadd vec_moadd] vec_vmaof (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long );
2761
2762    impl_mul!([VectorMoadd vec_moadd] vec_vmalob (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short );
2763    impl_mul!([VectorMoadd vec_moadd] vec_vmaloh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int);
2764    impl_mul!([VectorMoadd vec_moadd] vec_vmalof (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long );
2765
2766    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2767    pub trait VectorMhadd {
2768        type Result;
2769        unsafe fn vec_mhadd(self, b: Self, c: Self::Result) -> Self::Result;
2770    }
2771
2772    test_impl! { vec_vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [ vmahb, vmahb ] }
2773    test_impl! { vec_vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[ vmahh, vmahh ] }
2774    test_impl! { vec_vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [ vmahf, vmahf ] }
2775
2776    test_impl! { vec_vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [ vmalhb, vmalhb ] }
2777    test_impl! { vec_vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[ vmalhh, vmalhh ] }
2778    test_impl! { vec_vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [ vmalhf, vmalhf ] }
2779
2780    impl_mul!([VectorMhadd vec_mhadd] vec_vmahb (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char );
2781    impl_mul!([VectorMhadd vec_mhadd] vec_vmahh (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short);
2782    impl_mul!([VectorMhadd vec_mhadd] vec_vmahf (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int );
2783
2784    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char );
2785    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2786    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int );
2787
2788    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2789    pub trait VectorMladd {
2790        type Result;
2791        unsafe fn vec_mladd(self, b: Self, c: Self::Result) -> Self::Result;
2792    }
2793
2794    #[inline]
2795    #[target_feature(enable = "vector")]
2796    unsafe fn simd_mladd<T>(a: T, b: T, c: T) -> T {
2797        simd_add(simd_mul(a, b), c)
2798    }
2799
2800    test_impl! { vec_vmal_ib(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [simd_mladd, vmalb ] }
2801    test_impl! { vec_vmal_ih(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[simd_mladd, vmalhw ] }
2802    test_impl! { vec_vmal_if(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [simd_mladd, vmalf ] }
2803
2804    test_impl! { vec_vmal_ub(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [simd_mladd, vmalb ] }
2805    test_impl! { vec_vmal_uh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[simd_mladd, vmalhw ] }
2806    test_impl! { vec_vmal_uf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [simd_mladd, vmalf ] }
2807
2808    impl_mul!([VectorMladd vec_mladd] vec_vmal_ib (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char );
2809    impl_mul!([VectorMladd vec_mladd] vec_vmal_ih (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short);
2810    impl_mul!([VectorMladd vec_mladd] vec_vmal_if (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int );
2811
2812    impl_mul!([VectorMladd vec_mladd] vec_vmal_ub (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char );
2813    impl_mul!([VectorMladd vec_mladd] vec_vmal_uh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2814    impl_mul!([VectorMladd vec_mladd] vec_vmal_uf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int );
2815
2816    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2817    pub trait VectorGfmsum<Result> {
2818        unsafe fn vec_gfmsum(self, b: Self) -> Result;
2819    }
2820
2821    test_impl! { vec_vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vgfmb, vgfmb ] }
2822    test_impl! { vec_vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vgfmh, vgfmh] }
2823    test_impl! { vec_vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vgfmf, vgfmf ] }
2824
2825    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2826    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2827    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2828
2829    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2830    pub trait VectorGfmsumAccum {
2831        type Result;
2832        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result;
2833    }
2834
2835    test_impl! { vec_vgfmab(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vgfmab, vgfmab ] }
2836    test_impl! { vec_vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vgfmah, vgfmah] }
2837    test_impl! { vec_vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vgfmaf, vgfmaf ] }
2838
2839    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2840    impl VectorGfmsumAccum for vector_unsigned_char {
2841        type Result = vector_unsigned_short;
2842        #[inline]
2843        #[target_feature(enable = "vector")]
2844        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2845            vec_vgfmab(self, b, c)
2846        }
2847    }
2848    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2849    impl VectorGfmsumAccum for vector_unsigned_short {
2850        type Result = vector_unsigned_int;
2851        #[inline]
2852        #[target_feature(enable = "vector")]
2853        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2854            vec_vgfmah(self, b, c)
2855        }
2856    }
2857    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2858    impl VectorGfmsumAccum for vector_unsigned_int {
2859        type Result = vector_unsigned_long_long;
2860        #[inline]
2861        #[target_feature(enable = "vector")]
2862        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2863            vec_vgfmaf(self, b, c)
2864        }
2865    }
2866
2867    #[inline]
2868    #[target_feature(enable = "vector")]
2869    #[cfg_attr(test, assert_instr(vgef, D = 3))]
2870    unsafe fn vgef<const D: u32>(
2871        a: vector_unsigned_int,
2872        b: vector_unsigned_int,
2873        c: *const u32,
2874    ) -> vector_unsigned_int {
2875        static_assert_uimm_bits!(D, 2);
2876        let offset: u32 = simd_extract(b, D);
2877        let ptr = c.byte_add(offset as usize);
2878        let value = ptr.read();
2879        simd_insert(a, D, value)
2880    }
2881
2882    #[inline]
2883    #[target_feature(enable = "vector")]
2884    #[cfg_attr(test, assert_instr(vgeg, D = 1))]
2885    unsafe fn vgeg<const D: u32>(
2886        a: vector_unsigned_long_long,
2887        b: vector_unsigned_long_long,
2888        c: *const u64,
2889    ) -> vector_unsigned_long_long {
2890        static_assert_uimm_bits!(D, 1);
2891        let offset: u64 = simd_extract(b, D);
2892        let ptr = c.byte_add(offset as usize);
2893        let value = ptr.read();
2894        simd_insert(a, D, value)
2895    }
2896
2897    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2898    pub trait VectorGatherElement {
2899        type Element;
2900        type Offset;
2901        unsafe fn vec_gather_element<const D: u32>(
2902            self,
2903            b: Self::Offset,
2904            c: *const Self::Element,
2905        ) -> Self;
2906    }
2907
2908    macro_rules! impl_vec_gather_element {
2909        ($($instr:ident $ty:ident)*) => {
2910            $(
2911                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2912                impl VectorGatherElement for $ty {
2913                    type Element = l_t_t!($ty);
2914                    type Offset = t_u!($ty);
2915
2916                    #[inline]
2917                    #[target_feature(enable = "vector")]
2918                    unsafe fn vec_gather_element<const D: u32>(self, b: Self::Offset, c: *const Self::Element) -> Self {
2919                        transmute($instr::<D>(transmute(self), b, c.cast()))
2920                    }
2921                }
2922            )*
2923        }
2924    }
2925
2926    impl_vec_gather_element! {
2927        vgef vector_signed_int
2928        vgef vector_bool_int
2929        vgef vector_unsigned_int
2930
2931        vgeg vector_signed_long_long
2932        vgeg vector_bool_long_long
2933        vgeg vector_unsigned_long_long
2934
2935        vgef vector_float
2936        vgeg vector_double
2937    }
2938
2939    #[inline]
2940    #[target_feature(enable = "vector")]
2941    #[cfg_attr(test, assert_instr(vscef, D = 3))]
2942    unsafe fn vscef<const D: u32>(a: vector_unsigned_int, b: vector_unsigned_int, c: *mut u32) {
2943        static_assert_uimm_bits!(D, 2);
2944        let value = simd_extract(a, D);
2945        let offset: u32 = simd_extract(b, D);
2946        let ptr = c.byte_add(offset as usize);
2947        ptr.write(value);
2948    }
2949
2950    #[inline]
2951    #[target_feature(enable = "vector")]
2952    #[cfg_attr(test, assert_instr(vsceg, D = 1))]
2953    unsafe fn vsceg<const D: u32>(
2954        a: vector_unsigned_long_long,
2955        b: vector_unsigned_long_long,
2956        c: *mut u64,
2957    ) {
2958        static_assert_uimm_bits!(D, 1);
2959        let value = simd_extract(a, D);
2960        let offset: u64 = simd_extract(b, D);
2961        let ptr = c.byte_add(offset as usize);
2962        ptr.write(value);
2963    }
2964
2965    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2966    pub trait VectorScatterElement {
2967        type Element;
2968        type Offset;
2969        unsafe fn vec_scatter_element<const D: u32>(self, b: Self::Offset, c: *mut Self::Element);
2970    }
2971
2972    macro_rules! impl_vec_scatter_element {
2973        ($($instr:ident $ty:ident)*) => {
2974            $(
2975                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2976                impl VectorScatterElement for $ty {
2977                    type Element = l_t_t!($ty);
2978                    type Offset = t_u!($ty);
2979
2980                    #[inline]
2981                    #[target_feature(enable = "vector")]
2982                    unsafe fn vec_scatter_element<const D: u32>(self, b: Self::Offset, c: *mut Self::Element) {
2983                        $instr::<D>(transmute(self), b, c.cast())
2984                    }
2985                }
2986            )*
2987        }
2988    }
2989
2990    impl_vec_scatter_element! {
2991        vscef vector_signed_int
2992        vscef vector_bool_int
2993        vscef vector_unsigned_int
2994
2995        vsceg vector_signed_long_long
2996        vsceg vector_bool_long_long
2997        vsceg vector_unsigned_long_long
2998
2999        vscef vector_float
3000        vsceg vector_double
3001    }
3002
3003    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3004    pub trait VectorSel<Mask>: Sized {
3005        unsafe fn vec_sel(self, b: Self, c: Mask) -> Self;
3006    }
3007
3008    macro_rules! impl_vec_sel {
3009        ($($ty:ident)*) => {
3010            $(
3011                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3012                impl VectorSel<t_u!($ty)> for $ty {
3013                    #[inline]
3014                    #[target_feature(enable = "vector")]
3015                    unsafe fn vec_sel(self, b: Self, c: t_u!($ty)) -> Self {
3016                        let b = simd_and(transmute(b), c);
3017                        let a = simd_and(transmute(self), simd_xor(c, transmute(vector_signed_char([!0; 16]))));
3018                        transmute(simd_or(a, b))
3019                    }
3020                }
3021
3022                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3023                impl VectorSel<t_b!($ty)> for $ty {
3024                    #[inline]
3025                    #[target_feature(enable = "vector")]
3026                    unsafe fn vec_sel(self, b: Self, c: t_b!($ty)) -> Self {
3027                        // defer to the implementation with an unsigned mask
3028                        self.vec_sel(b, transmute::<_, t_u!($ty)>(c))
3029                    }
3030                }
3031            )*
3032        }
3033    }
3034
3035    impl_vec_sel! {
3036        vector_signed_char
3037        vector_signed_short
3038        vector_signed_int
3039        vector_signed_long_long
3040
3041        vector_unsigned_char
3042        vector_unsigned_short
3043        vector_unsigned_int
3044        vector_unsigned_long_long
3045
3046        vector_bool_char
3047        vector_bool_short
3048        vector_bool_int
3049        vector_bool_long_long
3050
3051        vector_float
3052        vector_double
3053    }
3054
3055    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3056    pub trait VectorFpTestDataClass {
3057        type Result;
3058        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32);
3059    }
3060
3061    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3062    impl VectorFpTestDataClass for vector_float {
3063        type Result = vector_bool_int;
3064
3065        #[inline]
3066        #[target_feature(enable = "vector")]
3067        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32) {
3068            let PackedTuple { x, y } = vftcisb(self, CLASS);
3069            (x, y)
3070        }
3071    }
3072
3073    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3074    impl VectorFpTestDataClass for vector_double {
3075        type Result = vector_bool_long_long;
3076
3077        #[inline]
3078        #[target_feature(enable = "vector")]
3079        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32) {
3080            let PackedTuple { x, y } = vftcidb(self, CLASS);
3081            (x, y)
3082        }
3083    }
3084
3085    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3086    pub trait VectorCompare {
3087        unsafe fn vec_all_lt(self, other: Self) -> i32;
3088        unsafe fn vec_all_le(self, other: Self) -> i32;
3089        unsafe fn vec_all_gt(self, other: Self) -> i32;
3090        unsafe fn vec_all_ge(self, other: Self) -> i32;
3091    }
3092
3093    // NOTE: this implementation is currently non-optimal, but it does work for floats even with
3094    // only `vector` enabled.
3095    //
3096    // - https://github.com/llvm/llvm-project/issues/129434
3097    // - https://github.com/llvm/llvm-project/issues/130424
3098    macro_rules! impl_vec_compare {
3099        ($($ty:ident)*) => {
3100            $(
3101                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3102                impl VectorCompare for $ty {
3103                    #[inline]
3104                    #[target_feature(enable = "vector")]
3105                    unsafe fn vec_all_lt(self, other: Self) -> i32 {
3106                        simd_reduce_all(simd_lt::<_, t_b!($ty)>(self, other)) as i32
3107                    }
3108                    #[inline]
3109                    #[target_feature(enable = "vector")]
3110                    unsafe fn vec_all_le(self, other: Self) -> i32 {
3111                        simd_reduce_all(simd_le::<_, t_b!($ty)>(self, other)) as i32
3112                    }
3113                    #[inline]
3114                    #[target_feature(enable = "vector")]
3115                    unsafe fn vec_all_gt(self, other: Self) -> i32 {
3116                        simd_reduce_all(simd_gt::<_, t_b!($ty)>(self, other)) as i32
3117                    }
3118                    #[inline]
3119                    #[target_feature(enable = "vector")]
3120                    unsafe fn vec_all_ge(self, other: Self) -> i32 {
3121                        simd_reduce_all(simd_ge::<_, t_b!($ty)>(self, other)) as i32
3122                    }
3123                }
3124            )*
3125        }
3126    }
3127
3128    impl_vec_compare! {
3129        vector_signed_char
3130        vector_unsigned_char
3131
3132        vector_signed_short
3133        vector_unsigned_short
3134
3135        vector_signed_int
3136        vector_unsigned_int
3137        vector_float
3138
3139        vector_signed_long_long
3140        vector_unsigned_long_long
3141        vector_double
3142    }
3143
3144    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3145    pub trait VectorTestMask {
3146        type Mask;
3147        unsafe fn vec_test_mask(self, other: Self::Mask) -> i32;
3148    }
3149
3150    macro_rules! impl_vec_test_mask {
3151        ($($instr:ident $ty:ident)*) => {
3152            $(
3153                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3154                impl VectorTestMask for $ty {
3155                    type Mask = t_u!($ty);
3156
3157                    #[inline]
3158                    #[target_feature(enable = "vector")]
3159                    unsafe fn vec_test_mask(self, other: Self::Mask) -> i32 {
3160                        vtm(transmute(self), transmute(other))
3161                    }
3162                }
3163            )*
3164        }
3165    }
3166
3167    impl_vec_test_mask! {
3168        vector_signed_char
3169        vector_signed_short
3170        vector_signed_int
3171        vector_signed_long_long
3172
3173        vector_unsigned_char
3174        vector_unsigned_short
3175        vector_unsigned_int
3176        vector_unsigned_long_long
3177
3178        vector_float
3179        vector_double
3180    }
3181
3182    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3183    pub trait VectorSearchString {
3184        unsafe fn vec_search_string_cc(
3185            self,
3186            b: Self,
3187            c: vector_unsigned_char,
3188        ) -> (vector_unsigned_char, i32);
3189
3190        unsafe fn vec_search_string_until_zero_cc(
3191            self,
3192            b: Self,
3193            c: vector_unsigned_char,
3194        ) -> (vector_unsigned_char, i32);
3195    }
3196
3197    macro_rules! impl_vec_search_string{
3198        ($($intr_s:ident $intr_sz:ident $ty:ident)*) => {
3199            $(
3200                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3201                impl VectorSearchString for $ty {
3202                    #[inline]
3203                    #[target_feature(enable = "vector-enhancements-2")]
3204                    unsafe fn vec_search_string_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) {
3205                        let PackedTuple { x,y } = $intr_s(transmute(self), transmute(b), c);
3206                        (x, y)
3207                    }
3208
3209                    #[inline]
3210                    #[target_feature(enable = "vector-enhancements-2")]
3211                    unsafe fn vec_search_string_until_zero_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) {
3212                        let PackedTuple { x,y } = $intr_sz(transmute(self), transmute(b), c);
3213                        (x, y)
3214                    }
3215                }
3216
3217            )*
3218        }
3219    }
3220
3221    impl_vec_search_string! {
3222        vstrsb vstrszb vector_signed_char
3223        vstrsb vstrszb vector_bool_char
3224        vstrsb vstrszb vector_unsigned_char
3225
3226        vstrsh vstrszh vector_signed_short
3227        vstrsh vstrszh vector_bool_short
3228        vstrsh vstrszh vector_unsigned_short
3229
3230        vstrsf vstrszf vector_signed_int
3231        vstrsf vstrszf vector_bool_int
3232        vstrsf vstrszf vector_unsigned_int
3233    }
3234
3235    #[inline]
3236    #[target_feature(enable = "vector")]
3237    #[cfg_attr(test, assert_instr(vcdgb))]
3238    pub unsafe fn vcdgb(a: vector_signed_long_long) -> vector_double {
3239        simd_as(a)
3240    }
3241
3242    #[inline]
3243    #[target_feature(enable = "vector")]
3244    #[cfg_attr(test, assert_instr(vcdlgb))]
3245    pub unsafe fn vcdlgb(a: vector_unsigned_long_long) -> vector_double {
3246        simd_as(a)
3247    }
3248
3249    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3250    pub trait VectorDouble {
3251        unsafe fn vec_double(self) -> vector_double;
3252    }
3253
3254    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3255    impl VectorDouble for vector_signed_long_long {
3256        #[inline]
3257        #[target_feature(enable = "vector")]
3258        unsafe fn vec_double(self) -> vector_double {
3259            vcdgb(self)
3260        }
3261    }
3262
3263    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3264    impl VectorDouble for vector_unsigned_long_long {
3265        #[inline]
3266        #[target_feature(enable = "vector")]
3267        unsafe fn vec_double(self) -> vector_double {
3268            vcdlgb(self)
3269        }
3270    }
3271
3272    #[inline]
3273    #[target_feature(enable = "vector")]
3274    #[cfg_attr(
3275        all(test, target_feature = "vector-enhancements-2"),
3276        assert_instr(vcefb)
3277    )]
3278    pub unsafe fn vcefb(a: vector_signed_int) -> vector_float {
3279        simd_as(a)
3280    }
3281
3282    #[inline]
3283    #[target_feature(enable = "vector")]
3284    #[cfg_attr(
3285        all(test, target_feature = "vector-enhancements-2"),
3286        assert_instr(vcelfb)
3287    )]
3288    pub unsafe fn vcelfb(a: vector_unsigned_int) -> vector_float {
3289        simd_as(a)
3290    }
3291
3292    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3293    pub trait VectorFloat {
3294        unsafe fn vec_float(self) -> vector_float;
3295    }
3296
3297    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3298    impl VectorFloat for vector_signed_int {
3299        #[inline]
3300        #[target_feature(enable = "vector")]
3301        unsafe fn vec_float(self) -> vector_float {
3302            vcefb(self)
3303        }
3304    }
3305
3306    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3307    impl VectorFloat for vector_unsigned_int {
3308        #[inline]
3309        #[target_feature(enable = "vector")]
3310        unsafe fn vec_float(self) -> vector_float {
3311            vcelfb(self)
3312        }
3313    }
3314
3315    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3316    pub trait VectorExtendSigned64 {
3317        unsafe fn vec_extend_s64(self) -> vector_signed_long_long;
3318    }
3319
3320    #[inline]
3321    #[target_feature(enable = "vector")]
3322    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
3323    // #[cfg_attr(test, assert_instr(vsegb))]
3324    pub unsafe fn vsegb(a: vector_signed_char) -> vector_signed_long_long {
3325        simd_as(simd_shuffle::<_, _, i8x2>(
3326            a,
3327            a,
3328            const { u32x2::from_array([7, 15]) },
3329        ))
3330    }
3331
3332    #[inline]
3333    #[target_feature(enable = "vector")]
3334    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
3335    // #[cfg_attr(test, assert_instr(vsegh))]
3336    pub unsafe fn vsegh(a: vector_signed_short) -> vector_signed_long_long {
3337        simd_as(simd_shuffle::<_, _, i16x2>(
3338            a,
3339            a,
3340            const { u32x2::from_array([3, 7]) },
3341        ))
3342    }
3343
3344    #[inline]
3345    #[target_feature(enable = "vector")]
3346    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
3347    // #[cfg_attr(test, assert_instr(vsegf))]
3348    pub unsafe fn vsegf(a: vector_signed_int) -> vector_signed_long_long {
3349        simd_as(simd_shuffle::<_, _, i32x2>(
3350            a,
3351            a,
3352            const { u32x2::from_array([1, 3]) },
3353        ))
3354    }
3355
3356    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3357    impl VectorExtendSigned64 for vector_signed_char {
3358        #[inline]
3359        #[target_feature(enable = "vector")]
3360        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3361            vsegb(self)
3362        }
3363    }
3364    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3365    impl VectorExtendSigned64 for vector_signed_short {
3366        #[inline]
3367        #[target_feature(enable = "vector")]
3368        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3369            vsegh(self)
3370        }
3371    }
3372    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3373    impl VectorExtendSigned64 for vector_signed_int {
3374        #[inline]
3375        #[target_feature(enable = "vector")]
3376        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3377            vsegf(self)
3378        }
3379    }
3380
3381    // NOTE: VectorSigned and VectorUnsigned make strong safety assumptions around floats.
3382    // This is what C provides, but even IBM does not clearly document these constraints.
3383    //
3384    // https://doc.rust-lang.org/std/intrinsics/simd/fn.simd_cast.html
3385
3386    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3387    pub trait VectorSigned {
3388        type Result;
3389        unsafe fn vec_signed(self) -> Self::Result;
3390    }
3391
3392    test_impl! { vcgsb (a: vector_float) -> vector_signed_int [simd_cast, "vector-enhancements-2" vcgsb] }
3393    test_impl! { vcgdb (a: vector_double) -> vector_signed_long_long [simd_cast, vcgdb] }
3394
3395    impl_vec_trait! { [VectorSigned vec_signed] vcgsb (vector_float) -> vector_signed_int }
3396    impl_vec_trait! { [VectorSigned vec_signed] vcgdb (vector_double) -> vector_signed_long_long }
3397
3398    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3399    pub trait VectorUnsigned {
3400        type Result;
3401        unsafe fn vec_unsigned(self) -> Self::Result;
3402    }
3403
3404    test_impl! { vclgsb (a: vector_float) -> vector_unsigned_int [simd_cast, "vector-enhancements-2" vclgsb] }
3405    test_impl! { vclgdb (a: vector_double) -> vector_unsigned_long_long [simd_cast, vclgdb] }
3406
3407    impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgsb (vector_float) -> vector_unsigned_int }
3408    impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgdb (vector_double) -> vector_unsigned_long_long }
3409
3410    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3411    pub trait VectorCopyUntilZero {
3412        unsafe fn vec_cp_until_zero(self) -> Self;
3413    }
3414
3415    test_impl! { vec_vistrb (a: vector_unsigned_char) -> vector_unsigned_char [vistrb, vistrb] }
3416    test_impl! { vec_vistrh (a: vector_unsigned_short) -> vector_unsigned_short [vistrh, vistrh] }
3417    test_impl! { vec_vistrf (a: vector_unsigned_int) -> vector_unsigned_int [vistrf, vistrf] }
3418
3419    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_signed_char) }
3420    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_bool_char) }
3421    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_unsigned_char) }
3422
3423    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_signed_short) }
3424    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_bool_short) }
3425    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_unsigned_short) }
3426
3427    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_signed_int) }
3428    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_bool_int) }
3429    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_unsigned_int) }
3430
3431    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3432    pub trait VectorCopyUntilZeroCC: Sized {
3433        unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32);
3434    }
3435
3436    test_impl! { vec_vistrbs (a: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32> [vistrbs, vistrbs] }
3437    test_impl! { vec_vistrhs (a: vector_unsigned_short) -> PackedTuple<vector_unsigned_short, i32> [vistrhs, vistrhs] }
3438    test_impl! { vec_vistrfs (a: vector_unsigned_int) -> PackedTuple<vector_unsigned_int, i32> [vistrfs, vistrfs] }
3439
3440    macro_rules! impl_vec_copy_until_zero_cc {
3441        ($($intr:ident $ty:ident)*) => {
3442            $(
3443                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3444                impl VectorCopyUntilZeroCC for $ty {
3445                    #[inline]
3446                    #[target_feature(enable = "vector")]
3447                    unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32) {
3448                        let PackedTuple { x,y } = $intr(transmute(self));
3449                        (transmute(x), y)
3450                    }
3451                }
3452
3453            )*
3454        }
3455    }
3456
3457    impl_vec_copy_until_zero_cc! {
3458        vec_vistrbs vector_signed_char
3459        vec_vistrbs vector_bool_char
3460        vec_vistrbs vector_unsigned_char
3461
3462        vec_vistrhs vector_signed_short
3463        vec_vistrhs vector_bool_short
3464        vec_vistrhs vector_unsigned_short
3465
3466        vec_vistrfs vector_signed_int
3467        vec_vistrfs vector_bool_int
3468        vec_vistrfs vector_unsigned_int
3469    }
3470
3471    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3472    pub trait VectorSrdb {
3473        unsafe fn vec_srdb<const C: u32>(self, b: Self) -> Self;
3474    }
3475
3476    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3477    pub trait VectorSld {
3478        unsafe fn vec_sld<const C: u32>(self, b: Self) -> Self;
3479
3480        unsafe fn vec_sldw<const C: u32>(self, b: Self) -> Self;
3481
3482        unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self;
3483    }
3484
3485    // FIXME(llvm) https://github.com/llvm/llvm-project/issues/129955
3486    // ideally we could implement this in terms of llvm.fshl.i128
3487    // #[link_name = "llvm.fshl.i128"] fn fshl_i128(a: u128, b: u128, c: u128) -> u128;
3488    // transmute(fshl_i128(transmute(a), transmute(b), const { C * 8 } ))
3489
3490    macro_rules! impl_vec_sld {
3491        ($($ty:ident)*) => {
3492            $(
3493                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3494                impl VectorSld for $ty {
3495                    #[inline]
3496                    #[target_feature(enable = "vector")]
3497                    unsafe fn vec_sld<const C: u32>(self, b: Self) -> Self {
3498                        static_assert_uimm_bits!(C, 4);
3499                        transmute(vsldb(transmute(self), transmute(b), C))
3500                    }
3501
3502                    #[inline]
3503                    #[target_feature(enable = "vector")]
3504                    unsafe fn vec_sldw<const C: u32>(self, b: Self) -> Self {
3505                        static_assert_uimm_bits!(C, 2);
3506                        transmute(vsldb(transmute(self), transmute(b), const { 4 * C }))
3507                    }
3508
3509                    #[inline]
3510                    #[target_feature(enable = "vector-enhancements-2")]
3511                    unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self {
3512                        static_assert_uimm_bits!(C, 3);
3513                        transmute(vsld(transmute(self), transmute(b), C))
3514                    }
3515                }
3516
3517                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3518                impl VectorSrdb for $ty {
3519                    #[inline]
3520                    #[target_feature(enable = "vector-enhancements-2")]
3521                    unsafe fn vec_srdb<const C: u32>(self, b: Self) -> Self {
3522                        static_assert_uimm_bits!(C, 3);
3523                        transmute(vsrd(transmute(self), transmute(b), C))
3524                    }
3525                }
3526            )*
3527        }
3528    }
3529
3530    impl_vec_sld! {
3531        vector_signed_char
3532        vector_bool_char
3533        vector_unsigned_char
3534
3535        vector_signed_short
3536        vector_bool_short
3537        vector_unsigned_short
3538
3539        vector_signed_int
3540        vector_bool_int
3541        vector_unsigned_int
3542
3543        vector_signed_long_long
3544        vector_bool_long_long
3545        vector_unsigned_long_long
3546
3547        vector_float
3548        vector_double
3549    }
3550
3551    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3552    pub trait VectorCompareRange: Sized {
3553        type Result;
3554
3555        unsafe fn vstrc<const IMM: u32>(self, b: Self, c: Self) -> Self::Result;
3556        unsafe fn vstrcz<const IMM: u32>(self, b: Self, c: Self) -> Self::Result;
3557        unsafe fn vstrcs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32);
3558        unsafe fn vstrczs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32);
3559    }
3560
3561    const fn validate_compare_range_imm(imm: u32) {
3562        if !matches!(imm, 0 | 4 | 8 | 12) {
3563            panic!("IMM needs to be one of 0, 4, 8, 12");
3564        }
3565    }
3566
3567    macro_rules! impl_compare_range {
3568        ($($ty:ident $vstrc:ident $vstrcs:ident $vstrcz:ident $vstrczs:ident)*) => {
3569            $(
3570                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3571                impl VectorCompareRange for $ty {
3572                    type Result = t_b!($ty);
3573
3574                    #[inline]
3575                    #[target_feature(enable = "vector")]
3576                    unsafe fn vstrc<const IMM: u32>(self, b: Self, c: Self) -> Self::Result {
3577                        const { validate_compare_range_imm };
3578                        $vstrc(self, b, c, IMM)
3579                    }
3580
3581                    #[inline]
3582                    #[target_feature(enable = "vector")]
3583                    unsafe fn vstrcz<const IMM: u32>(self, b: Self, c: Self) -> Self::Result {
3584                        const { validate_compare_range_imm };
3585                        $vstrcz(self, b, c, IMM)
3586                    }
3587
3588                    #[inline]
3589                    #[target_feature(enable = "vector")]
3590                    unsafe fn vstrcs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32) {
3591                        const { validate_compare_range_imm };
3592                        let PackedTuple { x, y } = $vstrcs(self, b, c, IMM);
3593                        (x,y)
3594                    }
3595
3596                    #[inline]
3597                    #[target_feature(enable = "vector")]
3598                    unsafe fn vstrczs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32) {
3599                        const { validate_compare_range_imm };
3600                        let PackedTuple { x, y } = $vstrczs(self, b, c, IMM);
3601                        (x,y)
3602                    }
3603                }
3604            )*
3605        }
3606    }
3607
3608    impl_compare_range! {
3609        vector_unsigned_char    vstrcb vstrcbs vstrczb vstrczbs
3610        vector_unsigned_short   vstrch vstrchs vstrczh vstrczhs
3611        vector_unsigned_int     vstrcf vstrcfs vstrczf vstrczfs
3612    }
3613
3614    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3615    pub trait VectorComparePredicate: Sized {
3616        type Result;
3617
3618        #[inline]
3619        #[target_feature(enable = "vector")]
3620        unsafe fn vec_cmpgt(self, other: Self) -> Self::Result {
3621            simd_gt(self, other)
3622        }
3623
3624        #[inline]
3625        #[target_feature(enable = "vector")]
3626        unsafe fn vec_cmpge(self, other: Self) -> Self::Result {
3627            simd_ge(self, other)
3628        }
3629
3630        #[inline]
3631        #[target_feature(enable = "vector")]
3632        unsafe fn vec_cmplt(self, other: Self) -> Self::Result {
3633            simd_lt(self, other)
3634        }
3635
3636        #[inline]
3637        #[target_feature(enable = "vector")]
3638        unsafe fn vec_cmple(self, other: Self) -> Self::Result {
3639            simd_le(self, other)
3640        }
3641    }
3642
3643    macro_rules! impl_compare_predicate {
3644        ($($ty:ident)*) => {
3645            $(
3646                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3647                impl VectorComparePredicate for $ty {
3648                    type Result = t_b!($ty);
3649                }
3650            )*
3651        }
3652    }
3653
3654    impl_compare_predicate! {
3655        vector_signed_char
3656        vector_unsigned_char
3657
3658        vector_signed_short
3659        vector_unsigned_short
3660
3661        vector_signed_int
3662        vector_unsigned_int
3663        vector_float
3664
3665        vector_signed_long_long
3666        vector_unsigned_long_long
3667        vector_double
3668    }
3669
3670    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3671    pub trait VectorEquality: Sized {
3672        type Result;
3673
3674        #[inline]
3675        #[target_feature(enable = "vector")]
3676        unsafe fn vec_cmpeq(self, other: Self) -> Self::Result {
3677            simd_eq(self, other)
3678        }
3679
3680        #[inline]
3681        #[target_feature(enable = "vector")]
3682        unsafe fn vec_cmpne(self, other: Self) -> Self::Result {
3683            simd_ne(self, other)
3684        }
3685    }
3686
3687    macro_rules! impl_compare_equality {
3688        ($($ty:ident)*) => {
3689            $(
3690                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3691                impl VectorEquality for $ty {
3692                    type Result = t_b!($ty);
3693                }
3694            )*
3695        }
3696    }
3697
3698    impl_compare_equality! {
3699        vector_bool_char
3700        vector_signed_char
3701        vector_unsigned_char
3702
3703        vector_bool_short
3704        vector_signed_short
3705        vector_unsigned_short
3706
3707        vector_bool_int
3708        vector_signed_int
3709        vector_unsigned_int
3710        vector_float
3711
3712        vector_bool_long_long
3713        vector_signed_long_long
3714        vector_unsigned_long_long
3715        vector_double
3716    }
3717
3718    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3719    pub trait VectorEqualityIdx: Sized {
3720        type Result;
3721
3722        unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result;
3723        unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result;
3724
3725        unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32);
3726        unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32);
3727
3728        unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result;
3729        unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result;
3730
3731        unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32);
3732        unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32);
3733    }
3734
3735    macro_rules! impl_compare_equality_idx {
3736        ($($ty:ident $ret:ident
3737                $cmpeq:ident $cmpne:ident
3738                $cmpeq_or_0:ident $cmpne_or_0:ident
3739                $cmpeq_cc:ident $cmpne_cc:ident
3740                $cmpeq_or_0_cc:ident $cmpne_or_0_cc:ident
3741        )*) => {
3742            $(
3743                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3744                impl VectorEqualityIdx for $ty {
3745                    type Result = $ret;
3746
3747                    #[inline]
3748                    #[target_feature(enable = "vector")]
3749                    unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result {
3750                        transmute($cmpeq(transmute(self), transmute(other)))
3751                    }
3752
3753                    #[inline]
3754                    #[target_feature(enable = "vector")]
3755                    unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result {
3756                        transmute($cmpne(transmute(self), transmute(other)))
3757                    }
3758
3759                    #[inline]
3760                    #[target_feature(enable = "vector")]
3761                    unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result {
3762                        transmute($cmpeq_or_0(transmute(self), transmute(other)))
3763                    }
3764
3765                    #[inline]
3766                    #[target_feature(enable = "vector")]
3767                    unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result {
3768                        transmute($cmpne_or_0(transmute(self), transmute(other)))
3769                    }
3770
3771                    #[inline]
3772                    #[target_feature(enable = "vector")]
3773                    unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32) {
3774                        let PackedTuple { x, y } = $cmpeq_cc(transmute(self), transmute(other));
3775                        (transmute(x), y)
3776                    }
3777
3778                    #[inline]
3779                    #[target_feature(enable = "vector")]
3780                    unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32) {
3781                        let PackedTuple { x, y } = $cmpne_cc(transmute(self), transmute(other));
3782                        (transmute(x),y)
3783                    }
3784
3785                    #[inline]
3786                    #[target_feature(enable = "vector")]
3787                    unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) {
3788                        let PackedTuple { x, y } = $cmpeq_or_0_cc(transmute(self), transmute(other));
3789                        (transmute(x), y)
3790                    }
3791
3792                    #[inline]
3793                    #[target_feature(enable = "vector")]
3794                    unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) {
3795                        let PackedTuple { x, y } = $cmpne_or_0_cc(transmute(self), transmute(other));
3796                        (transmute(x),y)
3797                    }
3798                }
3799            )*
3800        }
3801    }
3802
3803    impl_compare_equality_idx! {
3804        vector_signed_char vector_signed_char               vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3805        vector_bool_char vector_unsigned_char               vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3806        vector_unsigned_char vector_unsigned_char           vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3807        vector_signed_short vector_signed_short             vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3808        vector_bool_short  vector_unsigned_short            vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3809        vector_unsigned_short vector_unsigned_short         vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3810        vector_signed_int vector_signed_int                 vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3811        vector_bool_int  vector_unsigned_int                vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3812        vector_unsigned_int vector_unsigned_int             vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3813    }
3814
3815    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3816    pub trait VectorExtract {
3817        type ElementType;
3818
3819        unsafe fn vec_extract(a: Self, b: i32) -> Self::ElementType;
3820    }
3821
3822    #[inline]
3823    #[target_feature(enable = "vector")]
3824    #[cfg_attr(test, assert_instr(vlgvb))]
3825    unsafe fn vlgvb(a: vector_unsigned_char, b: i32) -> u8 {
3826        simd_extract_dyn(a, b as u32 % 16)
3827    }
3828
3829    #[inline]
3830    #[target_feature(enable = "vector")]
3831    #[cfg_attr(test, assert_instr(vlgvh))]
3832    unsafe fn vlgvh(a: vector_unsigned_short, b: i32) -> u16 {
3833        simd_extract_dyn(a, b as u32 % 8)
3834    }
3835
3836    #[inline]
3837    #[target_feature(enable = "vector")]
3838    #[cfg_attr(test, assert_instr(vlgvf))]
3839    unsafe fn vlgvf(a: vector_unsigned_int, b: i32) -> u32 {
3840        simd_extract_dyn(a, b as u32 % 4)
3841    }
3842
3843    #[inline]
3844    #[target_feature(enable = "vector")]
3845    #[cfg_attr(test, assert_instr(vlgvg))]
3846    unsafe fn vlgvg(a: vector_unsigned_long_long, b: i32) -> u64 {
3847        simd_extract_dyn(a, b as u32 % 2)
3848    }
3849
3850    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3851    pub trait VectorInsert {
3852        type ElementType;
3853
3854        unsafe fn vec_insert(a: Self::ElementType, b: Self, c: i32) -> Self;
3855    }
3856
3857    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3858    pub trait VectorPromote: Sized {
3859        type ElementType;
3860
3861        unsafe fn vec_promote(a: Self::ElementType, b: i32) -> MaybeUninit<Self>;
3862    }
3863
3864    #[inline]
3865    #[target_feature(enable = "vector")]
3866    #[cfg_attr(test, assert_instr(vlvgb))]
3867    unsafe fn vlvgb(a: u8, b: vector_unsigned_char, c: i32) -> vector_unsigned_char {
3868        simd_insert_dyn(b, c as u32 % 16, a)
3869    }
3870
3871    #[inline]
3872    #[target_feature(enable = "vector")]
3873    #[cfg_attr(test, assert_instr(vlvgh))]
3874    unsafe fn vlvgh(a: u16, b: vector_unsigned_short, c: i32) -> vector_unsigned_short {
3875        simd_insert_dyn(b, c as u32 % 8, a)
3876    }
3877
3878    #[inline]
3879    #[target_feature(enable = "vector")]
3880    #[cfg_attr(test, assert_instr(vlvgf))]
3881    unsafe fn vlvgf(a: u32, b: vector_unsigned_int, c: i32) -> vector_unsigned_int {
3882        simd_insert_dyn(b, c as u32 % 4, a)
3883    }
3884
3885    #[inline]
3886    #[target_feature(enable = "vector")]
3887    #[cfg_attr(test, assert_instr(vlvgg))]
3888    unsafe fn vlvgg(a: u64, b: vector_unsigned_long_long, c: i32) -> vector_unsigned_long_long {
3889        simd_insert_dyn(b, c as u32 % 2, a)
3890    }
3891
3892    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3893    pub trait VectorInsertAndZero {
3894        type ElementType;
3895
3896        unsafe fn vec_insert_and_zero(a: *const Self::ElementType) -> Self;
3897    }
3898
3899    #[inline]
3900    #[target_feature(enable = "vector")]
3901    #[cfg_attr(test, assert_instr(vllezb))]
3902    unsafe fn vllezb(x: *const u8) -> vector_unsigned_char {
3903        vector_unsigned_char([0, 0, 0, 0, 0, 0, 0, *x, 0, 0, 0, 0, 0, 0, 0, 0])
3904    }
3905
3906    #[inline]
3907    #[target_feature(enable = "vector")]
3908    #[cfg_attr(test, assert_instr(vllezh))]
3909    unsafe fn vllezh(x: *const u16) -> vector_unsigned_short {
3910        vector_unsigned_short([0, 0, 0, *x, 0, 0, 0, 0])
3911    }
3912
3913    #[inline]
3914    #[target_feature(enable = "vector")]
3915    #[cfg_attr(test, assert_instr(vllezf))]
3916    unsafe fn vllezf(x: *const u32) -> vector_unsigned_int {
3917        vector_unsigned_int([0, *x, 0, 0])
3918    }
3919
3920    #[inline]
3921    #[target_feature(enable = "vector")]
3922    #[cfg_attr(test, assert_instr(vllezg))]
3923    unsafe fn vllezg(x: *const u64) -> vector_unsigned_long_long {
3924        vector_unsigned_long_long([*x, 0])
3925    }
3926
3927    macro_rules! impl_extract_insert {
3928        ($($ty:ident $extract_intr:ident $insert_intr:ident $insert_and_zero_intr:ident)*) => {
3929            $(
3930                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3931                impl VectorExtract for $ty {
3932                    type ElementType = l_t_t!($ty);
3933
3934                    #[inline]
3935                    #[target_feature(enable = "vector")]
3936                    unsafe fn vec_extract(a: Self, b: i32) -> Self::ElementType {
3937                        transmute($extract_intr(transmute(a), b))
3938                    }
3939                }
3940
3941                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3942                impl VectorInsert for $ty {
3943                    type ElementType = l_t_t!($ty);
3944
3945                    #[inline]
3946                    #[target_feature(enable = "vector")]
3947                    unsafe fn vec_insert(a: Self::ElementType, b: Self, c: i32) -> Self {
3948                        transmute($insert_intr(transmute(a), transmute(b), c))
3949                    }
3950                }
3951
3952                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3953                impl VectorInsertAndZero for $ty {
3954                    type ElementType = l_t_t!($ty);
3955
3956                    #[inline]
3957                    #[target_feature(enable = "vector")]
3958                    unsafe fn vec_insert_and_zero(a: *const Self::ElementType) -> Self {
3959                        transmute($insert_and_zero_intr(a.cast()))
3960                    }
3961                }
3962
3963                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3964                impl VectorPromote for $ty {
3965                    type ElementType = l_t_t!($ty);
3966
3967                    #[inline]
3968                    #[target_feature(enable = "vector")]
3969                    unsafe fn vec_promote(a: Self::ElementType, c: i32) -> MaybeUninit<Self> {
3970                        // Rust does not currently support `MaybeUninit` element types to simd
3971                        // vectors. In C/LLVM that is allowed (using poison values). So rust will
3972                        // use an extra instruction to zero the memory.
3973                        let b = MaybeUninit::<$ty>::zeroed();
3974                        MaybeUninit::new(transmute($insert_intr(transmute(a), transmute(b), c)))
3975                    }
3976                }
3977            )*
3978        }
3979
3980    }
3981
3982    impl_extract_insert! {
3983        vector_signed_char          vlgvb vlvgb vllezb
3984        vector_unsigned_char        vlgvb vlvgb vllezb
3985        vector_signed_short         vlgvh vlvgh vllezh
3986        vector_unsigned_short       vlgvh vlvgh vllezh
3987        vector_signed_int           vlgvf vlvgf vllezf
3988        vector_unsigned_int         vlgvf vlvgf vllezf
3989        vector_signed_long_long     vlgvg vlvgg vllezg
3990        vector_unsigned_long_long   vlgvg vlvgg vllezg
3991        vector_float                vlgvf vlvgf vllezf
3992        vector_double               vlgvg vlvgg vllezg
3993    }
3994}
3995
3996/// Load Count to Block Boundary
3997#[inline]
3998#[target_feature(enable = "vector")]
3999#[unstable(feature = "stdarch_s390x", issue = "135681")]
4000#[cfg_attr(test, assert_instr(lcbb, BLOCK_BOUNDARY = 512))]
4001unsafe fn __lcbb<const BLOCK_BOUNDARY: u16>(ptr: *const u8) -> u32 {
4002    lcbb(ptr, const { validate_block_boundary(BLOCK_BOUNDARY) })
4003}
4004
4005/// Vector Add
4006#[inline]
4007#[target_feature(enable = "vector")]
4008#[unstable(feature = "stdarch_s390x", issue = "135681")]
4009pub unsafe fn vec_add<T: sealed::VectorAdd<U>, U>(a: T, b: U) -> T::Result {
4010    a.vec_add(b)
4011}
4012
4013/// Vector Subtract
4014#[inline]
4015#[target_feature(enable = "vector")]
4016#[unstable(feature = "stdarch_s390x", issue = "135681")]
4017pub unsafe fn vec_sub<T: sealed::VectorSub<U>, U>(a: T, b: U) -> T::Result {
4018    a.vec_sub(b)
4019}
4020
4021/// Vector Multiply
4022///
4023/// ## Purpose
4024/// Compute the products of corresponding elements of two vectors.
4025///
4026/// ## Result value
4027/// Each element of r receives the product of the corresponding elements of a and b.
4028#[inline]
4029#[target_feature(enable = "vector")]
4030#[unstable(feature = "stdarch_s390x", issue = "135681")]
4031pub unsafe fn vec_mul<T: sealed::VectorMul>(a: T, b: T) -> T {
4032    a.vec_mul(b)
4033}
4034
4035/// Vector Count Leading Zeros
4036#[inline]
4037#[target_feature(enable = "vector")]
4038#[unstable(feature = "stdarch_s390x", issue = "135681")]
4039pub unsafe fn vec_cntlz<T: sealed::CountBits>(a: T) -> T::Result {
4040    a.vec_cntlz()
4041}
4042
4043/// Vector Count Trailing Zeros
4044#[inline]
4045#[target_feature(enable = "vector")]
4046#[unstable(feature = "stdarch_s390x", issue = "135681")]
4047pub unsafe fn vec_cnttz<T: sealed::CountBits>(a: T) -> T::Result {
4048    a.vec_cnttz()
4049}
4050
4051/// Vector Population Count
4052///
4053/// Computes the population count (number of set bits) in each element of the input.
4054#[inline]
4055#[target_feature(enable = "vector")]
4056#[unstable(feature = "stdarch_s390x", issue = "135681")]
4057pub unsafe fn vec_popcnt<T: sealed::CountBits>(a: T) -> T::Result {
4058    a.vec_popcnt()
4059}
4060
4061/// Vector Maximum
4062#[inline]
4063#[target_feature(enable = "vector")]
4064#[unstable(feature = "stdarch_s390x", issue = "135681")]
4065pub unsafe fn vec_max<T: sealed::VectorMax<U>, U>(a: T, b: U) -> T::Result {
4066    a.vec_max(b)
4067}
4068
4069/// Vector  Minimum
4070#[inline]
4071#[target_feature(enable = "vector")]
4072#[unstable(feature = "stdarch_s390x", issue = "135681")]
4073pub unsafe fn vec_min<T: sealed::VectorMin<U>, U>(a: T, b: U) -> T::Result {
4074    a.vec_min(b)
4075}
4076
4077/// Vector Absolute
4078#[inline]
4079#[target_feature(enable = "vector")]
4080#[unstable(feature = "stdarch_s390x", issue = "135681")]
4081pub unsafe fn vec_abs<T: sealed::VectorAbs>(a: T) -> T {
4082    a.vec_abs()
4083}
4084
4085/// Vector Negative Absolute
4086#[inline]
4087#[target_feature(enable = "vector")]
4088#[unstable(feature = "stdarch_s390x", issue = "135681")]
4089pub unsafe fn vec_nabs<T: sealed::VectorNabs>(a: T) -> T {
4090    a.vec_nabs()
4091}
4092
4093/// Vector Negative Multiply Add
4094#[inline]
4095#[target_feature(enable = "vector")]
4096#[unstable(feature = "stdarch_s390x", issue = "135681")]
4097pub unsafe fn vec_nmadd<T: sealed::VectorNmadd>(a: T, b: T, c: T) -> T {
4098    a.vec_nmadd(b, c)
4099}
4100
4101/// Vector Negative Multiply Subtract
4102#[inline]
4103#[target_feature(enable = "vector")]
4104#[unstable(feature = "stdarch_s390x", issue = "135681")]
4105pub unsafe fn vec_nmsub<T: sealed::VectorNmsub>(a: T, b: T, c: T) -> T {
4106    a.vec_nmsub(b, c)
4107}
4108
4109/// Vector Square Root
4110#[inline]
4111#[target_feature(enable = "vector")]
4112#[unstable(feature = "stdarch_s390x", issue = "135681")]
4113pub unsafe fn vec_sqrt<T: sealed::VectorSqrt>(a: T) -> T {
4114    a.vec_sqrt()
4115}
4116
4117/// Vector Splat
4118#[inline]
4119#[target_feature(enable = "vector")]
4120#[unstable(feature = "stdarch_s390x", issue = "135681")]
4121pub unsafe fn vec_splat<T: sealed::VectorSplat, const IMM: u32>(a: T) -> T {
4122    a.vec_splat::<IMM>()
4123}
4124
4125/// Vector Splats
4126#[inline]
4127#[target_feature(enable = "vector")]
4128#[unstable(feature = "stdarch_s390x", issue = "135681")]
4129pub unsafe fn vec_splats<T: sealed::VectorSplats<U>, U>(a: T) -> U {
4130    a.vec_splats()
4131}
4132
4133/// Vector AND
4134#[inline]
4135#[target_feature(enable = "vector")]
4136#[unstable(feature = "stdarch_s390x", issue = "135681")]
4137pub unsafe fn vec_and<T: sealed::VectorAnd<U>, U>(a: T, b: U) -> T::Result {
4138    a.vec_and(b)
4139}
4140
4141/// Vector OR
4142#[inline]
4143#[target_feature(enable = "vector")]
4144#[unstable(feature = "stdarch_s390x", issue = "135681")]
4145pub unsafe fn vec_or<T: sealed::VectorOr<U>, U>(a: T, b: U) -> T::Result {
4146    a.vec_or(b)
4147}
4148
4149/// Vector XOR
4150#[inline]
4151#[target_feature(enable = "vector")]
4152#[unstable(feature = "stdarch_s390x", issue = "135681")]
4153pub unsafe fn vec_xor<T: sealed::VectorXor<U>, U>(a: T, b: U) -> T::Result {
4154    a.vec_xor(b)
4155}
4156
4157/// Vector NOR
4158#[inline]
4159#[target_feature(enable = "vector")]
4160#[unstable(feature = "stdarch_s390x", issue = "135681")]
4161pub unsafe fn vec_nor<T: sealed::VectorNor<U>, U>(a: T, b: U) -> T::Result {
4162    a.vec_nor(b)
4163}
4164
4165/// Vector NAND
4166#[inline]
4167#[target_feature(enable = "vector")]
4168#[unstable(feature = "stdarch_s390x", issue = "135681")]
4169pub unsafe fn vec_nand<T: sealed::VectorNand<U>, U>(a: T, b: U) -> T::Result {
4170    a.vec_nand(b)
4171}
4172
4173/// Vector XNOR
4174#[inline]
4175#[target_feature(enable = "vector")]
4176#[unstable(feature = "stdarch_s390x", issue = "135681")]
4177pub unsafe fn vec_eqv<T: sealed::VectorEqv<U>, U>(a: T, b: U) -> T::Result {
4178    a.vec_eqv(b)
4179}
4180
4181/// Vector ANDC
4182#[inline]
4183#[target_feature(enable = "vector")]
4184#[unstable(feature = "stdarch_s390x", issue = "135681")]
4185pub unsafe fn vec_andc<T: sealed::VectorAndc<U>, U>(a: T, b: U) -> T::Result {
4186    a.vec_andc(b)
4187}
4188
4189/// Vector OR with Complement
4190///
4191/// ## Purpose
4192/// Performs a bitwise OR of the first vector with the bitwise-complemented second vector.
4193///
4194/// ## Result value
4195/// r is the bitwise OR of a and the bitwise complement of b.
4196#[inline]
4197#[target_feature(enable = "vector")]
4198#[unstable(feature = "stdarch_s390x", issue = "135681")]
4199pub unsafe fn vec_orc<T: sealed::VectorOrc<U>, U>(a: T, b: U) -> T::Result {
4200    a.vec_orc(b)
4201}
4202
4203/// Vector Floor
4204#[inline]
4205#[target_feature(enable = "vector")]
4206#[unstable(feature = "stdarch_s390x", issue = "135681")]
4207pub unsafe fn vec_floor<T: sealed::VectorFloor>(a: T) -> T {
4208    a.vec_floor()
4209}
4210
4211/// Vector Ceil
4212#[inline]
4213#[target_feature(enable = "vector")]
4214#[unstable(feature = "stdarch_s390x", issue = "135681")]
4215pub unsafe fn vec_ceil<T: sealed::VectorCeil>(a: T) -> T {
4216    a.vec_ceil()
4217}
4218
4219/// Vector Truncate
4220///
4221/// Returns a vector containing the truncated values of the corresponding elements of the given vector.
4222/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value.
4223#[inline]
4224#[target_feature(enable = "vector")]
4225#[unstable(feature = "stdarch_s390x", issue = "135681")]
4226pub unsafe fn vec_trunc<T: sealed::VectorTrunc>(a: T) -> T {
4227    a.vec_trunc()
4228}
4229
4230/// Vector Round
4231///
4232/// Returns a vector containing the rounded values to the nearest representable floating-point integer,
4233/// using IEEE round-to-nearest rounding, of the corresponding elements of the given vector
4234#[inline]
4235#[target_feature(enable = "vector")]
4236#[unstable(feature = "stdarch_s390x", issue = "135681")]
4237pub unsafe fn vec_round<T: sealed::VectorRound>(a: T) -> T {
4238    a.vec_round()
4239}
4240
4241/// Vector Round to Current
4242///
4243/// Returns a vector by using the current rounding mode to round every
4244/// floating-point element in the given vector to integer.
4245#[inline]
4246#[target_feature(enable = "vector")]
4247#[unstable(feature = "stdarch_s390x", issue = "135681")]
4248pub unsafe fn vec_roundc<T: sealed::VectorRoundc>(a: T) -> T {
4249    a.vec_roundc()
4250}
4251
4252/// Vector Round toward Negative Infinity
4253///
4254/// Returns a vector containing the largest representable floating-point integral values less
4255/// than or equal to the values of the corresponding elements of the given vector.
4256#[inline]
4257#[target_feature(enable = "vector")]
4258#[unstable(feature = "stdarch_s390x", issue = "135681")]
4259pub unsafe fn vec_roundm<T: sealed::VectorFloor>(a: T) -> T {
4260    // the IBM docs note
4261    //
4262    // > vec_roundm provides the same functionality as vec_floor, except that vec_roundz would not trigger the IEEE-inexact exception.
4263    //
4264    // but in practice `vec_floor` also does not trigger that exception, so both are equivalent
4265    a.vec_floor()
4266}
4267
4268/// Vector Round toward Positive Infinity
4269///
4270/// Returns a vector containing the smallest representable floating-point integral values greater
4271/// than or equal to the values of the corresponding elements of the given vector.
4272#[inline]
4273#[target_feature(enable = "vector")]
4274#[unstable(feature = "stdarch_s390x", issue = "135681")]
4275pub unsafe fn vec_roundp<T: sealed::VectorCeil>(a: T) -> T {
4276    // the IBM docs note
4277    //
4278    // > vec_roundp provides the same functionality as vec_ceil, except that vec_roundz would not trigger the IEEE-inexact exception.
4279    //
4280    // but in practice `vec_ceil` also does not trigger that exception, so both are equivalent
4281    a.vec_ceil()
4282}
4283
4284/// Vector Round toward Zero
4285///
4286/// Returns a vector containing the truncated values of the corresponding elements of the given vector.
4287/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value.
4288#[inline]
4289#[target_feature(enable = "vector")]
4290#[unstable(feature = "stdarch_s390x", issue = "135681")]
4291pub unsafe fn vec_roundz<T: sealed::VectorTrunc>(a: T) -> T {
4292    // the IBM docs note
4293    //
4294    // > vec_roundz provides the same functionality as vec_trunc, except that vec_roundz would not trigger the IEEE-inexact exception.
4295    //
4296    // but in practice `vec_trunc` also does not trigger that exception, so both are equivalent
4297    a.vec_trunc()
4298}
4299
4300/// Vector Round to Integer
4301///
4302/// Returns a vector by using the current rounding mode to round every floating-point element in the given vector to integer.
4303#[inline]
4304#[target_feature(enable = "vector")]
4305#[unstable(feature = "stdarch_s390x", issue = "135681")]
4306pub unsafe fn vec_rint<T: sealed::VectorRint>(a: T) -> T {
4307    a.vec_rint()
4308}
4309
4310/// Vector Average
4311#[inline]
4312#[target_feature(enable = "vector")]
4313#[unstable(feature = "stdarch_s390x", issue = "135681")]
4314pub unsafe fn vec_avg<T: sealed::VectorAvg<U>, U>(a: T, b: U) -> T::Result {
4315    a.vec_avg(b)
4316}
4317
4318/// Vector Shift Left
4319#[inline]
4320#[target_feature(enable = "vector")]
4321#[unstable(feature = "stdarch_s390x", issue = "135681")]
4322pub unsafe fn vec_sl<T: sealed::VectorSl<U>, U>(a: T, b: U) -> T::Result {
4323    a.vec_sl(b)
4324}
4325
4326/// Vector Shift Right
4327#[inline]
4328#[target_feature(enable = "vector")]
4329#[unstable(feature = "stdarch_s390x", issue = "135681")]
4330pub unsafe fn vec_sr<T: sealed::VectorSr<U>, U>(a: T, b: U) -> T::Result {
4331    a.vec_sr(b)
4332}
4333
4334/// Vector Shift Right Algebraic
4335#[inline]
4336#[target_feature(enable = "vector")]
4337#[unstable(feature = "stdarch_s390x", issue = "135681")]
4338pub unsafe fn vec_sra<T: sealed::VectorSra<U>, U>(a: T, b: U) -> T::Result {
4339    a.vec_sra(b)
4340}
4341
4342/// Vector Shift Left by Byte
4343#[inline]
4344#[target_feature(enable = "vector")]
4345#[unstable(feature = "stdarch_s390x", issue = "135681")]
4346pub unsafe fn vec_slb<T: sealed::VectorSlb<U>, U>(a: T, b: U) -> T::Result {
4347    a.vec_slb(b)
4348}
4349
4350/// Vector Shift Right by Byte
4351#[inline]
4352#[target_feature(enable = "vector")]
4353#[unstable(feature = "stdarch_s390x", issue = "135681")]
4354pub unsafe fn vec_srb<T: sealed::VectorSrb<U>, U>(a: T, b: U) -> T::Result {
4355    a.vec_srb(b)
4356}
4357
4358/// Vector Shift Right Algebraic by Byte
4359#[inline]
4360#[target_feature(enable = "vector")]
4361#[unstable(feature = "stdarch_s390x", issue = "135681")]
4362pub unsafe fn vec_srab<T: sealed::VectorSrab<U>, U>(a: T, b: U) -> T::Result {
4363    a.vec_srab(b)
4364}
4365
4366/// Vector Element Rotate Left
4367#[inline]
4368#[target_feature(enable = "vector")]
4369#[unstable(feature = "stdarch_s390x", issue = "135681")]
4370pub unsafe fn vec_rl<T: sealed::VectorRl<U>, U>(a: T, b: U) -> T::Result {
4371    a.vec_rl(b)
4372}
4373
4374/// Vector Shift Left
4375///
4376/// Performs a left shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4377/// element of a left by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros.
4378#[inline]
4379#[target_feature(enable = "vector")]
4380#[unstable(feature = "stdarch_s390x", issue = "135681")]
4381pub unsafe fn vec_sll<T>(a: T, b: vector_unsigned_char) -> T
4382where
4383    T: sealed::VectorSll<vector_unsigned_char, Result = T>,
4384{
4385    a.vec_sll(b)
4386}
4387
4388/// Vector Shift Right
4389///
4390/// Performs a right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4391/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros.
4392#[inline]
4393#[target_feature(enable = "vector")]
4394#[unstable(feature = "stdarch_s390x", issue = "135681")]
4395pub unsafe fn vec_srl<T>(a: T, b: vector_unsigned_char) -> T
4396where
4397    T: sealed::VectorSrl<vector_unsigned_char, Result = T>,
4398{
4399    a.vec_srl(b)
4400}
4401
4402/// Vector Shift Right Arithmetic
4403///
4404/// Performs an algebraic right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4405/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by copies of
4406/// the most significant bit of the element of a.
4407#[inline]
4408#[target_feature(enable = "vector")]
4409#[unstable(feature = "stdarch_s390x", issue = "135681")]
4410pub unsafe fn vec_sral<T>(a: T, b: vector_unsigned_char) -> T
4411where
4412    T: sealed::VectorSral<vector_unsigned_char, Result = T>,
4413{
4414    a.vec_sral(b)
4415}
4416
4417/// Vector Element Rotate Left Immediate
4418///
4419/// Rotates each element of a vector left by a given number of bits. Each element of the result is obtained by rotating the corresponding element
4420/// of a left by the number of bits specified by b, modulo the number of bits in the element.
4421#[inline]
4422#[target_feature(enable = "vector")]
4423#[unstable(feature = "stdarch_s390x", issue = "135681")]
4424pub unsafe fn vec_rli<T: sealed::VectorRli>(a: T, bits: core::ffi::c_ulong) -> T {
4425    a.vec_rli(bits)
4426}
4427
4428/// Vector Reverse Elements
4429///
4430/// Returns a vector with the elements of the input vector in reversed order.
4431#[inline]
4432#[target_feature(enable = "vector")]
4433#[unstable(feature = "stdarch_s390x", issue = "135681")]
4434pub unsafe fn vec_reve<T: sealed::VectorReve>(a: T) -> T {
4435    a.vec_reve()
4436}
4437
4438/// Vector Byte Reverse
4439///
4440/// Returns a vector where each vector element contains the corresponding byte-reversed vector element of the input vector.
4441#[inline]
4442#[target_feature(enable = "vector")]
4443#[unstable(feature = "stdarch_s390x", issue = "135681")]
4444pub unsafe fn vec_revb<T: sealed::VectorRevb>(a: T) -> T {
4445    a.vec_revb()
4446}
4447
4448/// Vector Merge High
4449///
4450/// Merges the most significant ("high") halves of two vectors.
4451#[inline]
4452#[target_feature(enable = "vector")]
4453#[unstable(feature = "stdarch_s390x", issue = "135681")]
4454pub unsafe fn vec_mergeh<T: sealed::VectorMergeh>(a: T, b: T) -> T {
4455    a.vec_mergeh(b)
4456}
4457
4458/// Vector Merge Low
4459///
4460/// Merges the least significant ("low") halves of two vectors.
4461#[inline]
4462#[target_feature(enable = "vector")]
4463#[unstable(feature = "stdarch_s390x", issue = "135681")]
4464pub unsafe fn vec_mergel<T: sealed::VectorMergel>(a: T, b: T) -> T {
4465    a.vec_mergel(b)
4466}
4467
4468/// Vector Pack
4469#[inline]
4470#[target_feature(enable = "vector")]
4471#[unstable(feature = "stdarch_s390x", issue = "135681")]
4472pub unsafe fn vec_pack<T: sealed::VectorPack<U>, U>(a: T, b: U) -> T::Result {
4473    a.vec_pack(b)
4474}
4475
4476/// Vector Pack Saturated
4477#[inline]
4478#[target_feature(enable = "vector")]
4479#[unstable(feature = "stdarch_s390x", issue = "135681")]
4480pub unsafe fn vec_packs<T: sealed::VectorPacks<U>, U>(a: T, b: U) -> T::Result {
4481    a.vec_packs(b)
4482}
4483
4484/// Vector Pack Saturated Condition Code
4485#[inline]
4486#[target_feature(enable = "vector")]
4487#[unstable(feature = "stdarch_s390x", issue = "135681")]
4488pub unsafe fn vec_packs_cc<T: sealed::VectorPacksCC>(a: T, b: T) -> (T::Result, i32) {
4489    a.vec_packs_cc(b)
4490}
4491
4492/// Vector Pack Saturated Unsigned
4493#[inline]
4494#[target_feature(enable = "vector")]
4495#[unstable(feature = "stdarch_s390x", issue = "135681")]
4496pub unsafe fn vec_packsu<T: sealed::VectorPacksu<U>, U>(a: T, b: U) -> T::Result {
4497    a.vec_packsu(b)
4498}
4499
4500/// Vector Pack Saturated Unsigned Condition Code
4501#[inline]
4502#[target_feature(enable = "vector")]
4503#[unstable(feature = "stdarch_s390x", issue = "135681")]
4504pub unsafe fn vec_packsu_cc<T: sealed::VectorPacksuCC>(a: T, b: T) -> (T::Result, i32) {
4505    a.vec_packsu_cc(b)
4506}
4507
4508/// Vector Unpack High
4509#[inline]
4510#[target_feature(enable = "vector")]
4511#[unstable(feature = "stdarch_s390x", issue = "135681")]
4512pub unsafe fn vec_unpackh<T: sealed::VectorUnpackh>(a: T) -> <T as sealed::VectorUnpackh>::Result {
4513    a.vec_unpackh()
4514}
4515
4516/// Vector Unpack Low
4517#[inline]
4518#[target_feature(enable = "vector")]
4519#[unstable(feature = "stdarch_s390x", issue = "135681")]
4520pub unsafe fn vec_unpackl<T: sealed::VectorUnpackl>(a: T) -> <T as sealed::VectorUnpackl>::Result {
4521    a.vec_unpackl()
4522}
4523
4524/// Vector Generate Byte Mask
4525///
4526/// Generates byte masks for elements in the return vector. For each bit in a, if the bit is one, all bit positions
4527/// in the corresponding byte element of d are set to ones. Otherwise, if the bit is zero, the corresponding byte element is set to zero.
4528#[inline]
4529#[target_feature(enable = "vector")]
4530#[unstable(feature = "stdarch_s390x", issue = "135681")]
4531#[cfg_attr(test, assert_instr(vgbm, MASK = 0x00FF))]
4532pub unsafe fn vec_genmask<const MASK: u16>() -> vector_unsigned_char {
4533    vector_unsigned_char(const { genmask::<MASK>() })
4534}
4535
4536/// Vector Generate Mask (Byte)
4537#[inline]
4538#[target_feature(enable = "vector")]
4539#[unstable(feature = "stdarch_s390x", issue = "135681")]
4540#[cfg_attr(test, assert_instr(vrepib, L = 3, H = 5))]
4541pub unsafe fn vec_genmasks_8<const L: u8, const H: u8>() -> vector_unsigned_char {
4542    vector_unsigned_char(const { [genmasks(u8::BITS, L, H) as u8; 16] })
4543}
4544
4545/// Vector Generate Mask (Halfword)
4546#[inline]
4547#[target_feature(enable = "vector")]
4548#[unstable(feature = "stdarch_s390x", issue = "135681")]
4549#[cfg_attr(test, assert_instr(vrepih, L = 3, H = 5))]
4550pub unsafe fn vec_genmasks_16<const L: u8, const H: u8>() -> vector_unsigned_short {
4551    vector_unsigned_short(const { [genmasks(u16::BITS, L, H) as u16; 8] })
4552}
4553
4554/// Vector Generate Mask (Word)
4555#[inline]
4556#[target_feature(enable = "vector")]
4557#[unstable(feature = "stdarch_s390x", issue = "135681")]
4558#[cfg_attr(test, assert_instr(vgmf, L = 3, H = 5))]
4559pub unsafe fn vec_genmasks_32<const L: u8, const H: u8>() -> vector_unsigned_int {
4560    vector_unsigned_int(const { [genmasks(u32::BITS, L, H) as u32; 4] })
4561}
4562
4563/// Vector Generate Mask (Doubleword)
4564#[inline]
4565#[target_feature(enable = "vector")]
4566#[unstable(feature = "stdarch_s390x", issue = "135681")]
4567#[cfg_attr(test, assert_instr(vgmg, L = 3, H = 5))]
4568pub unsafe fn vec_genmasks_64<const L: u8, const H: u8>() -> vector_unsigned_long_long {
4569    vector_unsigned_long_long(const { [genmasks(u64::BITS, L, H); 2] })
4570}
4571
4572/// Vector Permute
4573///
4574/// Returns a vector that contains some elements of two vectors, in the order specified by a third vector.
4575/// Each byte of the result is selected by using the least significant 5 bits of the corresponding byte of c as an index into the concatenated bytes of a and b.
4576/// Note: The vector generate mask built-in function [`vec_genmask`] could help generate the mask c.
4577#[inline]
4578#[target_feature(enable = "vector")]
4579#[unstable(feature = "stdarch_s390x", issue = "135681")]
4580pub unsafe fn vec_perm<T: sealed::VectorPerm>(a: T, b: T, c: vector_unsigned_char) -> T {
4581    a.vec_perm(b, c)
4582}
4583
4584/// Vector Sum Across Quadword
4585///
4586/// Returns a vector containing the results of performing a sum across all the elements in each of the quadword of vector a,
4587/// and the rightmost word or doubleword element of the b. The result is an unsigned 128-bit integer.
4588#[inline]
4589#[target_feature(enable = "vector")]
4590#[unstable(feature = "stdarch_s390x", issue = "135681")]
4591pub unsafe fn vec_sum_u128<T: sealed::VectorSumU128>(a: T, b: T) -> vector_unsigned_char {
4592    a.vec_sum_u128(b)
4593}
4594
4595/// Vector Sum Across Doubleword
4596///
4597/// Returns a vector containing the results of performing a sum across all the elements in each of the doubleword of vector a,
4598/// and the rightmost sub-element of the corresponding doubleword of b.
4599#[inline]
4600#[target_feature(enable = "vector")]
4601#[unstable(feature = "stdarch_s390x", issue = "135681")]
4602pub unsafe fn vec_sum2<T: sealed::VectorSum2>(a: T, b: T) -> vector_unsigned_long_long {
4603    a.vec_sum2(b)
4604}
4605
4606/// Vector Sum Across Word
4607///
4608/// Returns a vector containing the results of performing a sum across all the elements in each of the word of vector a,
4609/// and the rightmost sub-element of the corresponding word of b.
4610#[inline]
4611#[target_feature(enable = "vector")]
4612#[unstable(feature = "stdarch_s390x", issue = "135681")]
4613pub unsafe fn vec_sum4<T: sealed::VectorSum4>(a: T, b: T) -> vector_unsigned_int {
4614    a.vec_sum4(b)
4615}
4616
4617/// Vector Addition unsigned 128-bits
4618///
4619/// Adds unsigned quadword values.
4620///
4621/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a + b.
4622#[inline]
4623#[target_feature(enable = "vector")]
4624#[unstable(feature = "stdarch_s390x", issue = "135681")]
4625#[cfg_attr(test, assert_instr(vaq))]
4626pub unsafe fn vec_add_u128(
4627    a: vector_unsigned_char,
4628    b: vector_unsigned_char,
4629) -> vector_unsigned_char {
4630    let a: u128 = transmute(a);
4631    let b: u128 = transmute(b);
4632    transmute(a.wrapping_add(b))
4633}
4634
4635/// Vector Subtract unsigned 128-bits
4636///
4637/// Subtracts unsigned quadword values.
4638///
4639/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a - b.
4640#[inline]
4641#[target_feature(enable = "vector")]
4642#[unstable(feature = "stdarch_s390x", issue = "135681")]
4643#[cfg_attr(test, assert_instr(vsq))]
4644pub unsafe fn vec_sub_u128(
4645    a: vector_unsigned_char,
4646    b: vector_unsigned_char,
4647) -> vector_unsigned_char {
4648    let a: u128 = transmute(a);
4649    let b: u128 = transmute(b);
4650
4651    transmute(a.wrapping_sub(b))
4652}
4653
4654/// Vector Subtract Carryout
4655///
4656/// Returns a vector containing the borrow produced by subtracting each of corresponding elements of b from a.
4657///
4658/// On each resulting element, the value is 0 if a borrow occurred, or 1 if no borrow occurred.
4659#[inline]
4660#[target_feature(enable = "vector")]
4661#[unstable(feature = "stdarch_s390x", issue = "135681")]
4662pub unsafe fn vec_subc<T: sealed::VectorSubc<U>, U>(a: T, b: U) -> T::Result {
4663    a.vec_subc(b)
4664}
4665
4666/// Vector Subtract Carryout unsigned 128-bits
4667///
4668/// Gets the carry bit of the 128-bit subtraction of two quadword values.
4669/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the borrow produced by subtracting b from a, as unsigned 128-bits integers.
4670/// If no borrow occurred, the bit 127 of d is 1; otherwise it is set to 0. All other bits of d are 0.
4671#[inline]
4672#[target_feature(enable = "vector")]
4673#[unstable(feature = "stdarch_s390x", issue = "135681")]
4674#[cfg_attr(test, assert_instr(vscbiq))]
4675pub unsafe fn vec_subc_u128(
4676    a: vector_unsigned_char,
4677    b: vector_unsigned_char,
4678) -> vector_unsigned_char {
4679    // FIXME(llvm) sadly this does not work https://github.com/llvm/llvm-project/issues/129608
4680    // let a: u128 = transmute(a);
4681    // let b: u128 = transmute(b);
4682    // transmute(!a.overflowing_sub(b).1 as u128)
4683    transmute(vscbiq(transmute(a), transmute(b)))
4684}
4685
4686/// Vector Add Compute Carryout unsigned 128-bits
4687#[inline]
4688#[target_feature(enable = "vector")]
4689#[unstable(feature = "stdarch_s390x", issue = "135681")]
4690#[cfg_attr(test, assert_instr(vaccq))]
4691pub unsafe fn vec_addc_u128(
4692    a: vector_unsigned_char,
4693    b: vector_unsigned_char,
4694) -> vector_unsigned_char {
4695    let a: u128 = transmute(a);
4696    let b: u128 = transmute(b);
4697    transmute(a.overflowing_add(b).1 as u128)
4698}
4699
4700/// Vector Add With Carry unsigned 128-bits
4701#[inline]
4702#[target_feature(enable = "vector")]
4703#[unstable(feature = "stdarch_s390x", issue = "135681")]
4704#[cfg_attr(test, assert_instr(vacq))]
4705pub unsafe fn vec_adde_u128(
4706    a: vector_unsigned_char,
4707    b: vector_unsigned_char,
4708    c: vector_unsigned_char,
4709) -> vector_unsigned_char {
4710    let a: u128 = transmute(a);
4711    let b: u128 = transmute(b);
4712    let c: u128 = transmute(c);
4713    // FIXME(llvm) sadly this does not work
4714    //     let (d, _carry) = a.carrying_add(b, c & 1 != 0);
4715    //     transmute(d)
4716    transmute(vacq(a, b, c))
4717}
4718
4719/// Vector Add With Carry Compute Carry unsigned 128-bits
4720#[inline]
4721#[target_feature(enable = "vector")]
4722#[unstable(feature = "stdarch_s390x", issue = "135681")]
4723#[cfg_attr(test, assert_instr(vacccq))]
4724pub unsafe fn vec_addec_u128(
4725    a: vector_unsigned_char,
4726    b: vector_unsigned_char,
4727    c: vector_unsigned_char,
4728) -> vector_unsigned_char {
4729    let a: u128 = transmute(a);
4730    let b: u128 = transmute(b);
4731    let c: u128 = transmute(c);
4732    let (_d, carry) = a.carrying_add(b, c & 1 != 0);
4733    transmute(carry as u128)
4734}
4735
4736/// Vector Subtract with Carryout
4737///
4738/// Subtracts unsigned quadword values with carry bit from a previous operation.
4739///
4740/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the result of subtracting of b from a,
4741/// and the carryout bit from a previous operation.
4742///
4743/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored.
4744#[inline]
4745#[target_feature(enable = "vector")]
4746#[unstable(feature = "stdarch_s390x", issue = "135681")]
4747#[cfg_attr(test, assert_instr(vsbiq))]
4748pub unsafe fn vec_sube_u128(
4749    a: vector_unsigned_char,
4750    b: vector_unsigned_char,
4751    c: vector_unsigned_char,
4752) -> vector_unsigned_char {
4753    transmute(vsbiq(transmute(a), transmute(b), transmute(c)))
4754}
4755
4756/// Vector Subtract with Carryout, Carryout
4757///
4758/// Gets the carry bit of the 128-bit subtraction of two quadword values with carry bit from the previous operation.
4759///
4760/// It returns a vector containing the carryout produced from the result of subtracting of b from a,
4761/// and the carryout bit from a previous operation. If no borrow occurred, the 127-bit of d is 1, otherwise 0.
4762/// All other bits of d are 0.
4763///
4764/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored.
4765#[inline]
4766#[target_feature(enable = "vector")]
4767#[unstable(feature = "stdarch_s390x", issue = "135681")]
4768#[cfg_attr(test, assert_instr(vsbcbiq))]
4769pub unsafe fn vec_subec_u128(
4770    a: vector_unsigned_char,
4771    b: vector_unsigned_char,
4772    c: vector_unsigned_char,
4773) -> vector_unsigned_char {
4774    transmute(vsbcbiq(transmute(a), transmute(b), transmute(c)))
4775}
4776
4777/// Vector Splat Signed Byte
4778#[inline]
4779#[target_feature(enable = "vector")]
4780#[unstable(feature = "stdarch_s390x", issue = "135681")]
4781#[cfg_attr(test, assert_instr(vrepib, IMM = 42))]
4782pub unsafe fn vec_splat_s8<const IMM: i8>() -> vector_signed_char {
4783    vector_signed_char([IMM; 16])
4784}
4785
4786/// Vector Splat Signed Halfword
4787#[inline]
4788#[target_feature(enable = "vector")]
4789#[unstable(feature = "stdarch_s390x", issue = "135681")]
4790#[cfg_attr(test, assert_instr(vrepih, IMM = 42))]
4791pub unsafe fn vec_splat_s16<const IMM: i16>() -> vector_signed_short {
4792    vector_signed_short([IMM; 8])
4793}
4794
4795/// Vector Splat Signed Word
4796#[inline]
4797#[target_feature(enable = "vector")]
4798#[unstable(feature = "stdarch_s390x", issue = "135681")]
4799#[cfg_attr(test, assert_instr(vrepif, IMM = 42))]
4800pub unsafe fn vec_splat_s32<const IMM: i16>() -> vector_signed_int {
4801    vector_signed_int([IMM as i32; 4])
4802}
4803
4804/// Vector Splat Signed Doubleword
4805#[inline]
4806#[target_feature(enable = "vector")]
4807#[unstable(feature = "stdarch_s390x", issue = "135681")]
4808#[cfg_attr(test, assert_instr(vrepig, IMM = 42))]
4809pub unsafe fn vec_splat_s64<const IMM: i16>() -> vector_signed_long_long {
4810    vector_signed_long_long([IMM as i64; 2])
4811}
4812
4813/// Vector Splat Unsigned Byte
4814#[inline]
4815#[target_feature(enable = "vector")]
4816#[unstable(feature = "stdarch_s390x", issue = "135681")]
4817#[cfg_attr(test, assert_instr(vrepib, IMM = 42))]
4818pub unsafe fn vec_splat_u8<const IMM: u8>() -> vector_unsigned_char {
4819    vector_unsigned_char([IMM; 16])
4820}
4821
4822/// Vector Splat Unsigned Halfword
4823#[inline]
4824#[target_feature(enable = "vector")]
4825#[unstable(feature = "stdarch_s390x", issue = "135681")]
4826#[cfg_attr(test, assert_instr(vrepih, IMM = 42))]
4827pub unsafe fn vec_splat_u16<const IMM: i16>() -> vector_unsigned_short {
4828    vector_unsigned_short([IMM as u16; 8])
4829}
4830
4831/// Vector Splat Unsigned Word
4832#[inline]
4833#[target_feature(enable = "vector")]
4834#[unstable(feature = "stdarch_s390x", issue = "135681")]
4835#[cfg_attr(test, assert_instr(vrepif, IMM = 42))]
4836pub unsafe fn vec_splat_u32<const IMM: i16>() -> vector_unsigned_int {
4837    vector_unsigned_int([IMM as u32; 4])
4838}
4839
4840/// Vector Splat Unsigned Doubleword
4841#[inline]
4842#[target_feature(enable = "vector")]
4843#[unstable(feature = "stdarch_s390x", issue = "135681")]
4844#[cfg_attr(test, assert_instr(vrepig, IMM = 42))]
4845pub unsafe fn vec_splat_u64<const IMM: i16>() -> vector_unsigned_long_long {
4846    vector_unsigned_long_long([IMM as u64; 2])
4847}
4848
4849macro_rules! vec_find_any {
4850    ($($Trait:ident $fun:ident $doc:literal)*) => {
4851        $(
4852            #[inline]
4853            #[target_feature(enable = "vector")]
4854            #[unstable(feature = "stdarch_s390x", issue = "135681")]
4855            #[doc = $doc]
4856            pub unsafe fn $fun<T: sealed::$Trait<U>, U>(a: T, b: U) -> T::Result {
4857                a.$fun(b)
4858            }
4859        )*
4860    }
4861}
4862
4863vec_find_any! {
4864    VectorFindAnyEq vec_find_any_eq "Vector Find Any Element Equal with Condition Code"
4865    VectorFindAnyNe vec_find_any_ne "Vector Find Any Element Not Equal with Condition Code"
4866    VectorFindAnyEqIdx vec_find_any_eq_idx "Vector Find Any Element Equal Index with Condition Code"
4867    VectorFindAnyNeIdx vec_find_any_ne_idx "Vector Find Any Element Not Equal Index with Condition Code"
4868    VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx "Vector Find Any Element Equal or Zero Index with Condition Code"
4869    VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx "Vector Find Any Element Not Equal or Zero Index with Condition Code"
4870}
4871
4872macro_rules! vec_find_any_cc {
4873    ($($Trait:ident $fun:ident $doc:literal)*) => {
4874        $(
4875            #[inline]
4876            #[target_feature(enable = "vector")]
4877            #[unstable(feature = "stdarch_s390x", issue = "135681")]
4878            #[doc = $doc]
4879            pub unsafe fn $fun<T: sealed::$Trait<U>, U>(a: T, b: U) -> (T::Result, i32) {
4880                a.$fun(b)
4881            }
4882        )*
4883    }
4884}
4885
4886vec_find_any_cc! {
4887    VectorFindAnyEqCC vec_find_any_eq_cc "Vector Find Any Element Equal with Condition Code"
4888    VectorFindAnyNeCC vec_find_any_ne_cc "Vector Find Any Element Not Equal with Condition Code"
4889    VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc "Vector Find Any Element Equal Index with Condition Code"
4890    VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc "Vector Find Any Element Not Equal Index with Condition Code"
4891    VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc "Vector Find Any Element Equal or Zero Index with Condition Code"
4892    VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc "Vector Find Any Element Not Equal or Zero Index with Condition Code"
4893}
4894
4895/// Vector Load
4896#[inline]
4897#[target_feature(enable = "vector")]
4898#[unstable(feature = "stdarch_s390x", issue = "135681")]
4899pub unsafe fn vec_xl<T: sealed::VectorLoad>(offset: isize, ptr: *const T::ElementType) -> T {
4900    T::vec_xl(offset, ptr)
4901}
4902
4903/// Vector Load Pair
4904#[inline]
4905#[target_feature(enable = "vector")]
4906#[unstable(feature = "stdarch_s390x", issue = "135681")]
4907pub unsafe fn vec_load_pair<T: sealed::VectorLoadPair>(a: T::ElementType, b: T::ElementType) -> T {
4908    T::vec_load_pair(a, b)
4909}
4910
4911/// Vector Load to Block Boundary
4912#[inline]
4913#[target_feature(enable = "vector")]
4914#[unstable(feature = "stdarch_s390x", issue = "135681")]
4915pub unsafe fn vec_load_bndry<T: sealed::VectorLoad, const BLOCK_BOUNDARY: u16>(
4916    ptr: *const T::ElementType,
4917) -> MaybeUninit<T> {
4918    T::vec_load_bndry::<BLOCK_BOUNDARY>(ptr)
4919}
4920
4921/// Vector Store
4922#[inline]
4923#[target_feature(enable = "vector")]
4924#[unstable(feature = "stdarch_s390x", issue = "135681")]
4925pub unsafe fn vec_xst<T: sealed::VectorStore>(vector: T, offset: isize, ptr: *mut T::ElementType) {
4926    vector.vec_xst(offset, ptr)
4927}
4928
4929/// Vector Load with Length
4930#[inline]
4931#[target_feature(enable = "vector")]
4932#[unstable(feature = "stdarch_s390x", issue = "135681")]
4933pub unsafe fn vec_load_len<T: sealed::VectorLoad>(
4934    ptr: *const T::ElementType,
4935    byte_count: u32,
4936) -> T {
4937    T::vec_load_len(ptr, byte_count)
4938}
4939
4940/// Vector Store with Length
4941#[inline]
4942#[target_feature(enable = "vector")]
4943#[unstable(feature = "stdarch_s390x", issue = "135681")]
4944pub unsafe fn vec_store_len<T: sealed::VectorStore>(
4945    vector: T,
4946    ptr: *mut T::ElementType,
4947    byte_count: u32,
4948) {
4949    vector.vec_store_len(ptr, byte_count)
4950}
4951
4952/// Vector Load Rightmost with Length
4953#[inline]
4954#[target_feature(enable = "vector-packed-decimal")]
4955#[unstable(feature = "stdarch_s390x", issue = "135681")]
4956#[cfg_attr(test, assert_instr(vlrlr))]
4957pub unsafe fn vec_load_len_r(ptr: *const u8, byte_count: u32) -> vector_unsigned_char {
4958    vlrl(byte_count, ptr)
4959}
4960
4961/// Vector Store Rightmost with Length
4962#[inline]
4963#[target_feature(enable = "vector-packed-decimal")]
4964#[unstable(feature = "stdarch_s390x", issue = "135681")]
4965#[cfg_attr(test, assert_instr(vstrlr))]
4966pub unsafe fn vec_store_len_r(vector: vector_unsigned_char, ptr: *mut u8, byte_count: u32) {
4967    vstrl(vector, byte_count, ptr)
4968}
4969
4970/// Vector Multiply Add
4971#[inline]
4972#[target_feature(enable = "vector-packed-decimal")]
4973#[unstable(feature = "stdarch_s390x", issue = "135681")]
4974pub unsafe fn vec_madd<T: sealed::VectorMadd>(a: T, b: T, c: T) -> T {
4975    a.vec_madd(b, c)
4976}
4977
4978/// Vector Multiply Add
4979#[inline]
4980#[target_feature(enable = "vector-packed-decimal")]
4981#[unstable(feature = "stdarch_s390x", issue = "135681")]
4982pub unsafe fn vec_msub<T: sealed::VectorMadd>(a: T, b: T, c: T) -> T {
4983    a.vec_msub(b, c)
4984}
4985
4986/// Vector Multiply and Add Even
4987#[inline]
4988#[target_feature(enable = "vector-packed-decimal")]
4989#[unstable(feature = "stdarch_s390x", issue = "135681")]
4990pub unsafe fn vec_meadd<T: sealed::VectorMeadd>(a: T, b: T, c: T::Result) -> T::Result {
4991    a.vec_meadd(b, c)
4992}
4993
4994/// Vector Multiply and Add Odd
4995#[inline]
4996#[target_feature(enable = "vector-packed-decimal")]
4997#[unstable(feature = "stdarch_s390x", issue = "135681")]
4998pub unsafe fn vec_moadd<T: sealed::VectorMoadd>(a: T, b: T, c: T::Result) -> T::Result {
4999    a.vec_moadd(b, c)
5000}
5001
5002/// Vector Multiply and Add High
5003#[inline]
5004#[target_feature(enable = "vector-packed-decimal")]
5005#[unstable(feature = "stdarch_s390x", issue = "135681")]
5006pub unsafe fn vec_mhadd<T: sealed::VectorMhadd>(a: T, b: T, c: T::Result) -> T::Result {
5007    a.vec_mhadd(b, c)
5008}
5009
5010/// Vector Multiply and Add Low
5011#[inline]
5012#[target_feature(enable = "vector-packed-decimal")]
5013#[unstable(feature = "stdarch_s390x", issue = "135681")]
5014pub unsafe fn vec_mladd<T: sealed::VectorMladd>(a: T, b: T, c: T::Result) -> T::Result {
5015    a.vec_mladd(b, c)
5016}
5017
5018/// Vector Checksum
5019#[inline]
5020#[target_feature(enable = "vector")]
5021#[unstable(feature = "stdarch_s390x", issue = "135681")]
5022#[cfg_attr(test, assert_instr(vcksm))]
5023pub unsafe fn vec_checksum(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int {
5024    vcksm(a, b)
5025}
5026
5027/// Vector Multiply Even
5028#[inline]
5029#[target_feature(enable = "vector")]
5030#[unstable(feature = "stdarch_s390x", issue = "135681")]
5031pub unsafe fn vec_mule<T: sealed::VectorMule<U>, U>(a: T, b: T) -> U {
5032    a.vec_mule(b)
5033}
5034
5035/// Vector Multiply Odd
5036#[inline]
5037#[target_feature(enable = "vector")]
5038#[unstable(feature = "stdarch_s390x", issue = "135681")]
5039pub unsafe fn vec_mulo<T: sealed::VectorMulo<U>, U>(a: T, b: T) -> U {
5040    a.vec_mulo(b)
5041}
5042
5043/// Vector Multiply High
5044#[inline]
5045#[target_feature(enable = "vector")]
5046#[unstable(feature = "stdarch_s390x", issue = "135681")]
5047pub unsafe fn vec_mulh<T: sealed::VectorMulh<U>, U>(a: T, b: T) -> U {
5048    a.vec_mulh(b)
5049}
5050
5051/// Vector Galois Field Multiply Sum
5052#[inline]
5053#[target_feature(enable = "vector")]
5054#[unstable(feature = "stdarch_s390x", issue = "135681")]
5055pub unsafe fn vec_gfmsum<T: sealed::VectorGfmsum<U>, U>(a: T, b: T) -> U {
5056    a.vec_gfmsum(b)
5057}
5058
5059/// Vector Galois Field Multiply Sum
5060#[inline]
5061#[target_feature(enable = "vector")]
5062#[unstable(feature = "stdarch_s390x", issue = "135681")]
5063pub unsafe fn vec_gfmsum_accum<T: sealed::VectorGfmsumAccum>(
5064    a: T,
5065    b: T,
5066    c: T::Result,
5067) -> T::Result {
5068    a.vec_gfmsum_accum(b, c)
5069}
5070
5071/// Vector Galois Field Multiply Sum 128-bits
5072#[inline]
5073#[target_feature(enable = "vector")]
5074#[unstable(feature = "stdarch_s390x", issue = "135681")]
5075#[cfg_attr(test, assert_instr(vgfmg))]
5076pub unsafe fn vec_gfmsum_128(
5077    a: vector_unsigned_long_long,
5078    b: vector_unsigned_long_long,
5079) -> vector_unsigned_char {
5080    transmute(vgfmg(a, b))
5081}
5082
5083/// Vector Galois Field Multiply Sum and Accumulate 128-bits
5084#[inline]
5085#[target_feature(enable = "vector")]
5086#[unstable(feature = "stdarch_s390x", issue = "135681")]
5087#[cfg_attr(test, assert_instr(vgfmag))]
5088pub unsafe fn vec_gfmsum_accum_128(
5089    a: vector_unsigned_long_long,
5090    b: vector_unsigned_long_long,
5091    c: vector_unsigned_char,
5092) -> vector_unsigned_char {
5093    transmute(vgfmag(a, b, transmute(c)))
5094}
5095
5096/// Vector Bit Permute
5097#[inline]
5098#[target_feature(enable = "vector-enhancements-1")]
5099#[unstable(feature = "stdarch_s390x", issue = "135681")]
5100#[cfg_attr(test, assert_instr(vbperm))]
5101pub unsafe fn vec_bperm_u128(
5102    a: vector_unsigned_char,
5103    b: vector_unsigned_char,
5104) -> vector_unsigned_long_long {
5105    vbperm(a, b)
5106}
5107
5108/// Vector Gather Element
5109#[inline]
5110#[target_feature(enable = "vector")]
5111#[unstable(feature = "stdarch_s390x", issue = "135681")]
5112pub unsafe fn vec_gather_element<T: sealed::VectorGatherElement, const D: u32>(
5113    a: T,
5114    b: T::Offset,
5115    c: *const T::Element,
5116) -> T {
5117    a.vec_gather_element::<D>(b, c)
5118}
5119
5120/// Vector Select
5121#[inline]
5122#[target_feature(enable = "vector")]
5123#[unstable(feature = "stdarch_s390x", issue = "135681")]
5124pub unsafe fn vec_sel<T: sealed::VectorSel<U>, U>(a: T, b: T, c: U) -> T {
5125    a.vec_sel(b, c)
5126}
5127
5128#[unstable(feature = "stdarch_s390x", issue = "135681")]
5129pub const __VEC_CLASS_FP_ZERO_P: u32 = 1 << 11;
5130#[unstable(feature = "stdarch_s390x", issue = "135681")]
5131pub const __VEC_CLASS_FP_ZERO_N: u32 = 1 << 10;
5132#[unstable(feature = "stdarch_s390x", issue = "135681")]
5133pub const __VEC_CLASS_FP_ZERO: u32 = __VEC_CLASS_FP_ZERO_P | __VEC_CLASS_FP_ZERO_N;
5134#[unstable(feature = "stdarch_s390x", issue = "135681")]
5135pub const __VEC_CLASS_FP_NORMAL_P: u32 = 1 << 9;
5136#[unstable(feature = "stdarch_s390x", issue = "135681")]
5137pub const __VEC_CLASS_FP_NORMAL_N: u32 = 1 << 8;
5138#[unstable(feature = "stdarch_s390x", issue = "135681")]
5139pub const __VEC_CLASS_FP_NORMAL: u32 = __VEC_CLASS_FP_NORMAL_P | __VEC_CLASS_FP_NORMAL_N;
5140#[unstable(feature = "stdarch_s390x", issue = "135681")]
5141pub const __VEC_CLASS_FP_SUBNORMAL_P: u32 = 1 << 7;
5142#[unstable(feature = "stdarch_s390x", issue = "135681")]
5143pub const __VEC_CLASS_FP_SUBNORMAL_N: u32 = 1 << 6;
5144#[unstable(feature = "stdarch_s390x", issue = "135681")]
5145pub const __VEC_CLASS_FP_SUBNORMAL: u32 = __VEC_CLASS_FP_SUBNORMAL_P | __VEC_CLASS_FP_SUBNORMAL_N;
5146#[unstable(feature = "stdarch_s390x", issue = "135681")]
5147pub const __VEC_CLASS_FP_INFINITY_P: u32 = 1 << 5;
5148#[unstable(feature = "stdarch_s390x", issue = "135681")]
5149pub const __VEC_CLASS_FP_INFINITY_N: u32 = 1 << 4;
5150#[unstable(feature = "stdarch_s390x", issue = "135681")]
5151pub const __VEC_CLASS_FP_INFINITY: u32 = __VEC_CLASS_FP_INFINITY_P | __VEC_CLASS_FP_INFINITY_N;
5152#[unstable(feature = "stdarch_s390x", issue = "135681")]
5153pub const __VEC_CLASS_FP_QNAN_P: u32 = 1 << 3;
5154#[unstable(feature = "stdarch_s390x", issue = "135681")]
5155pub const __VEC_CLASS_FP_QNAN_N: u32 = 1 << 2;
5156#[unstable(feature = "stdarch_s390x", issue = "135681")]
5157pub const __VEC_CLASS_FP_QNAN: u32 = __VEC_CLASS_FP_QNAN_P | __VEC_CLASS_FP_QNAN_N;
5158#[unstable(feature = "stdarch_s390x", issue = "135681")]
5159pub const __VEC_CLASS_FP_SNAN_P: u32 = 1 << 1;
5160#[unstable(feature = "stdarch_s390x", issue = "135681")]
5161pub const __VEC_CLASS_FP_SNAN_N: u32 = 1 << 0;
5162#[unstable(feature = "stdarch_s390x", issue = "135681")]
5163pub const __VEC_CLASS_FP_SNAN: u32 = __VEC_CLASS_FP_SNAN_P | __VEC_CLASS_FP_SNAN_N;
5164#[unstable(feature = "stdarch_s390x", issue = "135681")]
5165pub const __VEC_CLASS_FP_NAN: u32 = __VEC_CLASS_FP_QNAN | __VEC_CLASS_FP_SNAN;
5166#[unstable(feature = "stdarch_s390x", issue = "135681")]
5167pub const __VEC_CLASS_FP_NOT_NORMAL: u32 =
5168    __VEC_CLASS_FP_NAN | __VEC_CLASS_FP_SUBNORMAL | __VEC_CLASS_FP_ZERO | __VEC_CLASS_FP_INFINITY;
5169
5170/// Vector Floating-Point Test Data Class
5171///
5172/// You can use the `__VEC_CLASS_FP_*` constants as the argument for this operand
5173#[inline]
5174#[target_feature(enable = "vector")]
5175#[unstable(feature = "stdarch_s390x", issue = "135681")]
5176pub unsafe fn vec_fp_test_data_class<T: sealed::VectorFpTestDataClass, const CLASS: u32>(
5177    a: T,
5178    c: *mut i32,
5179) -> T::Result {
5180    let (x, y) = a.vec_fp_test_data_class::<CLASS>();
5181    c.write(y);
5182    x
5183}
5184
5185/// All Elements Not a Number
5186#[inline]
5187#[target_feature(enable = "vector")]
5188#[unstable(feature = "stdarch_s390x", issue = "135681")]
5189pub unsafe fn vec_all_nan<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5190    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 0)
5191}
5192
5193/// All Elements Numeric
5194#[inline]
5195#[target_feature(enable = "vector")]
5196#[unstable(feature = "stdarch_s390x", issue = "135681")]
5197pub unsafe fn vec_all_numeric<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5198    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 3)
5199}
5200
5201/// Any Elements Not a Number
5202#[inline]
5203#[target_feature(enable = "vector")]
5204#[unstable(feature = "stdarch_s390x", issue = "135681")]
5205pub unsafe fn vec_any_nan<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5206    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 3)
5207}
5208
5209/// Any Elements Numeric
5210#[inline]
5211#[target_feature(enable = "vector")]
5212#[unstable(feature = "stdarch_s390x", issue = "135681")]
5213pub unsafe fn vec_any_numeric<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5214    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 0)
5215}
5216
5217/// Vector Test under Mask
5218#[inline]
5219#[target_feature(enable = "vector")]
5220#[unstable(feature = "stdarch_s390x", issue = "135681")]
5221pub unsafe fn vec_test_mask<T: sealed::VectorTestMask>(a: T, b: T::Mask) -> i32 {
5222    // I can't find much information about this, but this might just be a check for whether the
5223    // bitwise and of a and b is non-zero?
5224    a.vec_test_mask(b)
5225}
5226
5227/// Vector Search String
5228#[inline]
5229#[target_feature(enable = "vector")]
5230#[unstable(feature = "stdarch_s390x", issue = "135681")]
5231pub unsafe fn vec_search_string_cc<T: sealed::VectorSearchString>(
5232    a: T,
5233    b: T,
5234    c: vector_unsigned_char,
5235) -> (vector_unsigned_char, i32) {
5236    a.vec_search_string_cc(b, c)
5237}
5238
5239/// Vector Search String Until Zero
5240#[inline]
5241#[target_feature(enable = "vector")]
5242#[unstable(feature = "stdarch_s390x", issue = "135681")]
5243pub unsafe fn vec_search_string_until_zero_cc<T: sealed::VectorSearchString>(
5244    a: T,
5245    b: T,
5246    c: vector_unsigned_char,
5247) -> (vector_unsigned_char, i32) {
5248    a.vec_search_string_until_zero_cc(b, c)
5249}
5250
5251/// Vector Convert from float (even elements) to double
5252#[inline]
5253#[target_feature(enable = "vector-enhancements-1")]
5254#[unstable(feature = "stdarch_s390x", issue = "135681")]
5255// FIXME: this emits `vflls` where `vldeb` is expected
5256// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vldeb))]
5257pub unsafe fn vec_doublee(a: vector_float) -> vector_double {
5258    let even = simd_shuffle::<_, _, f32x2>(a, a, const { u32x2::from_array([0, 2]) });
5259    simd_as(even)
5260}
5261
5262/// Vector Convert from double to float (even elements)
5263#[inline]
5264#[target_feature(enable = "vector-enhancements-1")]
5265#[unstable(feature = "stdarch_s390x", issue = "135681")]
5266// FIXME: the C version uses a shuffle mask with poison; we can't do that
5267// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vledb))]
5268pub unsafe fn vec_floate(a: vector_double) -> vector_float {
5269    let truncated: f32x2 = simd_as(a);
5270    simd_shuffle(
5271        truncated,
5272        truncated,
5273        const { u32x4::from_array([0, 0, 1, 1]) },
5274    )
5275}
5276
5277/// Vector Convert from int to float
5278#[inline]
5279#[target_feature(enable = "vector")]
5280#[unstable(feature = "stdarch_s390x", issue = "135681")]
5281pub unsafe fn vec_float(a: impl sealed::VectorFloat) -> vector_float {
5282    a.vec_float()
5283}
5284
5285/// Vector Convert from long long to double
5286#[inline]
5287#[target_feature(enable = "vector")]
5288#[unstable(feature = "stdarch_s390x", issue = "135681")]
5289pub unsafe fn vec_double(a: impl sealed::VectorDouble) -> vector_double {
5290    a.vec_double()
5291}
5292
5293/// Vector Sign Extend to Doubleword
5294#[inline]
5295#[target_feature(enable = "vector")]
5296#[unstable(feature = "stdarch_s390x", issue = "135681")]
5297pub unsafe fn vec_extend_s64(a: impl sealed::VectorExtendSigned64) -> vector_signed_long_long {
5298    a.vec_extend_s64()
5299}
5300
5301/// Vector Convert floating point to signed
5302#[inline]
5303#[target_feature(enable = "vector")]
5304#[unstable(feature = "stdarch_s390x", issue = "135681")]
5305pub unsafe fn vec_signed<T: sealed::VectorSigned>(a: T) -> T::Result {
5306    a.vec_signed()
5307}
5308
5309/// Vector Convert floating point to unsigned
5310#[inline]
5311#[target_feature(enable = "vector")]
5312#[unstable(feature = "stdarch_s390x", issue = "135681")]
5313pub unsafe fn vec_unsigned<T: sealed::VectorUnsigned>(a: T) -> T::Result {
5314    a.vec_unsigned()
5315}
5316
5317/// Vector Copy Until Zero
5318#[inline]
5319#[target_feature(enable = "vector")]
5320#[unstable(feature = "stdarch_s390x", issue = "135681")]
5321pub unsafe fn vec_cp_until_zero<T: sealed::VectorCopyUntilZero>(a: T) -> T {
5322    a.vec_cp_until_zero()
5323}
5324
5325/// Vector Copy Until Zero
5326#[inline]
5327#[target_feature(enable = "vector")]
5328#[unstable(feature = "stdarch_s390x", issue = "135681")]
5329pub unsafe fn vec_cp_until_zero_cc<T: sealed::VectorCopyUntilZeroCC>(a: T) -> (T, i32) {
5330    a.vec_cp_until_zero_cc()
5331}
5332
5333/// Vector Multiply Sum Logical
5334#[inline]
5335#[target_feature(enable = "vector-enhancements-1")]
5336#[unstable(feature = "stdarch_s390x", issue = "135681")]
5337#[cfg_attr(
5338    all(test, target_feature = "vector-enhancements-1"),
5339    assert_instr(vmslg, D = 4)
5340)]
5341pub unsafe fn vec_msum_u128<const D: u32>(
5342    a: vector_unsigned_long_long,
5343    b: vector_unsigned_long_long,
5344    c: vector_unsigned_char,
5345) -> vector_unsigned_char {
5346    const {
5347        if !matches!(D, 0 | 4 | 8 | 12) {
5348            panic!("D needs to be one of 0, 4, 8, 12");
5349        }
5350    };
5351    transmute(vmslg(a, b, transmute(c), D))
5352}
5353
5354/// Vector Shift Left Double by Byte
5355#[inline]
5356#[target_feature(enable = "vector")]
5357#[unstable(feature = "stdarch_s390x", issue = "135681")]
5358pub unsafe fn vec_sld<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5359    static_assert_uimm_bits!(C, 4);
5360    a.vec_sld::<C>(b)
5361}
5362
5363/// Vector Shift Left Double by Word
5364#[inline]
5365#[target_feature(enable = "vector")]
5366#[unstable(feature = "stdarch_s390x", issue = "135681")]
5367pub unsafe fn vec_sldw<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5368    static_assert_uimm_bits!(C, 2);
5369    a.vec_sldw::<C>(b)
5370}
5371
5372/// Vector Shift Left Double by Bit
5373#[inline]
5374#[target_feature(enable = "vector-enhancements-2")]
5375#[unstable(feature = "stdarch_s390x", issue = "135681")]
5376pub unsafe fn vec_sldb<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5377    static_assert_uimm_bits!(C, 3);
5378    a.vec_sldb::<C>(b)
5379}
5380
5381/// Vector Shift Right Double by Bit
5382#[inline]
5383#[target_feature(enable = "vector-enhancements-2")]
5384#[unstable(feature = "stdarch_s390x", issue = "135681")]
5385pub unsafe fn vec_srdb<T: sealed::VectorSrdb, const C: u32>(a: T, b: T) -> T {
5386    static_assert_uimm_bits!(C, 3);
5387    a.vec_srdb::<C>(b)
5388}
5389
5390/// Vector Compare Ranges
5391#[inline]
5392#[target_feature(enable = "vector")]
5393#[unstable(feature = "stdarch_s390x", issue = "135681")]
5394pub unsafe fn vec_cmprg<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5395    a.vstrc::<{ FindImm::Eq as u32 }>(b, c)
5396}
5397
5398/// Vector Compare Not in Ranges
5399#[inline]
5400#[target_feature(enable = "vector")]
5401#[unstable(feature = "stdarch_s390x", issue = "135681")]
5402pub unsafe fn vec_cmpnrg<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5403    a.vstrc::<{ FindImm::Ne as u32 }>(b, c)
5404}
5405
5406/// Vector Compare Ranges Index
5407#[inline]
5408#[target_feature(enable = "vector")]
5409#[unstable(feature = "stdarch_s390x", issue = "135681")]
5410pub unsafe fn vec_cmprg_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5411    a.vstrc::<{ FindImm::EqIdx as u32 }>(b, c)
5412}
5413
5414/// Vector Compare Not in Ranges Index
5415#[inline]
5416#[target_feature(enable = "vector")]
5417#[unstable(feature = "stdarch_s390x", issue = "135681")]
5418pub unsafe fn vec_cmpnrg_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5419    a.vstrc::<{ FindImm::NeIdx as u32 }>(b, c)
5420}
5421
5422/// Vector Compare Ranges with Condition Code
5423#[inline]
5424#[target_feature(enable = "vector")]
5425#[unstable(feature = "stdarch_s390x", issue = "135681")]
5426pub unsafe fn vec_cmprg_cc<T: sealed::VectorCompareRange>(
5427    a: T,
5428    b: T,
5429    c: T,
5430    d: *mut i32,
5431) -> T::Result {
5432    let (x, y) = a.vstrcs::<{ FindImm::Eq as u32 }>(b, c);
5433    d.write(y);
5434    x
5435}
5436
5437/// Vector Compare Not in Ranges with Condition Code
5438#[inline]
5439#[target_feature(enable = "vector")]
5440#[unstable(feature = "stdarch_s390x", issue = "135681")]
5441pub unsafe fn vec_cmpnrg_cc<T: sealed::VectorCompareRange>(
5442    a: T,
5443    b: T,
5444    c: T,
5445    d: *mut i32,
5446) -> T::Result {
5447    let (x, y) = a.vstrcs::<{ FindImm::Ne as u32 }>(b, c);
5448    d.write(y);
5449    x
5450}
5451
5452/// Vector Compare Ranges Index with Condition Code
5453#[inline]
5454#[target_feature(enable = "vector")]
5455#[unstable(feature = "stdarch_s390x", issue = "135681")]
5456pub unsafe fn vec_cmprg_idx_cc<T: sealed::VectorCompareRange>(
5457    a: T,
5458    b: T,
5459    c: T,
5460    d: *mut i32,
5461) -> T::Result {
5462    let (x, y) = a.vstrcs::<{ FindImm::EqIdx as u32 }>(b, c);
5463    d.write(y);
5464    x
5465}
5466
5467/// Vector Compare Not in Ranges Index with Condition Code
5468#[inline]
5469#[target_feature(enable = "vector")]
5470#[unstable(feature = "stdarch_s390x", issue = "135681")]
5471pub unsafe fn vec_cmpnrg_idx_cc<T: sealed::VectorCompareRange>(
5472    a: T,
5473    b: T,
5474    c: T,
5475    d: *mut i32,
5476) -> T::Result {
5477    let (x, y) = a.vstrcs::<{ FindImm::NeIdx as u32 }>(b, c);
5478    d.write(y);
5479    x
5480}
5481
5482/// Vector Compare Ranges or Zero Index
5483#[inline]
5484#[target_feature(enable = "vector")]
5485#[unstable(feature = "stdarch_s390x", issue = "135681")]
5486pub unsafe fn vec_cmprg_or_0_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5487    a.vstrcz::<{ FindImm::EqIdx as u32 }>(b, c)
5488}
5489
5490/// Vector Compare Not in Ranges or Zero Index
5491#[inline]
5492#[target_feature(enable = "vector")]
5493#[unstable(feature = "stdarch_s390x", issue = "135681")]
5494pub unsafe fn vec_cmpnrg_or_0_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5495    a.vstrcz::<{ FindImm::NeIdx as u32 }>(b, c)
5496}
5497
5498/// Vector Compare Ranges or Zero Index with Condition Code
5499#[inline]
5500#[target_feature(enable = "vector")]
5501#[unstable(feature = "stdarch_s390x", issue = "135681")]
5502pub unsafe fn vec_cmprg_or_0_idx_cc<T: sealed::VectorCompareRange>(
5503    a: T,
5504    b: T,
5505    c: T,
5506    d: *mut i32,
5507) -> T::Result {
5508    let (x, y) = a.vstrczs::<{ FindImm::EqIdx as u32 }>(b, c);
5509    d.write(y);
5510    x
5511}
5512
5513/// Vector Compare Not in Ranges or Zero Index with Condition Code
5514#[inline]
5515#[target_feature(enable = "vector")]
5516#[unstable(feature = "stdarch_s390x", issue = "135681")]
5517pub unsafe fn vec_cmpnrg_or_0_idx_cc<T: sealed::VectorCompareRange>(
5518    a: T,
5519    b: T,
5520    c: T,
5521    d: *mut i32,
5522) -> T::Result {
5523    let (x, y) = a.vstrczs::<{ FindImm::NeIdx as u32 }>(b, c);
5524    d.write(y);
5525    x
5526}
5527
5528/// Vector Compare Equal
5529#[inline]
5530#[target_feature(enable = "vector")]
5531#[unstable(feature = "stdarch_s390x", issue = "135681")]
5532pub unsafe fn vec_cmpeq<T: sealed::VectorEquality>(a: T, b: T) -> T::Result {
5533    a.vec_cmpeq(b)
5534}
5535
5536/// Vector Compare Not Equal
5537#[inline]
5538#[target_feature(enable = "vector")]
5539#[unstable(feature = "stdarch_s390x", issue = "135681")]
5540pub unsafe fn vec_cmpne<T: sealed::VectorEquality>(a: T, b: T) -> T::Result {
5541    a.vec_cmpne(b)
5542}
5543
5544/// Vector Compare Greater Than
5545#[inline]
5546#[target_feature(enable = "vector")]
5547#[unstable(feature = "stdarch_s390x", issue = "135681")]
5548pub unsafe fn vec_cmpgt<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5549    a.vec_cmpgt(b)
5550}
5551
5552/// Vector Compare Greater Than or Equal
5553#[inline]
5554#[target_feature(enable = "vector")]
5555#[unstable(feature = "stdarch_s390x", issue = "135681")]
5556pub unsafe fn vec_cmpge<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5557    a.vec_cmpge(b)
5558}
5559
5560/// Vector Compare Less
5561#[inline]
5562#[target_feature(enable = "vector")]
5563#[unstable(feature = "stdarch_s390x", issue = "135681")]
5564pub unsafe fn vec_cmplt<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5565    a.vec_cmplt(b)
5566}
5567
5568/// Vector Compare Less Than or Equal
5569#[inline]
5570#[target_feature(enable = "vector")]
5571#[unstable(feature = "stdarch_s390x", issue = "135681")]
5572pub unsafe fn vec_cmple<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5573    a.vec_cmple(b)
5574}
5575
5576/// Vector Compare Equal Index
5577#[inline]
5578#[target_feature(enable = "vector")]
5579#[unstable(feature = "stdarch_s390x", issue = "135681")]
5580pub unsafe fn vec_cmpeq_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5581    a.vec_cmpeq_idx(b)
5582}
5583/// Vector Compare Not Equal Index
5584#[inline]
5585#[target_feature(enable = "vector")]
5586#[unstable(feature = "stdarch_s390x", issue = "135681")]
5587pub unsafe fn vec_cmpne_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5588    a.vec_cmpne_idx(b)
5589}
5590/// Vector Compare Equal Index with Condition Code
5591#[inline]
5592#[target_feature(enable = "vector")]
5593#[unstable(feature = "stdarch_s390x", issue = "135681")]
5594pub unsafe fn vec_cmpeq_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5595    a.vec_cmpeq_idx_cc(b)
5596}
5597/// Vector Compare Not Equal Index with Condition Code
5598#[inline]
5599#[target_feature(enable = "vector")]
5600#[unstable(feature = "stdarch_s390x", issue = "135681")]
5601pub unsafe fn vec_cmpne_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5602    a.vec_cmpne_idx_cc(b)
5603}
5604/// Vector Compare Equal or Zero Index
5605#[inline]
5606#[target_feature(enable = "vector")]
5607#[unstable(feature = "stdarch_s390x", issue = "135681")]
5608pub unsafe fn vec_cmpeq_or_0_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5609    a.vec_cmpeq_or_0_idx(b)
5610}
5611/// Vector Compare Not Equal or Zero Index
5612#[inline]
5613#[target_feature(enable = "vector")]
5614#[unstable(feature = "stdarch_s390x", issue = "135681")]
5615pub unsafe fn vec_cmpne_or_0_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5616    a.vec_cmpne_or_0_idx(b)
5617}
5618/// Vector Compare Equal or Zero Index with Condition Code
5619#[inline]
5620#[target_feature(enable = "vector")]
5621#[unstable(feature = "stdarch_s390x", issue = "135681")]
5622pub unsafe fn vec_cmpeq_or_0_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5623    a.vec_cmpeq_or_0_idx_cc(b)
5624}
5625/// Vector Compare Not Equal or Zero Index with Condition Code
5626#[inline]
5627#[target_feature(enable = "vector")]
5628#[unstable(feature = "stdarch_s390x", issue = "135681")]
5629pub unsafe fn vec_cmpne_or_0_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5630    a.vec_cmpne_or_0_idx_cc(b)
5631}
5632
5633/// All Elements Equal
5634#[inline]
5635#[target_feature(enable = "vector")]
5636#[unstable(feature = "stdarch_s390x", issue = "135681")]
5637pub unsafe fn vec_all_eq<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5638    simd_reduce_all(vec_cmpeq(a, b)) as i32 as i32
5639}
5640
5641/// All Elements Not Equal
5642#[inline]
5643#[target_feature(enable = "vector")]
5644#[unstable(feature = "stdarch_s390x", issue = "135681")]
5645pub unsafe fn vec_all_ne<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5646    simd_reduce_all(vec_cmpne(a, b)) as i32
5647}
5648
5649/// Any Element Equal
5650#[inline]
5651#[target_feature(enable = "vector")]
5652#[unstable(feature = "stdarch_s390x", issue = "135681")]
5653pub unsafe fn vec_any_eq<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5654    simd_reduce_any(vec_cmpeq(a, b)) as i32
5655}
5656
5657/// Any Element Not Equal
5658#[inline]
5659#[target_feature(enable = "vector")]
5660#[unstable(feature = "stdarch_s390x", issue = "135681")]
5661pub unsafe fn vec_any_ne<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5662    simd_reduce_any(vec_cmpne(a, b)) as i32
5663}
5664
5665/// All Elements Less Than
5666#[inline]
5667#[target_feature(enable = "vector")]
5668#[unstable(feature = "stdarch_s390x", issue = "135681")]
5669pub unsafe fn vec_all_lt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5670    a.vec_all_lt(b)
5671}
5672
5673/// All Elements Less Than or Equal
5674#[inline]
5675#[target_feature(enable = "vector")]
5676#[unstable(feature = "stdarch_s390x", issue = "135681")]
5677pub unsafe fn vec_all_le<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5678    a.vec_all_le(b)
5679}
5680
5681/// All Elements Greater Than
5682#[inline]
5683#[target_feature(enable = "vector")]
5684#[unstable(feature = "stdarch_s390x", issue = "135681")]
5685pub unsafe fn vec_all_gt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5686    a.vec_all_gt(b)
5687}
5688
5689/// All Elements Greater Than or Equal
5690#[inline]
5691#[target_feature(enable = "vector")]
5692#[unstable(feature = "stdarch_s390x", issue = "135681")]
5693pub unsafe fn vec_all_ge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5694    a.vec_all_ge(b)
5695}
5696
5697/// All Elements Not Less Than
5698#[inline]
5699#[target_feature(enable = "vector")]
5700#[unstable(feature = "stdarch_s390x", issue = "135681")]
5701pub unsafe fn vec_all_nlt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5702    vec_all_ge(a, b)
5703}
5704
5705/// All Elements Not Less Than or Equal
5706#[inline]
5707#[target_feature(enable = "vector")]
5708#[unstable(feature = "stdarch_s390x", issue = "135681")]
5709pub unsafe fn vec_all_nle<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5710    vec_all_gt(a, b)
5711}
5712
5713/// All Elements Not Greater Than
5714#[inline]
5715#[target_feature(enable = "vector")]
5716#[unstable(feature = "stdarch_s390x", issue = "135681")]
5717pub unsafe fn vec_all_ngt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5718    vec_all_le(a, b)
5719}
5720
5721/// All Elements Not Greater Than or Equal
5722#[inline]
5723#[target_feature(enable = "vector")]
5724#[unstable(feature = "stdarch_s390x", issue = "135681")]
5725pub unsafe fn vec_all_nge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5726    vec_all_lt(a, b)
5727}
5728
5729/// Any Elements Less Than
5730#[inline]
5731#[target_feature(enable = "vector")]
5732#[unstable(feature = "stdarch_s390x", issue = "135681")]
5733pub unsafe fn vec_any_lt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5734    !vec_all_ge(a, b)
5735}
5736
5737/// Any Elements Less Than or Equal
5738#[inline]
5739#[target_feature(enable = "vector")]
5740#[unstable(feature = "stdarch_s390x", issue = "135681")]
5741pub unsafe fn vec_any_le<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5742    !vec_all_gt(a, b)
5743}
5744
5745/// Any Elements Greater Than
5746#[inline]
5747#[target_feature(enable = "vector")]
5748#[unstable(feature = "stdarch_s390x", issue = "135681")]
5749pub unsafe fn vec_any_gt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5750    !vec_all_le(a, b)
5751}
5752
5753/// Any Elements Greater Than or Equal
5754#[inline]
5755#[target_feature(enable = "vector")]
5756#[unstable(feature = "stdarch_s390x", issue = "135681")]
5757pub unsafe fn vec_any_ge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5758    !vec_all_lt(a, b)
5759}
5760
5761/// Any Elements Not Less Than
5762#[inline]
5763#[target_feature(enable = "vector")]
5764#[unstable(feature = "stdarch_s390x", issue = "135681")]
5765pub unsafe fn vec_any_nlt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5766    vec_any_ge(a, b)
5767}
5768
5769/// Any Elements Not Less Than or Equal
5770#[inline]
5771#[target_feature(enable = "vector")]
5772#[unstable(feature = "stdarch_s390x", issue = "135681")]
5773pub unsafe fn vec_any_nle<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5774    vec_any_gt(a, b)
5775}
5776
5777/// Any Elements Not Greater Than
5778#[inline]
5779#[target_feature(enable = "vector")]
5780#[unstable(feature = "stdarch_s390x", issue = "135681")]
5781pub unsafe fn vec_any_ngt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5782    vec_any_le(a, b)
5783}
5784
5785/// Any Elements Not Greater Than or Equal
5786#[inline]
5787#[target_feature(enable = "vector")]
5788#[unstable(feature = "stdarch_s390x", issue = "135681")]
5789pub unsafe fn vec_any_nge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5790    vec_any_lt(a, b)
5791}
5792
5793/// Vector Extract
5794#[inline]
5795#[target_feature(enable = "vector")]
5796#[unstable(feature = "stdarch_s390x", issue = "135681")]
5797pub unsafe fn vec_extract<T: sealed::VectorExtract>(a: T, b: i32) -> T::ElementType {
5798    T::vec_extract(a, b)
5799}
5800
5801/// Vector Insert
5802#[inline]
5803#[target_feature(enable = "vector")]
5804#[unstable(feature = "stdarch_s390x", issue = "135681")]
5805pub unsafe fn vec_insert<T: sealed::VectorInsert>(a: T::ElementType, b: T, c: i32) -> T {
5806    T::vec_insert(a, b, c)
5807}
5808
5809/// Vector Insert and Zero
5810#[inline]
5811#[target_feature(enable = "vector")]
5812#[unstable(feature = "stdarch_s390x", issue = "135681")]
5813pub unsafe fn vec_insert_and_zero<T: sealed::VectorInsertAndZero>(a: *const T::ElementType) -> T {
5814    T::vec_insert_and_zero(a)
5815}
5816
5817/// Vector Promote
5818#[inline]
5819#[target_feature(enable = "vector")]
5820#[unstable(feature = "stdarch_s390x", issue = "135681")]
5821pub unsafe fn vec_promote<T: sealed::VectorPromote>(a: T::ElementType, b: i32) -> MaybeUninit<T> {
5822    T::vec_promote(a, b)
5823}
5824
5825#[cfg(test)]
5826mod tests {
5827    use super::*;
5828
5829    use std::mem::transmute;
5830
5831    use crate::core_arch::simd::*;
5832    use stdarch_test::simd_test;
5833
5834    impl<const N: usize> ShuffleMask<N> {
5835        fn as_array(&self) -> &[u32; N] {
5836            unsafe { std::mem::transmute(self) }
5837        }
5838    }
5839
5840    #[test]
5841    fn reverse_mask() {
5842        assert_eq!(ShuffleMask::<4>::reverse().as_array(), &[3, 2, 1, 0]);
5843    }
5844
5845    #[test]
5846    fn mergel_mask() {
5847        assert_eq!(ShuffleMask::<4>::merge_low().as_array(), &[2, 6, 3, 7]);
5848    }
5849
5850    #[test]
5851    fn mergeh_mask() {
5852        assert_eq!(ShuffleMask::<4>::merge_high().as_array(), &[0, 4, 1, 5]);
5853    }
5854
5855    #[test]
5856    fn pack_mask() {
5857        assert_eq!(ShuffleMask::<4>::pack().as_array(), &[1, 3, 5, 7]);
5858    }
5859
5860    #[test]
5861    fn test_vec_mask() {
5862        assert_eq!(
5863            genmask::<0x00FF>(),
5864            [
5865                0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
5866            ]
5867        );
5868    }
5869
5870    #[test]
5871    fn test_genmasks() {
5872        assert_eq!(genmasks(u8::BITS, 3, 5), 28);
5873        assert_eq!(genmasks(u8::BITS, 3, 7), 31);
5874
5875        // If a or b is greater than 8, the operation is performed as if the value gets modulo by 8.
5876        assert_eq!(genmasks(u8::BITS, 3 + 8, 7 + 8), 31);
5877        // If a is greater than b, the operation is perform as if b equals 7.
5878        assert_eq!(genmasks(u8::BITS, 5, 4), genmasks(u8::BITS, 5, 7));
5879
5880        assert_eq!(
5881            genmasks(u16::BITS, 4, 12) as u16,
5882            u16::from_be_bytes([15, -8i8 as u8])
5883        );
5884        assert_eq!(
5885            genmasks(u32::BITS, 4, 29) as u32,
5886            u32::from_be_bytes([15, 0xFF, 0xFF, -4i8 as u8])
5887        );
5888    }
5889
5890    macro_rules! test_vec_1 {
5891        { $name: ident, $fn:ident, f32x4, [$($a:expr),+], ~[$($d:expr),+] } => {
5892            #[simd_test(enable = "vector")]
5893            unsafe fn $name() {
5894                let a: vector_float = transmute(f32x4::new($($a),+));
5895
5896                let d: vector_float = transmute(f32x4::new($($d),+));
5897                let r = transmute(vec_cmple(vec_abs(vec_sub($fn(a), d)), vec_splats(f32::EPSILON)));
5898                let e = m32x4::new(true, true, true, true);
5899                assert_eq!(e, r);
5900            }
5901        };
5902        { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($d:expr),+] } => {
5903            test_vec_1! { $name, $fn, $ty -> $ty, [$($a),+], [$($d),+] }
5904        };
5905        { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($d:expr),+] } => {
5906            #[simd_test(enable = "vector")]
5907            unsafe fn $name() {
5908                let a: s_t_l!($ty) = transmute($ty::new($($a),+));
5909
5910                let d = $ty_out::new($($d),+);
5911                let r : $ty_out = transmute($fn(a));
5912                assert_eq!(d, r);
5913            }
5914        }
5915    }
5916
5917    macro_rules! test_vec_2 {
5918        { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5919            test_vec_2! { $name, $fn, $ty -> $ty, [$($a),+], [$($b),+], [$($d),+] }
5920        };
5921        { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5922            test_vec_2! { $name, $fn, $ty, $ty -> $ty, [$($a),+], [$($b),+], [$($d),+] }
5923         };
5924        { $name: ident, $fn:ident, $ty1: ident, $ty2: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5925            #[simd_test(enable = "vector")]
5926            unsafe fn $name() {
5927                let a: s_t_l!($ty1) = transmute($ty1::new($($a),+));
5928                let b: s_t_l!($ty2) = transmute($ty2::new($($b),+));
5929
5930                let d = $ty_out::new($($d),+);
5931                let r : $ty_out = transmute($fn(a, b));
5932                assert_eq!(d, r);
5933            }
5934         };
5935         { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], $d:expr } => {
5936            #[simd_test(enable = "vector")]
5937            unsafe fn $name() {
5938                let a: s_t_l!($ty) = transmute($ty::new($($a),+));
5939                let b: s_t_l!($ty) = transmute($ty::new($($b),+));
5940
5941                let r : $ty_out = transmute($fn(a, b));
5942                assert_eq!($d, r);
5943            }
5944         }
5945   }
5946
5947    #[simd_test(enable = "vector")]
5948    unsafe fn vec_add_i32x4_i32x4() {
5949        let x = i32x4::new(1, 2, 3, 4);
5950        let y = i32x4::new(4, 3, 2, 1);
5951        let x: vector_signed_int = transmute(x);
5952        let y: vector_signed_int = transmute(y);
5953        let z = vec_add(x, y);
5954        assert_eq!(i32x4::splat(5), transmute(z));
5955    }
5956
5957    macro_rules! test_vec_sub {
5958        { $name: ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5959            test_vec_2! {$name, vec_sub, $ty, [$($a),+], [$($b),+], [$($d),+] }
5960        }
5961    }
5962
5963    test_vec_sub! { test_vec_sub_f32x4, f32x4,
5964    [-1.0, 0.0, 1.0, 2.0],
5965    [2.0, 1.0, -1.0, -2.0],
5966    [-3.0, -1.0, 2.0, 4.0] }
5967
5968    test_vec_sub! { test_vec_sub_f64x2, f64x2,
5969    [-1.0, 0.0],
5970    [2.0, 1.0],
5971    [-3.0, -1.0] }
5972
5973    test_vec_sub! { test_vec_sub_i64x2, i64x2,
5974    [-1, 0],
5975    [2, 1],
5976    [-3, -1] }
5977
5978    test_vec_sub! { test_vec_sub_u64x2, u64x2,
5979    [0, 1],
5980    [1, 0],
5981    [u64::MAX, 1] }
5982
5983    test_vec_sub! { test_vec_sub_i32x4, i32x4,
5984    [-1, 0, 1, 2],
5985    [2, 1, -1, -2],
5986    [-3, -1, 2, 4] }
5987
5988    test_vec_sub! { test_vec_sub_u32x4, u32x4,
5989    [0, 0, 1, 2],
5990    [2, 1, 0, 0],
5991    [4294967294, 4294967295, 1, 2] }
5992
5993    test_vec_sub! { test_vec_sub_i16x8, i16x8,
5994    [-1, 0, 1, 2, -1, 0, 1, 2],
5995    [2, 1, -1, -2, 2, 1, -1, -2],
5996    [-3, -1, 2, 4, -3, -1, 2, 4] }
5997
5998    test_vec_sub! { test_vec_sub_u16x8, u16x8,
5999    [0, 0, 1, 2, 0, 0, 1, 2],
6000    [2, 1, 0, 0, 2, 1, 0, 0],
6001    [65534, 65535, 1, 2, 65534, 65535, 1, 2] }
6002
6003    test_vec_sub! { test_vec_sub_i8x16, i8x16,
6004    [-1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2],
6005    [2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2],
6006    [-3, -1, 2, 4, -3, -1, 2, 4, -3, -1, 2, 4, -3, -1, 2, 4] }
6007
6008    test_vec_sub! { test_vec_sub_u8x16, u8x16,
6009    [0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 1, 2],
6010    [2, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 0],
6011    [254, 255, 1, 2, 254, 255, 1, 2, 254, 255, 1, 2, 254, 255, 1, 2] }
6012
6013    macro_rules! test_vec_mul {
6014        { $name: ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
6015            test_vec_2! {$name, vec_mul, $ty, [$($a),+], [$($b),+], [$($d),+] }
6016        }
6017    }
6018
6019    test_vec_mul! { test_vec_mul_f32x4, f32x4,
6020    [-1.0, 0.0, 1.0, 2.0],
6021    [2.0, 1.0, -1.0, -2.0],
6022    [-2.0, 0.0, -1.0, -4.0] }
6023
6024    test_vec_mul! { test_vec_mul_f64x2, f64x2,
6025    [-1.0, 0.0],
6026    [2.0, 1.0],
6027    [-2.0, 0.0] }
6028
6029    test_vec_mul! { test_vec_mul_i64x2, i64x2,
6030    [i64::MAX, -4],
6031    [2, 3],
6032    [i64::MAX.wrapping_mul(2), -12] }
6033
6034    test_vec_mul! { test_vec_mul_u64x2, u64x2,
6035    [u64::MAX, 4],
6036    [2, 3],
6037    [u64::MAX.wrapping_mul(2), 12] }
6038
6039    test_vec_mul! { test_vec_mul_i32x4, i32x4,
6040    [-1, 0, 1, 2],
6041    [2, 1, -1, -2],
6042    [-2, 0, -1, -4] }
6043
6044    test_vec_mul! { test_vec_mul_u32x4, u32x4,
6045    [0, u32::MAX - 1, 1, 2],
6046    [5, 6, 7, 8],
6047    [0, 4294967284, 7, 16] }
6048
6049    test_vec_mul! { test_vec_mul_i16x8, i16x8,
6050    [-1, 0, 1, 2, -1, 0, 1, 2],
6051    [2, 1, -1, -2, 2, 1, -1, -2],
6052    [-2, 0, -1, -4, -2, 0, -1, -4] }
6053
6054    test_vec_mul! { test_vec_mul_u16x8, u16x8,
6055    [0, u16::MAX - 1, 1, 2, 3, 4, 5, 6],
6056    [5, 6, 7, 8, 9, 8, 7, 6],
6057    [0, 65524, 7, 16, 27, 32, 35, 36] }
6058
6059    test_vec_mul! { test_vec_mul_i8x16, i8x16,
6060    [-1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2],
6061    [2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2],
6062    [-2, 0, -1, -4, -2, 0, -1, -4, -2, 0, -1, -4, -2, 0, -1, -4] }
6063
6064    test_vec_mul! { test_vec_mul_u8x16, u8x16,
6065    [0, u8::MAX - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4],
6066    [5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 0, u8::MAX, 1, 2, 3, 4],
6067    [0, 244, 7, 16, 27, 32, 35, 36, 35, 32, 0, 248, 7, 12, 15, 16] }
6068
6069    macro_rules! test_vec_abs {
6070        { $name: ident, $ty: ident, $a: expr, $d: expr } => {
6071            #[simd_test(enable = "vector")]
6072            unsafe fn $name() {
6073                let a: s_t_l!($ty) = vec_splats($a);
6074                let a: s_t_l!($ty) = vec_abs(a);
6075                let d = $ty::splat($d);
6076                assert_eq!(d, transmute(a));
6077            }
6078        }
6079    }
6080
6081    test_vec_abs! { test_vec_abs_i8, i8x16, -42i8, 42i8 }
6082    test_vec_abs! { test_vec_abs_i16, i16x8, -42i16, 42i16 }
6083    test_vec_abs! { test_vec_abs_i32, i32x4, -42i32, 42i32 }
6084    test_vec_abs! { test_vec_abs_i64, i64x2, -42i64, 42i64 }
6085    test_vec_abs! { test_vec_abs_f32, f32x4, -42f32, 42f32 }
6086    test_vec_abs! { test_vec_abs_f64, f64x2, -42f64, 42f64 }
6087
6088    test_vec_1! { test_vec_nabs, vec_nabs, f32x4,
6089    [core::f32::consts::PI, 1.0, 0.0, -1.0],
6090    [-core::f32::consts::PI, -1.0, 0.0, -1.0] }
6091
6092    test_vec_2! { test_vec_andc, vec_andc, i32x4,
6093    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6094    [0b00110011, 0b11110011, 0b00001100, 0b10000000],
6095    [0b11001100, 0b00001100, 0b11000000, 0b01001100] }
6096
6097    test_vec_2! { test_vec_and, vec_and, i32x4,
6098    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6099    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6100    [0b00000000, 0b11000000, 0b00001100, 0b00000000] }
6101
6102    test_vec_2! { test_vec_nand, vec_nand, i32x4,
6103    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6104    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6105    [!0b00000000, !0b11000000, !0b00001100, !0b00000000] }
6106
6107    test_vec_2! { test_vec_orc, vec_orc, u32x4,
6108    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6109    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6110    [0b11001100 | !0b00110011, 0b11001100 | !0b11110011, 0b11001100 | !0b00001100, 0b11001100 | !0b00000000] }
6111
6112    test_vec_2! { test_vec_or, vec_or, i32x4,
6113    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6114    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6115    [0b11111111, 0b11111111, 0b11001100, 0b11001100] }
6116
6117    test_vec_2! { test_vec_nor, vec_nor, i32x4,
6118    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6119    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6120    [!0b11111111, !0b11111111, !0b11001100, !0b11001100] }
6121
6122    test_vec_2! { test_vec_xor, vec_xor, i32x4,
6123    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6124    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6125    [0b11111111, 0b00111111, 0b11000000, 0b11001100] }
6126
6127    test_vec_2! { test_vec_eqv, vec_eqv, i32x4,
6128    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6129    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6130    [!0b11111111, !0b00111111, !0b11000000, !0b11001100] }
6131
6132    test_vec_1! { test_vec_floor_f32, vec_floor, f32x4,
6133        [1.1, 1.9, -0.5, -0.9],
6134        [1.0, 1.0, -1.0, -1.0]
6135    }
6136
6137    test_vec_1! { test_vec_floor_f64_1, vec_floor, f64x2,
6138        [1.1, 1.9],
6139        [1.0, 1.0]
6140    }
6141    test_vec_1! { test_vec_floor_f64_2, vec_floor, f64x2,
6142        [-0.5, -0.9],
6143        [-1.0, -1.0]
6144    }
6145
6146    test_vec_1! { test_vec_ceil_f32, vec_ceil, f32x4,
6147        [0.1, 0.5, 0.6, 0.9],
6148        [1.0, 1.0, 1.0, 1.0]
6149    }
6150    test_vec_1! { test_vec_ceil_f64_1, vec_ceil, f64x2,
6151        [0.1, 0.5],
6152        [1.0, 1.0]
6153    }
6154    test_vec_1! { test_vec_ceil_f64_2, vec_ceil, f64x2,
6155        [0.6, 0.9],
6156        [1.0, 1.0]
6157    }
6158
6159    test_vec_1! { test_vec_round_f32, vec_round, f32x4,
6160        [0.1, 0.5, 0.6, 0.9],
6161        [0.0, 0.0, 1.0, 1.0]
6162    }
6163
6164    test_vec_1! { test_vec_round_f32_even_odd, vec_round, f32x4,
6165        [0.5, 1.5, 2.5, 3.5],
6166        [0.0, 2.0, 2.0, 4.0]
6167    }
6168
6169    test_vec_1! { test_vec_round_f64_1, vec_round, f64x2,
6170        [0.1, 0.5],
6171        [0.0, 0.0]
6172    }
6173    test_vec_1! { test_vec_round_f64_2, vec_round, f64x2,
6174        [0.6, 0.9],
6175        [1.0, 1.0]
6176    }
6177
6178    test_vec_1! { test_vec_roundc_f32, vec_roundc, f32x4,
6179        [0.1, 0.5, 0.6, 0.9],
6180        [0.0, 0.0, 1.0, 1.0]
6181    }
6182
6183    test_vec_1! { test_vec_roundc_f32_even_odd, vec_roundc, f32x4,
6184        [0.5, 1.5, 2.5, 3.5],
6185        [0.0, 2.0, 2.0, 4.0]
6186    }
6187
6188    test_vec_1! { test_vec_roundc_f64_1, vec_roundc, f64x2,
6189        [0.1, 0.5],
6190        [0.0, 0.0]
6191    }
6192    test_vec_1! { test_vec_roundc_f64_2, vec_roundc, f64x2,
6193        [0.6, 0.9],
6194        [1.0, 1.0]
6195    }
6196
6197    test_vec_1! { test_vec_rint_f32, vec_rint, f32x4,
6198        [0.1, 0.5, 0.6, 0.9],
6199        [0.0, 0.0, 1.0, 1.0]
6200    }
6201
6202    test_vec_1! { test_vec_rint_f32_even_odd, vec_rint, f32x4,
6203        [0.5, 1.5, 2.5, 3.5],
6204        [0.0, 2.0, 2.0, 4.0]
6205    }
6206
6207    test_vec_1! { test_vec_rint_f64_1, vec_rint, f64x2,
6208        [0.1, 0.5],
6209        [0.0, 0.0]
6210    }
6211    test_vec_1! { test_vec_rint_f64_2, vec_rint, f64x2,
6212        [0.6, 0.9],
6213        [1.0, 1.0]
6214    }
6215
6216    test_vec_2! { test_vec_sll, vec_sll, i32x4, u8x16 -> i32x4,
6217    [1, 1, 1, 1],
6218    [0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 8],
6219    [1 << 2, 1 << 3, 1 << 4, 1] }
6220
6221    test_vec_2! { test_vec_srl, vec_srl, i32x4, u8x16 -> i32x4,
6222    [0b1000, 0b1000, 0b1000, 0b1000],
6223    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6224    [4, 2, 1, 8] }
6225
6226    test_vec_2! { test_vec_sral_pos, vec_sral, u32x4, u8x16 -> i32x4,
6227    [0b1000, 0b1000, 0b1000, 0b1000],
6228    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6229    [4, 2, 1, 8] }
6230
6231    test_vec_2! { test_vec_sral_neg, vec_sral, i32x4, u8x16 -> i32x4,
6232    [-8, -8, -8, -8],
6233    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6234    [-4, -2, -1, -8] }
6235
6236    test_vec_1! { test_vec_reve_f32, vec_reve, f32x4,
6237        [0.1, 0.5, 0.6, 0.9],
6238        [0.9, 0.6, 0.5, 0.1]
6239    }
6240
6241    test_vec_1! { test_vec_revb_u32, vec_revb, u32x4,
6242        [0xAABBCCDD, 0xEEFF0011, 0x22334455, 0x66778899],
6243        [0xDDCCBBAA, 0x1100FFEE, 0x55443322, 0x99887766]
6244    }
6245
6246    test_vec_2! { test_vec_mergeh_u32, vec_mergeh, u32x4,
6247        [0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD],
6248        [0x00000000, 0x11111111, 0x22222222, 0x33333333],
6249        [0xAAAAAAAA, 0x00000000, 0xBBBBBBBB, 0x11111111]
6250    }
6251
6252    test_vec_2! { test_vec_mergel_u32, vec_mergel, u32x4,
6253        [0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD],
6254        [0x00000000, 0x11111111, 0x22222222, 0x33333333],
6255        [0xCCCCCCCC, 0x22222222, 0xDDDDDDDD, 0x33333333]
6256    }
6257
6258    macro_rules! test_vec_perm {
6259        {$name:ident,
6260         $shorttype:ident, $longtype:ident,
6261         [$($a:expr),+], [$($b:expr),+], [$($c:expr),+], [$($d:expr),+]} => {
6262            #[simd_test(enable = "vector")]
6263            unsafe fn $name() {
6264                let a: $longtype = transmute($shorttype::new($($a),+));
6265                let b: $longtype = transmute($shorttype::new($($b),+));
6266                let c: vector_unsigned_char = transmute(u8x16::new($($c),+));
6267                let d = $shorttype::new($($d),+);
6268
6269                let r: $shorttype = transmute(vec_perm(a, b, c));
6270                assert_eq!(d, r);
6271            }
6272        }
6273    }
6274
6275    test_vec_perm! {test_vec_perm_u8x16,
6276    u8x16, vector_unsigned_char,
6277    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
6278    [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115],
6279    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6280     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6281    [0, 1, 100, 101, 2, 3, 102, 103, 4, 5, 104, 105, 6, 7, 106, 107]}
6282    test_vec_perm! {test_vec_perm_i8x16,
6283    i8x16, vector_signed_char,
6284    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
6285    [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115],
6286    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6287     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6288    [0, 1, 100, 101, 2, 3, 102, 103, 4, 5, 104, 105, 6, 7, 106, 107]}
6289
6290    test_vec_perm! {test_vec_perm_m8x16,
6291    m8x16, vector_bool_char,
6292    [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false],
6293    [true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true],
6294    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6295     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6296    [false, false, true, true, false, false, true, true, false, false, true, true, false, false, true, true]}
6297    test_vec_perm! {test_vec_perm_u16x8,
6298    u16x8, vector_unsigned_short,
6299    [0, 1, 2, 3, 4, 5, 6, 7],
6300    [10, 11, 12, 13, 14, 15, 16, 17],
6301    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6302     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6303    [0, 10, 1, 11, 2, 12, 3, 13]}
6304    test_vec_perm! {test_vec_perm_i16x8,
6305    i16x8, vector_signed_short,
6306    [0, 1, 2, 3, 4, 5, 6, 7],
6307    [10, 11, 12, 13, 14, 15, 16, 17],
6308    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6309     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6310    [0, 10, 1, 11, 2, 12, 3, 13]}
6311    test_vec_perm! {test_vec_perm_m16x8,
6312    m16x8, vector_bool_short,
6313    [false, false, false, false, false, false, false, false],
6314    [true, true, true, true, true, true, true, true],
6315    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6316     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6317    [false, true, false, true, false, true, false, true]}
6318
6319    test_vec_perm! {test_vec_perm_u32x4,
6320    u32x4, vector_unsigned_int,
6321    [0, 1, 2, 3],
6322    [10, 11, 12, 13],
6323    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6324     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6325    [0, 10, 1, 11]}
6326    test_vec_perm! {test_vec_perm_i32x4,
6327    i32x4, vector_signed_int,
6328    [0, 1, 2, 3],
6329    [10, 11, 12, 13],
6330    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6331     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6332    [0, 10, 1, 11]}
6333    test_vec_perm! {test_vec_perm_m32x4,
6334    m32x4, vector_bool_int,
6335    [false, false, false, false],
6336    [true, true, true, true],
6337    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6338     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6339    [false, true, false, true]}
6340    test_vec_perm! {test_vec_perm_f32x4,
6341    f32x4, vector_float,
6342    [0.0, 1.0, 2.0, 3.0],
6343    [1.0, 1.1, 1.2, 1.3],
6344    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6345     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6346    [0.0, 1.0, 1.0, 1.1]}
6347
6348    test_vec_1! { test_vec_sqrt, vec_sqrt, f32x4,
6349    [core::f32::consts::PI, 1.0, 25.0, 2.0],
6350    [core::f32::consts::PI.sqrt(), 1.0, 5.0, core::f32::consts::SQRT_2] }
6351
6352    test_vec_2! { test_vec_find_any_eq, vec_find_any_eq, i32x4, i32x4 -> u32x4,
6353        [1, -2, 3, -4],
6354        [-5, 3, -7, 8],
6355        [0, 0, 0xFFFFFFFF, 0]
6356    }
6357
6358    test_vec_2! { test_vec_find_any_ne, vec_find_any_ne, i32x4, i32x4 -> u32x4,
6359        [1, -2, 3, -4],
6360        [-5, 3, -7, 8],
6361        [0xFFFFFFFF, 0xFFFFFFFF, 0, 0xFFFFFFFF]
6362    }
6363
6364    test_vec_2! { test_vec_find_any_eq_idx_1, vec_find_any_eq_idx, i32x4, i32x4 -> u32x4,
6365        [1, 2, 3, 4],
6366        [5, 3, 7, 8],
6367        [0, 8, 0, 0]
6368    }
6369    test_vec_2! { test_vec_find_any_eq_idx_2, vec_find_any_eq_idx, i32x4, i32x4 -> u32x4,
6370        [1, 2, 3, 4],
6371        [5, 6, 7, 8],
6372        [0, 16, 0, 0]
6373    }
6374
6375    test_vec_2! { test_vec_find_any_ne_idx_1, vec_find_any_ne_idx, i32x4, i32x4 -> u32x4,
6376        [1, 2, 3, 4],
6377        [1, 5, 3, 4],
6378        [0, 4, 0, 0]
6379    }
6380    test_vec_2! { test_vec_find_any_ne_idx_2, vec_find_any_ne_idx, i32x4, i32x4 -> u32x4,
6381        [1, 2, 3, 4],
6382        [1, 2, 3, 4],
6383        [0, 16, 0, 0]
6384    }
6385
6386    test_vec_2! { test_vec_find_any_eq_or_0_idx_1, vec_find_any_eq_or_0_idx, i32x4, i32x4 -> u32x4,
6387        [1, 2, 0, 4],
6388        [5, 6, 7, 8],
6389        [0, 8, 0, 0]
6390    }
6391    test_vec_2! { test_vec_find_any_ne_or_0_idx_1, vec_find_any_ne_or_0_idx, i32x4, i32x4 -> u32x4,
6392        [1, 2, 0, 4],
6393        [1, 2, 3, 4],
6394        [0, 8, 0, 0]
6395    }
6396
6397    #[simd_test(enable = "vector")]
6398    fn test_vec_find_any_eq_cc() {
6399        let a = vector_unsigned_int([1, 2, 3, 4]);
6400        let b = vector_unsigned_int([5, 3, 7, 8]);
6401
6402        let (d, c) = unsafe { vec_find_any_eq_cc(a, b) };
6403        assert_eq!(c, 1);
6404        assert_eq!(d.as_array(), &[0, 0, -1, 0]);
6405
6406        let a = vector_unsigned_int([1, 2, 3, 4]);
6407        let b = vector_unsigned_int([5, 6, 7, 8]);
6408        let (d, c) = unsafe { vec_find_any_eq_cc(a, b) };
6409        assert_eq!(c, 3);
6410        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6411    }
6412
6413    #[simd_test(enable = "vector")]
6414    fn test_vec_find_any_ne_cc() {
6415        let a = vector_unsigned_int([1, 2, 3, 4]);
6416        let b = vector_unsigned_int([5, 3, 7, 8]);
6417
6418        let (d, c) = unsafe { vec_find_any_ne_cc(a, b) };
6419        assert_eq!(c, 1);
6420        assert_eq!(d.as_array(), &[-1, -1, 0, -1]);
6421
6422        let a = vector_unsigned_int([1, 2, 3, 4]);
6423        let b = vector_unsigned_int([1, 2, 3, 4]);
6424        let (d, c) = unsafe { vec_find_any_ne_cc(a, b) };
6425        assert_eq!(c, 3);
6426        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6427    }
6428
6429    #[simd_test(enable = "vector")]
6430    fn test_vec_find_any_eq_idx_cc() {
6431        let a = vector_unsigned_int([1, 2, 3, 4]);
6432        let b = vector_unsigned_int([5, 3, 7, 8]);
6433
6434        let (d, c) = unsafe { vec_find_any_eq_idx_cc(a, b) };
6435        assert_eq!(c, 1);
6436        assert_eq!(d.as_array(), &[0, 8, 0, 0]);
6437
6438        let a = vector_unsigned_int([1, 2, 3, 4]);
6439        let b = vector_unsigned_int([5, 6, 7, 8]);
6440        let (d, c) = unsafe { vec_find_any_eq_idx_cc(a, b) };
6441        assert_eq!(c, 3);
6442        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6443    }
6444
6445    #[simd_test(enable = "vector")]
6446    fn test_vec_find_any_ne_idx_cc() {
6447        let a = vector_unsigned_int([5, 2, 3, 4]);
6448        let b = vector_unsigned_int([5, 3, 7, 8]);
6449
6450        let (d, c) = unsafe { vec_find_any_ne_idx_cc(a, b) };
6451        assert_eq!(c, 1);
6452        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6453
6454        let a = vector_unsigned_int([1, 2, 3, 4]);
6455        let b = vector_unsigned_int([1, 2, 3, 4]);
6456        let (d, c) = unsafe { vec_find_any_ne_idx_cc(a, b) };
6457        assert_eq!(c, 3);
6458        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6459    }
6460
6461    #[simd_test(enable = "vector")]
6462    fn test_vec_find_any_eq_or_0_idx_cc() {
6463        // if no element of a matches any element of b with an equal value, and there is at least one element from a with a value of 0
6464        let a = vector_unsigned_int([0, 1, 2, 3]);
6465        let b = vector_unsigned_int([4, 5, 6, 7]);
6466        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6467        assert_eq!(c, 0);
6468        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6469
6470        // if at least one element of a matches any element of b with an equal value, and no elements of a with a value of 0
6471        let a = vector_unsigned_int([1, 2, 3, 4]);
6472        let b = vector_unsigned_int([5, 2, 3, 4]);
6473        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6474        assert_eq!(c, 1);
6475        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6476
6477        // if at least one element of a matches any element of b with an equal value, and there is at least one element from a has a value of 0
6478        let a = vector_unsigned_int([1, 2, 3, 0]);
6479        let b = vector_unsigned_int([1, 2, 3, 4]);
6480        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6481        assert_eq!(c, 2);
6482        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6483
6484        // if no element of a matches any element of b with an equal value, and there is no element from a with a value of 0.
6485        let a = vector_unsigned_int([1, 2, 3, 4]);
6486        let b = vector_unsigned_int([5, 6, 7, 8]);
6487        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6488        assert_eq!(c, 3);
6489        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6490    }
6491
6492    #[simd_test(enable = "vector")]
6493    fn test_vec_find_any_ne_or_0_idx_cc() {
6494        // if no element of a matches any element of b with a not equal value, and there is at least one element from a with a value of 0.
6495        let a = vector_unsigned_int([0, 1, 2, 3]);
6496        let b = vector_unsigned_int([4, 1, 2, 3]);
6497        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6498        assert_eq!(c, 0);
6499        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6500
6501        // if at least one element of a matches any element of b with a not equal value, and no elements of a with a value of 0.
6502        let a = vector_unsigned_int([4, 2, 3, 4]);
6503        let b = vector_unsigned_int([4, 5, 6, 7]);
6504        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6505        assert_eq!(c, 1);
6506        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6507
6508        // if at least one element of a matches any element of b with a not equal value, and there is at least one element from a has a value of 0.
6509        let a = vector_unsigned_int([1, 0, 1, 1]);
6510        let b = vector_unsigned_int([4, 5, 6, 7]);
6511        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6512        assert_eq!(c, 2);
6513        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6514
6515        // if no element of a matches any element of b with a not equal value, and there is no element from a with a value of 0.
6516        let a = vector_unsigned_int([4, 4, 4, 4]);
6517        let b = vector_unsigned_int([4, 5, 6, 7]);
6518        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6519        assert_eq!(c, 3);
6520        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6521    }
6522
6523    #[simd_test(enable = "vector")]
6524    fn test_vector_load() {
6525        let expected = [0xAAAA_AAAA, 0xBBBB_BBBB, 0xCCCC_CCCC, 0xDDDD_DDDD];
6526
6527        let source: [u32; 8] = [
6528            0xAAAA_AAAA,
6529            0xBBBB_BBBB,
6530            0xCCCC_CCCC,
6531            0xDDDD_DDDD,
6532            0,
6533            0,
6534            0,
6535            0,
6536        ];
6537        assert_eq!(
6538            unsafe { vec_xl::<vector_unsigned_int>(0, source.as_ptr()) }.as_array(),
6539            &expected
6540        );
6541
6542        // offset is in bytes
6543        let source: [u32; 8] = [
6544            0x0000_AAAA,
6545            0xAAAA_BBBB,
6546            0xBBBB_CCCC,
6547            0xCCCC_DDDD,
6548            0xDDDD_0000,
6549            0,
6550            0,
6551            0,
6552        ];
6553        assert_eq!(
6554            unsafe { vec_xl::<vector_unsigned_int>(2, source.as_ptr()) }.as_array(),
6555            &expected
6556        );
6557    }
6558
6559    #[simd_test(enable = "vector")]
6560    fn test_vector_store() {
6561        let vec = vector_unsigned_int([0xAAAA_AAAA, 0xBBBB_BBBB, 0xCCCC_CCCC, 0xDDDD_DDDD]);
6562
6563        let mut dest = [0u32; 8];
6564        unsafe { vec_xst(vec, 0, dest.as_mut_ptr()) };
6565        assert_eq!(
6566            dest,
6567            [
6568                0xAAAA_AAAA,
6569                0xBBBB_BBBB,
6570                0xCCCC_CCCC,
6571                0xDDDD_DDDD,
6572                0,
6573                0,
6574                0,
6575                0
6576            ]
6577        );
6578
6579        // offset is in bytes
6580        let mut dest = [0u32; 8];
6581        unsafe { vec_xst(vec, 2, dest.as_mut_ptr()) };
6582        assert_eq!(
6583            dest,
6584            [
6585                0x0000_AAAA,
6586                0xAAAA_BBBB,
6587                0xBBBB_CCCC,
6588                0xCCCC_DDDD,
6589                0xDDDD_0000,
6590                0,
6591                0,
6592                0,
6593            ]
6594        );
6595    }
6596
6597    #[simd_test(enable = "vector")]
6598    fn test_vector_lcbb() {
6599        #[repr(align(64))]
6600        struct Align64<T>(T);
6601
6602        static ARRAY: Align64<[u8; 128]> = Align64([0; 128]);
6603
6604        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[64..].as_ptr()) }, 16);
6605        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[63..].as_ptr()) }, 1);
6606        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[56..].as_ptr()) }, 8);
6607        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[48..].as_ptr()) }, 16);
6608    }
6609
6610    test_vec_2! { test_vec_pack, vec_pack, i16x8, i16x8 -> i8x16,
6611        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6612        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6613        [0, 1, -1, 42, -1, 0, 48, -48, -1, 0, 57, -57, 0, 1, -1, 42]
6614    }
6615
6616    test_vec_2! { test_vec_packs, vec_packs, i16x8, i16x8 -> i8x16,
6617        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6618        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6619        [0, 1, -1, 42, 127, -128, 127, -128, 127, -128, 127, -128, 0, 1, -1, 42]
6620    }
6621
6622    test_vec_2! { test_vec_packsu_signed, vec_packsu, i16x8, i16x8 -> u8x16,
6623        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6624        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6625        [0, 1, 0, 42, 255, 0, 255, 0, 255, 0, 255, 0, 0, 1, 0, 42]
6626    }
6627
6628    test_vec_2! { test_vec_packsu_unsigned, vec_packsu, u16x8, u16x8 -> u8x16,
6629        [65535, 32768, 1234, 5678, 16, 8, 4, 2],
6630        [30000, 25000, 20000, 15000, 31, 63, 127, 255],
6631        [255, 255, 255, 255, 16, 8, 4, 2, 255, 255, 255, 255, 31, 63, 127, 255]
6632    }
6633
6634    test_vec_2! { test_vec_rl, vec_rl, u32x4,
6635        [0x12345678, 0x9ABCDEF0, 0x0F0F0F0F, 0x12345678],
6636        [4, 8, 12, 68],
6637        [0x23456781, 0xBCDEF09A, 0xF0F0F0F0, 0x23456781]
6638    }
6639
6640    test_vec_1! { test_vec_unpackh_i, vec_unpackh, i16x8 -> i32x4,
6641        [0x1234, -2, 0x0F0F, -32768, 0, 0, 0, 0],
6642        [0x1234, -2, 0x0F0F, -32768]
6643    }
6644
6645    test_vec_1! { test_vec_unpackh_u, vec_unpackh, u16x8 -> u32x4,
6646        [0x1234, 0xFFFF, 0x0F0F, 0x8000, 0, 0, 0, 0],
6647        [0x1234, 0xFFFF, 0x0F0F, 0x8000]
6648    }
6649
6650    test_vec_1! { test_vec_unpackl_i, vec_unpackl, i16x8 -> i32x4,
6651        [0, 0, 0, 0, 0x1234, -2, 0x0F0F, -32768],
6652        [0x1234, -2, 0x0F0F, -32768]
6653    }
6654
6655    test_vec_1! { test_vec_unpackl_u, vec_unpackl, u16x8 -> u32x4,
6656        [0, 0, 0, 0, 0x1234, 0xFFFF, 0x0F0F, 0x8000],
6657        [0x1234, 0xFFFF, 0x0F0F, 0x8000]
6658    }
6659
6660    test_vec_2! { test_vec_avg, vec_avg, u32x4,
6661        [2, 1, u32::MAX, 0],
6662        [4, 2, 2, 0],
6663        [3, (1u32 + 2).div_ceil(2), (u32::MAX as u64 + 2u64).div_ceil(2) as u32, 0]
6664    }
6665
6666    test_vec_2! { test_vec_checksum, vec_checksum, u32x4,
6667        [1, 2, 3, u32::MAX],
6668        [5, 6, 7, 8],
6669        [0, 12, 0, 0]
6670    }
6671
6672    test_vec_2! { test_vec_add_u128, vec_add_u128, u8x16,
6673        [0x01, 0x05, 0x0F, 0x1A, 0x2F, 0x3F, 0x50, 0x65,
6674                              0x7A, 0x8F, 0x9A, 0xAD, 0xB0, 0xC3, 0xD5, 0xE8],
6675        [0xF0, 0xEF, 0xC3, 0xB1, 0x92, 0x71, 0x5A, 0x43,
6676                              0x3B, 0x29, 0x13, 0x04, 0xD7, 0xA1, 0x8C, 0x76],
6677        [0xF1, 0xF4, 0xD2, 0xCB, 0xC1, 0xB0, 0xAA, 0xA8, 0xB5, 0xB8, 0xAD, 0xB2, 0x88, 0x65, 0x62, 0x5E]
6678    }
6679
6680    #[simd_test(enable = "vector")]
6681    fn test_vec_addc_u128() {
6682        unsafe {
6683            let a = u128::MAX;
6684            let b = 1u128;
6685
6686            let d: u128 = transmute(vec_addc_u128(transmute(a), transmute(b)));
6687            assert!(a.checked_add(b).is_none());
6688            assert_eq!(d, 1);
6689
6690            let a = 1u128;
6691            let b = 1u128;
6692
6693            let d: u128 = transmute(vec_addc_u128(transmute(a), transmute(b)));
6694            assert!(a.checked_add(b).is_some());
6695            assert_eq!(d, 0);
6696        }
6697    }
6698
6699    #[simd_test(enable = "vector")]
6700    fn test_vec_subc_u128() {
6701        unsafe {
6702            let a = 0u128;
6703            let b = 1u128;
6704
6705            let d: u128 = transmute(vec_subc_u128(transmute(a), transmute(b)));
6706            assert!(a.checked_sub(b).is_none());
6707            assert_eq!(d, 0);
6708
6709            let a = 1u128;
6710            let b = 1u128;
6711
6712            let d: u128 = transmute(vec_subc_u128(transmute(a), transmute(b)));
6713            assert!(a.checked_sub(b).is_some());
6714            assert_eq!(d, 1);
6715        }
6716    }
6717
6718    test_vec_2! { test_vec_mule_u, vec_mule, u16x8, u16x8 -> u32x4,
6719        [0xFFFF, 0, 2, 0, 2, 0, 1, 0],
6720        [0xFFFF, 0, 4, 0, 0xFFFF, 0, 2, 0],
6721        [0xFFFE_0001, 8, 0x0001_FFFE, 2]
6722    }
6723
6724    test_vec_2! { test_vec_mule_i, vec_mule, i16x8, i16x8 -> i32x4,
6725        [i16::MIN, 0, -2, 0, 2, 0, 1, 0],
6726        [i16::MIN, 0, 4, 0, i16::MAX, 0, 2, 0],
6727        [0x4000_0000, -8, 0xFFFE, 2]
6728    }
6729
6730    test_vec_2! { test_vec_mulo_u, vec_mulo, u16x8, u16x8 -> u32x4,
6731        [0, 0xFFFF, 0, 2, 0, 2, 0, 1],
6732        [0, 0xFFFF, 0, 4, 0, 0xFFFF, 0, 2],
6733        [0xFFFE_0001, 8, 0x0001_FFFE, 2]
6734    }
6735
6736    test_vec_2! { test_vec_mulo_i, vec_mulo, i16x8, i16x8 -> i32x4,
6737        [0, i16::MIN, 0, -2, 0, 2, 0, 1],
6738        [0, i16::MIN, 0, 4, 0, i16::MAX, 0, 2],
6739        [0x4000_0000, -8, 0xFFFE, 2]
6740    }
6741
6742    test_vec_2! { test_vec_mulh_u, vec_mulh, u32x4, u32x4 -> u32x4,
6743        [u32::MAX, 2, 2, 1],
6744        [u32::MAX, 4, u32::MAX, 2],
6745        [u32::MAX - 1, 0, 1, 0]
6746    }
6747
6748    test_vec_2! { test_vec_mulh_i, vec_mulh, i32x4, i32x4 -> i32x4,
6749        [i32::MIN, -2, 2, 1],
6750        [i32::MIN, 4, i32::MAX, 2],
6751        [0x4000_0000, -1, 0, 0]
6752    }
6753
6754    test_vec_2! { test_vec_gfmsum_1, vec_gfmsum, u16x8, u16x8 -> u32x4,
6755        [0x1234, 0x5678, 0x9ABC, 0xDEF0, 0x1357, 0x2468, 0xACE0, 0xBDF0],
6756        [0xFFFF, 0x0001, 0x8000, 0x7FFF, 0xAAAA, 0x5555, 0x1234, 0x5678],
6757        [0xE13A794, 0x68764A50, 0x94AA3E, 0x2C93F300]
6758    }
6759
6760    test_vec_2! { test_vec_gfmsum_2, vec_gfmsum, u16x8, u16x8 -> u32x4,
6761        [0x0000, 0xFFFF, 0xAAAA, 0x5555, 0x1234, 0x5678, 0x9ABC, 0xDEF0],
6762        [0xFFFF, 0x0000, 0x5555, 0xAAAA, 0x0001, 0x8000, 0x7FFF, 0x1357],
6763        [0, 0, 0x2B3C1234, 0x3781D244]
6764    }
6765
6766    #[simd_test(enable = "vector")]
6767    fn test_vec_gfmsum_128() {
6768        let a = vector_unsigned_long_long([1, 2]);
6769        let b = vector_unsigned_long_long([3, 4]);
6770
6771        let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) };
6772        assert_eq!(d, 11);
6773
6774        let a = vector_unsigned_long_long([0x0101010101010101, 0x0202020202020202]);
6775        let b = vector_unsigned_long_long([0x0404040404040404, 0x0505050505050505]);
6776
6777        let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) };
6778        assert_eq!(d, 0xE000E000E000E000E000E000E000E);
6779    }
6780
6781    #[simd_test(enable = "vector-enhancements-1")]
6782    fn test_vec_bperm_u128() {
6783        let a = vector_unsigned_char([65, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]);
6784        let b = vector_unsigned_char([
6785            0, 0, 0, 0, 1, 1, 1, 1, 128, 128, 128, 128, 255, 255, 255, 255,
6786        ]);
6787        let d = unsafe { vec_bperm_u128(a, b) };
6788        assert_eq!(d.as_array(), &[0xF00, 0]);
6789    }
6790
6791    #[simd_test(enable = "vector")]
6792    fn test_vec_sel() {
6793        let a = vector_signed_int([1, 2, 3, 4]);
6794        let b = vector_signed_int([5, 6, 7, 8]);
6795
6796        let e = vector_unsigned_int([9, 10, 11, 12]);
6797        let f = vector_unsigned_int([9, 9, 11, 11]);
6798
6799        let c: vector_bool_int = unsafe { simd_eq(e, f) };
6800        assert_eq!(c.as_array(), &[!0, 0, !0, 0]);
6801        let d: vector_signed_int = unsafe { vec_sel(a, b, c) };
6802        assert_eq!(d.as_array(), &[5, 2, 7, 4]);
6803    }
6804
6805    #[simd_test(enable = "vector")]
6806    fn test_vec_gather_element() {
6807        let a1: [u32; 10] = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19];
6808        let a2: [u32; 10] = [20, 21, 22, 23, 24, 25, 26, 27, 28, 29];
6809
6810        let v1 = vector_unsigned_int([1, 2, 3, 4]);
6811        let v2 = vector_unsigned_int([1, 2, 3, 4]);
6812
6813        let sizeof_int = core::mem::size_of::<u32>() as u32;
6814        let v3 = vector_unsigned_int([
6815            5 * sizeof_int,
6816            8 * sizeof_int,
6817            9 * sizeof_int,
6818            6 * sizeof_int,
6819        ]);
6820
6821        unsafe {
6822            let d1 = vec_gather_element::<_, 0>(v1, v3, a1.as_ptr());
6823            assert_eq!(d1.as_array(), &[15, 2, 3, 4]);
6824            let d2 = vec_gather_element::<_, 0>(v2, v3, a2.as_ptr());
6825            assert_eq!(d2.as_array(), &[25, 2, 3, 4]);
6826        }
6827    }
6828
6829    #[simd_test(enable = "vector")]
6830    fn test_vec_fp_test_data_class() {
6831        let mut cc = 42;
6832
6833        let v1 = vector_double([0.0, f64::NAN]);
6834        let v2 = vector_double([f64::INFINITY, 1.0]);
6835        let v3 = vector_double([1.0, 2.0]);
6836
6837        unsafe {
6838            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_ZERO>(v1, &mut cc);
6839            assert_eq!(cc, 1);
6840            assert_eq!(d.as_array(), &[!0, 0]);
6841
6842            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NAN>(v1, &mut cc);
6843            assert_eq!(cc, 1);
6844            assert_eq!(d.as_array(), &[0, !0]);
6845
6846            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY>(v2, &mut cc);
6847            assert_eq!(cc, 1);
6848            assert_eq!(d.as_array(), &[!0, 0]);
6849
6850            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY_N>(v2, &mut cc);
6851            assert_eq!(cc, 3);
6852            assert_eq!(d.as_array(), &[0, 0]);
6853
6854            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v2, &mut cc);
6855            assert_eq!(cc, 1);
6856            assert_eq!(d.as_array(), &[0, !0]);
6857
6858            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v3, &mut cc);
6859            assert_eq!(cc, 0);
6860            assert_eq!(d.as_array(), &[!0, !0]);
6861        }
6862    }
6863
6864    #[simd_test(enable = "vector")]
6865    fn test_vec_fp_any_all_nan_numeric() {
6866        unsafe {
6867            assert_eq!(
6868                vec_all_nan(vector_double([f64::NAN, f64::NAN])),
6869                i32::from(true)
6870            );
6871            assert_eq!(
6872                vec_all_nan(vector_double([f64::NAN, 1.0])),
6873                i32::from(false)
6874            );
6875            assert_eq!(vec_all_nan(vector_double([0.0, 1.0])), i32::from(false));
6876
6877            assert_eq!(
6878                vec_any_nan(vector_double([f64::NAN, f64::NAN])),
6879                i32::from(true)
6880            );
6881            assert_eq!(vec_any_nan(vector_double([f64::NAN, 1.0])), i32::from(true));
6882            assert_eq!(vec_any_nan(vector_double([0.0, 1.0])), i32::from(false));
6883
6884            assert_eq!(
6885                vec_all_numeric(vector_double([f64::NAN, f64::NAN])),
6886                i32::from(false)
6887            );
6888            assert_eq!(
6889                vec_all_numeric(vector_double([f64::NAN, 1.0])),
6890                i32::from(false)
6891            );
6892            assert_eq!(vec_all_numeric(vector_double([0.0, 1.0])), i32::from(true));
6893
6894            assert_eq!(
6895                vec_any_numeric(vector_double([f64::NAN, f64::NAN])),
6896                i32::from(false)
6897            );
6898            assert_eq!(
6899                vec_any_numeric(vector_double([f64::NAN, 1.0])),
6900                i32::from(true)
6901            );
6902            assert_eq!(vec_any_numeric(vector_double([0.0, 1.0])), i32::from(true));
6903
6904            // "numeric" means "not NaN". infinities are numeric
6905            assert_eq!(
6906                vec_all_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])),
6907                i32::from(true)
6908            );
6909            assert_eq!(
6910                vec_any_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])),
6911                i32::from(true)
6912            );
6913        }
6914    }
6915
6916    #[simd_test(enable = "vector")]
6917    fn test_vec_test_mask() {
6918        unsafe {
6919            let v = vector_unsigned_long_long([0xFF00FF00FF00FF00; 2]);
6920            let m = vector_unsigned_long_long([0x0000FF000000FF00; 2]);
6921            assert_eq!(vec_test_mask(v, m), 3);
6922
6923            let v = vector_unsigned_long_long([u64::MAX; 2]);
6924            let m = vector_unsigned_long_long([0; 2]);
6925            assert_eq!(vec_test_mask(v, m), 0);
6926
6927            let v = vector_unsigned_long_long([0; 2]);
6928            let m = vector_unsigned_long_long([u64::MAX; 2]);
6929            assert_eq!(vec_test_mask(v, m), 0);
6930
6931            let v = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]);
6932            let m = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]);
6933            assert_eq!(vec_test_mask(v, m), 3);
6934        }
6935    }
6936
6937    #[simd_test(enable = "vector-enhancements-2")]
6938    fn test_vec_search_string_cc() {
6939        unsafe {
6940            let b = vector_unsigned_char(*b"ABCD------------");
6941            let c = vector_unsigned_char([4; 16]);
6942
6943            let haystack = vector_unsigned_char(*b"__ABCD__________");
6944            let (result, d) = vec_search_string_cc(haystack, b, c);
6945            assert_eq!(result.as_array()[7], 2);
6946            assert_eq!(d, 2);
6947
6948            let haystack = vector_unsigned_char(*b"___ABCD_________");
6949            let (result, d) = vec_search_string_cc(haystack, b, c);
6950            assert_eq!(result.as_array()[7], 3);
6951            assert_eq!(d, 2);
6952
6953            let haystack = vector_unsigned_char(*b"________________");
6954            let (result, d) = vec_search_string_cc(haystack, b, c);
6955            assert_eq!(result.as_array()[7], 16);
6956            assert_eq!(d, 0);
6957
6958            let haystack = vector_unsigned_char(*b"______\0_________");
6959            let (result, d) = vec_search_string_cc(haystack, b, c);
6960            assert_eq!(result.as_array()[7], 16);
6961            assert_eq!(d, 0);
6962
6963            let haystack = vector_unsigned_char(*b"______\0__ABCD___");
6964            let (result, d) = vec_search_string_cc(haystack, b, c);
6965            assert_eq!(result.as_array()[7], 9);
6966            assert_eq!(d, 2);
6967        }
6968    }
6969
6970    #[simd_test(enable = "vector-enhancements-2")]
6971    fn test_vec_search_string_until_zero_cc() {
6972        unsafe {
6973            let b = vector_unsigned_char(*b"ABCD\0\0\0\0\0\0\0\0\0\0\0\0");
6974            let c = vector_unsigned_char([16; 16]);
6975
6976            let haystack = vector_unsigned_char(*b"__ABCD__________");
6977            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6978            assert_eq!(result.as_array()[7], 2);
6979            assert_eq!(d, 2);
6980
6981            let haystack = vector_unsigned_char(*b"___ABCD_________");
6982            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6983            assert_eq!(result.as_array()[7], 3);
6984            assert_eq!(d, 2);
6985
6986            let haystack = vector_unsigned_char(*b"________________");
6987            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6988            assert_eq!(result.as_array()[7], 16);
6989            assert_eq!(d, 0);
6990
6991            let haystack = vector_unsigned_char(*b"______\0_________");
6992            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6993            assert_eq!(result.as_array()[7], 16);
6994            assert_eq!(d, 1);
6995
6996            let haystack = vector_unsigned_char(*b"______\0__ABCD___");
6997            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6998            assert_eq!(result.as_array()[7], 16);
6999            assert_eq!(d, 1);
7000        }
7001    }
7002
7003    #[simd_test(enable = "vector")]
7004    fn test_vec_doublee() {
7005        unsafe {
7006            let v = vector_float([1.0, 2.0, 3.0, 4.0]);
7007            assert_eq!(vec_doublee(v).as_array(), &[1.0, 3.0]);
7008
7009            let v = vector_float([f32::NAN, 2.0, f32::INFINITY, 4.0]);
7010            let d = vec_doublee(v);
7011            assert!(d.as_array()[0].is_nan());
7012            assert_eq!(d.as_array()[1], f64::INFINITY);
7013        }
7014    }
7015
7016    #[simd_test(enable = "vector")]
7017    fn test_vec_floate() {
7018        // NOTE: indices 1 and 3 can have an arbitrary value. With the C version
7019        // these are poison values, our version initializes the memory but its
7020        // value still should not be relied upon by application code.
7021        unsafe {
7022            let v = vector_double([1.0, 2.0]);
7023            let d = vec_floate(v);
7024            assert_eq!(d.as_array()[0], 1.0);
7025            assert_eq!(d.as_array()[2], 2.0);
7026
7027            let v = vector_double([f64::NAN, f64::INFINITY]);
7028            let d = vec_floate(v);
7029            assert!(d.as_array()[0].is_nan());
7030            assert_eq!(d.as_array()[2], f32::INFINITY);
7031
7032            let v = vector_double([f64::MIN, f64::MAX]);
7033            let d = vec_floate(v);
7034            assert_eq!(d.as_array()[0], f64::MIN as f32);
7035            assert_eq!(d.as_array()[2], f64::MAX as f32);
7036        }
7037    }
7038
7039    #[simd_test(enable = "vector")]
7040    fn test_vec_extend_s64() {
7041        unsafe {
7042            let v = vector_signed_char([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
7043            assert_eq!(vec_extend_s64(v).as_array(), &[7, 15]);
7044
7045            let v = vector_signed_short([0, 1, 2, 3, 4, 5, 6, 7]);
7046            assert_eq!(vec_extend_s64(v).as_array(), &[3, 7]);
7047
7048            let v = vector_signed_int([0, 1, 2, 3]);
7049            assert_eq!(vec_extend_s64(v).as_array(), &[1, 3]);
7050        }
7051    }
7052
7053    #[simd_test(enable = "vector")]
7054    fn test_vec_signed() {
7055        unsafe {
7056            let v = vector_float([1.0, 2.5, -2.5, -0.0]);
7057            assert_eq!(vec_signed(v).as_array(), &[1, 2, -2, 0]);
7058
7059            let v = vector_double([2.5, -2.5]);
7060            assert_eq!(vec_signed(v).as_array(), &[2, -2]);
7061        }
7062    }
7063
7064    #[simd_test(enable = "vector")]
7065    fn test_vec_unsigned() {
7066        // NOTE: converting a negative floating point value is UB!
7067        unsafe {
7068            let v = vector_float([1.0, 2.5, 3.5, 0.0]);
7069            assert_eq!(vec_unsigned(v).as_array(), &[1, 2, 3, 0]);
7070
7071            let v = vector_double([2.5, 3.5]);
7072            assert_eq!(vec_unsigned(v).as_array(), &[2, 3]);
7073        }
7074    }
7075
7076    #[simd_test(enable = "vector")]
7077    fn test_vec_cp_until_zero() {
7078        unsafe {
7079            let v = vector_signed_int([1, 2, 3, 4]);
7080            let d = vec_cp_until_zero(v);
7081            assert_eq!(d.as_array(), &[1, 2, 3, 4]);
7082
7083            let v = vector_signed_int([1, 2, 0, 4]);
7084            let d = vec_cp_until_zero(v);
7085            assert_eq!(d.as_array(), &[1, 2, 0, 0]);
7086        }
7087    }
7088
7089    #[simd_test(enable = "vector")]
7090    fn test_vec_cp_until_zero_cc() {
7091        unsafe {
7092            let v = vector_signed_int([1, 2, 3, 4]);
7093            let (d, cc) = vec_cp_until_zero_cc(v);
7094            assert_eq!(d.as_array(), &[1, 2, 3, 4]);
7095            assert_eq!(cc, 3);
7096
7097            let v = vector_signed_int([1, 2, 0, 4]);
7098            let (d, cc) = vec_cp_until_zero_cc(v);
7099            assert_eq!(d.as_array(), &[1, 2, 0, 0]);
7100            assert_eq!(cc, 0);
7101        }
7102    }
7103
7104    #[simd_test(enable = "vector-enhancements-1")]
7105    fn test_vec_msum_u128() {
7106        let a = vector_unsigned_long_long([1, 2]);
7107        let b = vector_unsigned_long_long([3, 4]);
7108
7109        unsafe {
7110            let c: vector_unsigned_char = transmute(100u128);
7111
7112            let d: u128 = transmute(vec_msum_u128::<0>(a, b, c));
7113            assert_eq!(d, (1 * 3) + (2 * 4) + 100);
7114
7115            let d: u128 = transmute(vec_msum_u128::<4>(a, b, c));
7116            assert_eq!(d, (1 * 3) + (2 * 4) * 2 + 100);
7117
7118            let d: u128 = transmute(vec_msum_u128::<8>(a, b, c));
7119            assert_eq!(d, (1 * 3) * 2 + (2 * 4) + 100);
7120
7121            let d: u128 = transmute(vec_msum_u128::<12>(a, b, c));
7122            assert_eq!(d, (1 * 3) * 2 + (2 * 4) * 2 + 100);
7123        }
7124    }
7125
7126    #[simd_test(enable = "vector")]
7127    fn test_vec_sld() {
7128        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7129        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7130
7131        unsafe {
7132            let d = vec_sld::<_, 4>(a, b);
7133            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]);
7134        }
7135    }
7136
7137    #[simd_test(enable = "vector")]
7138    fn test_vec_sldw() {
7139        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7140        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7141
7142        unsafe {
7143            let d = vec_sldw::<_, 1>(a, b);
7144            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]);
7145        }
7146    }
7147
7148    #[simd_test(enable = "vector-enhancements-2")]
7149    fn test_vec_sldb() {
7150        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7151        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7152
7153        unsafe {
7154            let d = vec_sldb::<_, 4>(a, b);
7155            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAB]);
7156        }
7157    }
7158
7159    #[simd_test(enable = "vector-enhancements-2")]
7160    fn test_vec_srdb() {
7161        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7162        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7163
7164        unsafe {
7165            let d = vec_srdb::<_, 4>(a, b);
7166            assert_eq!(d.as_array(), &[0xABBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7167        }
7168    }
7169
7170    const GT: u32 = 0x20000000;
7171    const LT: u32 = 0x40000000;
7172    const EQ: u32 = 0x80000000;
7173
7174    #[simd_test(enable = "vector")]
7175    fn test_vec_cmprg() {
7176        let a = vector_unsigned_int([11, 22, 33, 44]);
7177        let b = vector_unsigned_int([10, 20, 30, 40]);
7178
7179        let c = vector_unsigned_int([GT, LT, GT, LT]);
7180        let d = unsafe { vec_cmprg(a, b, c) };
7181        assert_eq!(d.as_array(), &[!0, 0, !0, 0]);
7182
7183        let c = vector_unsigned_int([GT, LT, 0, 0]);
7184        let d = unsafe { vec_cmprg(a, b, c) };
7185        assert_eq!(d.as_array(), &[!0, 0, 0, 0]);
7186
7187        let a = vector_unsigned_int([11, 22, 33, 30]);
7188        let b = vector_unsigned_int([10, 20, 30, 30]);
7189
7190        let c = vector_unsigned_int([GT, LT, EQ, EQ]);
7191        let d = unsafe { vec_cmprg(a, b, c) };
7192        assert_eq!(d.as_array(), &[!0, 0, 0, !0]);
7193    }
7194
7195    #[simd_test(enable = "vector")]
7196    fn test_vec_cmpnrg() {
7197        let a = vector_unsigned_int([11, 22, 33, 44]);
7198        let b = vector_unsigned_int([10, 20, 30, 40]);
7199
7200        let c = vector_unsigned_int([GT, LT, GT, LT]);
7201        let d = unsafe { vec_cmpnrg(a, b, c) };
7202        assert_eq!(d.as_array(), &[0, !0, 0, !0]);
7203
7204        let c = vector_unsigned_int([GT, LT, 0, 0]);
7205        let d = unsafe { vec_cmpnrg(a, b, c) };
7206        assert_eq!(d.as_array(), &[0, !0, !0, !0]);
7207
7208        let a = vector_unsigned_int([11, 22, 33, 30]);
7209        let b = vector_unsigned_int([10, 20, 30, 30]);
7210
7211        let c = vector_unsigned_int([GT, LT, EQ, EQ]);
7212        let d = unsafe { vec_cmpnrg(a, b, c) };
7213        assert_eq!(d.as_array(), &[0, !0, !0, 0]);
7214    }
7215
7216    #[simd_test(enable = "vector")]
7217    fn test_vec_cmprg_idx() {
7218        let a = vector_unsigned_int([1, 11, 22, 33]);
7219        let b = vector_unsigned_int([10, 20, 30, 40]);
7220
7221        let c = vector_unsigned_int([GT, LT, GT, LT]);
7222        let d = unsafe { vec_cmprg_idx(a, b, c) };
7223        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
7224    }
7225
7226    #[simd_test(enable = "vector")]
7227    fn test_vec_cmpnrg_idx() {
7228        let a = vector_unsigned_int([1, 11, 22, 33]);
7229        let b = vector_unsigned_int([10, 20, 30, 40]);
7230
7231        let c = vector_unsigned_int([GT, LT, GT, LT]);
7232        let d = unsafe { vec_cmpnrg_idx(a, b, c) };
7233        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
7234    }
7235
7236    #[simd_test(enable = "vector")]
7237    fn test_vec_cmprg_or_0_idx() {
7238        let a = vector_unsigned_int([1, 0, 22, 33]);
7239        let b = vector_unsigned_int([10, 20, 30, 40]);
7240
7241        let c = vector_unsigned_int([GT, LT, GT, LT]);
7242        let d = unsafe { vec_cmprg_or_0_idx(a, b, c) };
7243        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
7244    }
7245
7246    #[simd_test(enable = "vector")]
7247    fn test_vec_cmpnrg_or_0_idx() {
7248        let a = vector_unsigned_int([11, 33, 0, 22]);
7249        let b = vector_unsigned_int([10, 20, 30, 40]);
7250
7251        let c = vector_unsigned_int([GT, LT, GT, LT]);
7252        let d = unsafe { vec_cmpnrg_or_0_idx(a, b, c) };
7253        assert_eq!(d.as_array(), &[0, 8, 0, 0]);
7254    }
7255
7256    test_vec_2! { test_vec_cmpgt, vec_cmpgt, f32x4, f32x4 -> i32x4,
7257        [1.0, f32::NAN, f32::NAN, 3.14],
7258        [2.0, f32::NAN, 5.0, 2.0],
7259        [0, 0, 0, !0]
7260    }
7261
7262    test_vec_2! { test_vec_cmpge, vec_cmpge, f32x4, f32x4 -> i32x4,
7263        [1.0, f32::NAN, f32::NAN, 3.14],
7264        [1.0, f32::NAN, 5.0, 2.0],
7265        [!0, 0, 0, !0]
7266    }
7267
7268    test_vec_2! { test_vec_cmplt, vec_cmplt, f32x4, f32x4 -> i32x4,
7269        [1.0, f32::NAN, f32::NAN, 2.0],
7270        [2.0, f32::NAN, 5.0, 2.0],
7271        [!0, 0, 0, 0]
7272    }
7273
7274    test_vec_2! { test_vec_cmple, vec_cmple, f32x4, f32x4 -> i32x4,
7275        [1.0, f32::NAN, f32::NAN, 2.0],
7276        [1.0, f32::NAN, 5.0, 3.14],
7277        [!0, 0, 0, !0]
7278    }
7279
7280    test_vec_2! { test_vec_cmpeq, vec_cmpeq, f32x4, f32x4 -> i32x4,
7281        [1.0, f32::NAN, f32::NAN, 2.0],
7282        [1.0, f32::NAN, 5.0, 3.14],
7283        [!0, 0, 0, 0]
7284    }
7285
7286    test_vec_2! { test_vec_cmpne, vec_cmpne, f32x4, f32x4 -> i32x4,
7287        [1.0, f32::NAN, f32::NAN, 2.0],
7288        [1.0, f32::NAN, 5.0, 3.14],
7289        [0, !0, !0, !0]
7290    }
7291
7292    #[simd_test(enable = "vector")]
7293    fn test_vec_meadd() {
7294        let a = vector_unsigned_short([1, 0, 2, 0, 3, 0, 4, 0]);
7295        let b = vector_unsigned_short([5, 0, 6, 0, 7, 0, 8, 0]);
7296        let c = vector_unsigned_int([2, 2, 2, 2]);
7297
7298        let d = unsafe { vec_meadd(a, b, c) };
7299        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7300
7301        let a = vector_signed_short([1, 0, 2, 0, 3, 0, 4, 0]);
7302        let b = vector_signed_short([5, 0, 6, 0, 7, 0, 8, 0]);
7303        let c = vector_signed_int([2, -2, 2, -2]);
7304
7305        let d = unsafe { vec_meadd(a, b, c) };
7306        assert_eq!(d.as_array(), &[7, 10, 23, 30]);
7307    }
7308
7309    #[simd_test(enable = "vector")]
7310    fn test_vec_moadd() {
7311        let a = vector_unsigned_short([0, 1, 0, 2, 0, 3, 0, 4]);
7312        let b = vector_unsigned_short([0, 5, 0, 6, 0, 7, 0, 8]);
7313        let c = vector_unsigned_int([2, 2, 2, 2]);
7314
7315        let d = unsafe { vec_moadd(a, b, c) };
7316        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7317
7318        let a = vector_signed_short([0, 1, 0, 2, 0, 3, 0, 4]);
7319        let b = vector_signed_short([0, 5, 0, 6, 0, 7, 0, 8]);
7320        let c = vector_signed_int([2, -2, 2, -2]);
7321
7322        let d = unsafe { vec_moadd(a, b, c) };
7323        assert_eq!(d.as_array(), &[7, 10, 23, 30]);
7324    }
7325
7326    #[simd_test(enable = "vector")]
7327    fn test_vec_mhadd() {
7328        let a = vector_unsigned_int([1, 2, 3, 4]);
7329        let b = vector_unsigned_int([5, 6, 7, 8]);
7330        let c = vector_unsigned_int([u32::MAX; 4]);
7331
7332        let d = unsafe { vec_mhadd(a, b, c) };
7333        assert_eq!(d.as_array(), &[1, 1, 1, 1]);
7334
7335        let a = vector_signed_int([-1, -2, -3, -4]);
7336        let b = vector_signed_int([5, 6, 7, 8]);
7337        let c = vector_signed_int([i32::MIN; 4]);
7338
7339        let d = unsafe { vec_mhadd(a, b, c) };
7340        assert_eq!(d.as_array(), &[-1, -1, -1, -1]);
7341    }
7342
7343    #[simd_test(enable = "vector")]
7344    fn test_vec_mladd() {
7345        let a = vector_unsigned_int([1, 2, 3, 4]);
7346        let b = vector_unsigned_int([5, 6, 7, 8]);
7347        let c = vector_unsigned_int([2, 2, 2, 2]);
7348
7349        let d = unsafe { vec_mladd(a, b, c) };
7350        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7351
7352        let a = vector_signed_int([-1, -2, -3, -4]);
7353        let b = vector_signed_int([5, 6, 7, 8]);
7354        let c = vector_signed_int([2, 2, 2, 2]);
7355
7356        let d = unsafe { vec_mladd(a, b, c) };
7357        assert_eq!(d.as_array(), &[-3, -10, -19, -30]);
7358    }
7359
7360    #[simd_test(enable = "vector")]
7361    fn test_vec_extract() {
7362        let v = vector_unsigned_int([1, 2, 3, 4]);
7363
7364        assert_eq!(unsafe { vec_extract(v, 1) }, 2);
7365        assert_eq!(unsafe { vec_extract(v, 4 + 2) }, 3);
7366    }
7367
7368    #[simd_test(enable = "vector")]
7369    fn test_vec_insert() {
7370        let mut v = vector_unsigned_int([1, 2, 3, 4]);
7371
7372        v = unsafe { vec_insert(42, v, 1) };
7373        assert_eq!(v.as_array(), &[1, 42, 3, 4]);
7374
7375        v = unsafe { vec_insert(64, v, 6) };
7376        assert_eq!(v.as_array(), &[1, 42, 64, 4]);
7377    }
7378
7379    #[simd_test(enable = "vector")]
7380    fn test_vec_promote() {
7381        let v: vector_unsigned_int = unsafe { vec_promote(42, 1).assume_init() };
7382        assert_eq!(v.as_array(), &[0, 42, 0, 0]);
7383    }
7384
7385    #[simd_test(enable = "vector")]
7386    fn test_vec_insert_and_zero() {
7387        let v = unsafe { vec_insert_and_zero::<vector_unsigned_int>(&42u32) };
7388        assert_eq!(v.as_array(), vector_unsigned_int([0, 42, 0, 0]).as_array());
7389    }
7390}