1 /*
2 * ARM NEON vector operations.
3 *
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
6 *
7 * This code is licensed under the GNU GPL v2.
8 */
9
10 #include "qemu/osdep.h"
11 #include "cpu.h"
12 #include "exec/helper-proto.h"
13 #include "tcg/tcg-gvec-desc.h"
14 #include "fpu/softfloat.h"
15 #include "vec_internal.h"
16
17 #define SIGNBIT (uint32_t)0x80000000
18 #define SIGNBIT64 ((uint64_t)1 << 63)
19
20 #define SET_QC() env->vfp.qc[0] = 1
21
22 #define NEON_TYPE1(name, type) \
23 typedef struct \
24 { \
25 type v1; \
26 } neon_##name;
27 #if HOST_BIG_ENDIAN
28 #define NEON_TYPE2(name, type) \
29 typedef struct \
30 { \
31 type v2; \
32 type v1; \
33 } neon_##name;
34 #define NEON_TYPE4(name, type) \
35 typedef struct \
36 { \
37 type v4; \
38 type v3; \
39 type v2; \
40 type v1; \
41 } neon_##name;
42 #else
43 #define NEON_TYPE2(name, type) \
44 typedef struct \
45 { \
46 type v1; \
47 type v2; \
48 } neon_##name;
49 #define NEON_TYPE4(name, type) \
50 typedef struct \
51 { \
52 type v1; \
53 type v2; \
54 type v3; \
55 type v4; \
56 } neon_##name;
57 #endif
58
NEON_TYPE4(s8,int8_t)59 NEON_TYPE4(s8, int8_t)
60 NEON_TYPE4(u8, uint8_t)
61 NEON_TYPE2(s16, int16_t)
62 NEON_TYPE2(u16, uint16_t)
63 NEON_TYPE1(s32, int32_t)
64 NEON_TYPE1(u32, uint32_t)
65 #undef NEON_TYPE4
66 #undef NEON_TYPE2
67 #undef NEON_TYPE1
68
69 /* Copy from a uint32_t to a vector structure type. */
70 #define NEON_UNPACK(vtype, dest, val) do { \
71 union { \
72 vtype v; \
73 uint32_t i; \
74 } conv_u; \
75 conv_u.i = (val); \
76 dest = conv_u.v; \
77 } while(0)
78
79 /* Copy from a vector structure type to a uint32_t. */
80 #define NEON_PACK(vtype, dest, val) do { \
81 union { \
82 vtype v; \
83 uint32_t i; \
84 } conv_u; \
85 conv_u.v = (val); \
86 dest = conv_u.i; \
87 } while(0)
88
89 #define NEON_DO1 \
90 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
91 #define NEON_DO2 \
92 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
93 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
94 #define NEON_DO4 \
95 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
96 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
97 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
98 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
99
100 #define NEON_VOP_BODY(vtype, n) \
101 { \
102 uint32_t res; \
103 vtype vsrc1; \
104 vtype vsrc2; \
105 vtype vdest; \
106 NEON_UNPACK(vtype, vsrc1, arg1); \
107 NEON_UNPACK(vtype, vsrc2, arg2); \
108 NEON_DO##n; \
109 NEON_PACK(vtype, res, vdest); \
110 return res; \
111 }
112
113 #define NEON_VOP(name, vtype, n) \
114 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
115 NEON_VOP_BODY(vtype, n)
116
117 #define NEON_VOP_ENV(name, vtype, n) \
118 uint32_t HELPER(glue(neon_,name))(CPUARMState *env, uint32_t arg1, uint32_t arg2) \
119 NEON_VOP_BODY(vtype, n)
120
121 #define NEON_GVEC_VOP2(name, vtype) \
122 void HELPER(name)(void *vd, void *vn, void *vm, uint32_t desc) \
123 { \
124 intptr_t i, opr_sz = simd_oprsz(desc); \
125 vtype *d = vd, *n = vn, *m = vm; \
126 for (i = 0; i < opr_sz / sizeof(vtype); i++) { \
127 NEON_FN(d[i], n[i], m[i]); \
128 } \
129 clear_tail(d, opr_sz, simd_maxsz(desc)); \
130 }
131
132 #define NEON_GVEC_VOP2_ENV(name, vtype) \
133 void HELPER(name)(void *vd, void *vn, void *vm, void *venv, uint32_t desc) \
134 { \
135 intptr_t i, opr_sz = simd_oprsz(desc); \
136 vtype *d = vd, *n = vn, *m = vm; \
137 CPUARMState *env = venv; \
138 for (i = 0; i < opr_sz / sizeof(vtype); i++) { \
139 NEON_FN(d[i], n[i], m[i]); \
140 } \
141 clear_tail(d, opr_sz, simd_maxsz(desc)); \
142 }
143
144 #define NEON_GVEC_VOP2i_ENV(name, vtype) \
145 void HELPER(name)(void *vd, void *vn, void *venv, uint32_t desc) \
146 { \
147 intptr_t i, opr_sz = simd_oprsz(desc); \
148 int imm = simd_data(desc); \
149 vtype *d = vd, *n = vn; \
150 CPUARMState *env = venv; \
151 for (i = 0; i < opr_sz / sizeof(vtype); i++) { \
152 NEON_FN(d[i], n[i], imm); \
153 } \
154 clear_tail(d, opr_sz, simd_maxsz(desc)); \
155 }
156
157 /* Pairwise operations. */
158 /* For 32-bit elements each segment only contains a single element, so
159 the elementwise and pairwise operations are the same. */
160 #define NEON_PDO2 \
161 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
162 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
163 #define NEON_PDO4 \
164 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
165 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
166 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
167 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
168
169 #define NEON_POP(name, vtype, n) \
170 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
171 { \
172 uint32_t res; \
173 vtype vsrc1; \
174 vtype vsrc2; \
175 vtype vdest; \
176 NEON_UNPACK(vtype, vsrc1, arg1); \
177 NEON_UNPACK(vtype, vsrc2, arg2); \
178 NEON_PDO##n; \
179 NEON_PACK(vtype, res, vdest); \
180 return res; \
181 }
182
183 /* Unary operators. */
184 #define NEON_VOP1(name, vtype, n) \
185 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
186 { \
187 vtype vsrc1; \
188 vtype vdest; \
189 NEON_UNPACK(vtype, vsrc1, arg); \
190 NEON_DO##n; \
191 NEON_PACK(vtype, arg, vdest); \
192 return arg; \
193 }
194
195 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
196 NEON_POP(pmin_s8, neon_s8, 4)
197 NEON_POP(pmin_u8, neon_u8, 4)
198 NEON_POP(pmin_s16, neon_s16, 2)
199 NEON_POP(pmin_u16, neon_u16, 2)
200 #undef NEON_FN
201
202 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
203 NEON_POP(pmax_s8, neon_s8, 4)
204 NEON_POP(pmax_u8, neon_u8, 4)
205 NEON_POP(pmax_s16, neon_s16, 2)
206 NEON_POP(pmax_u16, neon_u16, 2)
207 #undef NEON_FN
208
209 #define NEON_FN(dest, src1, src2) \
210 (dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, false, NULL))
211 NEON_VOP(shl_u16, neon_u16, 2)
212 #undef NEON_FN
213
214 #define NEON_FN(dest, src1, src2) \
215 (dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, false, NULL))
216 NEON_VOP(shl_s16, neon_s16, 2)
217 #undef NEON_FN
218
219 #define NEON_FN(dest, src1, src2) \
220 (dest = do_sqrshl_bhs(src1, (int8_t)src2, 8, true, NULL))
221 NEON_VOP(rshl_s8, neon_s8, 4)
222 NEON_GVEC_VOP2(gvec_srshl_b, int8_t)
223 #undef NEON_FN
224
225 #define NEON_FN(dest, src1, src2) \
226 (dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, true, NULL))
227 NEON_VOP(rshl_s16, neon_s16, 2)
228 NEON_GVEC_VOP2(gvec_srshl_h, int16_t)
229 #undef NEON_FN
230
231 #define NEON_FN(dest, src1, src2) \
232 (dest = do_sqrshl_bhs(src1, (int8_t)src2, 32, true, NULL))
233 NEON_GVEC_VOP2(gvec_srshl_s, int32_t)
234 #undef NEON_FN
235
236 #define NEON_FN(dest, src1, src2) \
237 (dest = do_sqrshl_d(src1, (int8_t)src2, true, NULL))
238 NEON_GVEC_VOP2(gvec_srshl_d, int64_t)
239 #undef NEON_FN
240
241 uint32_t HELPER(neon_rshl_s32)(uint32_t val, uint32_t shift)
242 {
243 return do_sqrshl_bhs(val, (int8_t)shift, 32, true, NULL);
244 }
245
HELPER(neon_rshl_s64)246 uint64_t HELPER(neon_rshl_s64)(uint64_t val, uint64_t shift)
247 {
248 return do_sqrshl_d(val, (int8_t)shift, true, NULL);
249 }
250
251 #define NEON_FN(dest, src1, src2) \
252 (dest = do_uqrshl_bhs(src1, (int8_t)src2, 8, true, NULL))
253 NEON_VOP(rshl_u8, neon_u8, 4)
NEON_GVEC_VOP2(gvec_urshl_b,uint8_t)254 NEON_GVEC_VOP2(gvec_urshl_b, uint8_t)
255 #undef NEON_FN
256
257 #define NEON_FN(dest, src1, src2) \
258 (dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, true, NULL))
259 NEON_VOP(rshl_u16, neon_u16, 2)
260 NEON_GVEC_VOP2(gvec_urshl_h, uint16_t)
261 #undef NEON_FN
262
263 #define NEON_FN(dest, src1, src2) \
264 (dest = do_uqrshl_bhs(src1, (int8_t)src2, 32, true, NULL))
265 NEON_GVEC_VOP2(gvec_urshl_s, int32_t)
266 #undef NEON_FN
267
268 #define NEON_FN(dest, src1, src2) \
269 (dest = do_uqrshl_d(src1, (int8_t)src2, true, NULL))
270 NEON_GVEC_VOP2(gvec_urshl_d, int64_t)
271 #undef NEON_FN
272
273 uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shift)
274 {
275 return do_uqrshl_bhs(val, (int8_t)shift, 32, true, NULL);
276 }
277
HELPER(neon_rshl_u64)278 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shift)
279 {
280 return do_uqrshl_d(val, (int8_t)shift, true, NULL);
281 }
282
283 #define NEON_FN(dest, src1, src2) \
284 (dest = do_uqrshl_bhs(src1, (int8_t)src2, 8, false, env->vfp.qc))
285 NEON_VOP_ENV(qshl_u8, neon_u8, 4)
NEON_GVEC_VOP2_ENV(neon_uqshl_b,uint8_t)286 NEON_GVEC_VOP2_ENV(neon_uqshl_b, uint8_t)
287 NEON_GVEC_VOP2i_ENV(neon_uqshli_b, uint8_t)
288 #undef NEON_FN
289
290 #define NEON_FN(dest, src1, src2) \
291 (dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, false, env->vfp.qc))
292 NEON_VOP_ENV(qshl_u16, neon_u16, 2)
293 NEON_GVEC_VOP2_ENV(neon_uqshl_h, uint16_t)
294 NEON_GVEC_VOP2i_ENV(neon_uqshli_h, uint16_t)
295 #undef NEON_FN
296
297 #define NEON_FN(dest, src1, src2) \
298 (dest = do_uqrshl_bhs(src1, (int8_t)src2, 32, false, env->vfp.qc))
299 NEON_GVEC_VOP2_ENV(neon_uqshl_s, uint32_t)
300 NEON_GVEC_VOP2i_ENV(neon_uqshli_s, uint32_t)
301 #undef NEON_FN
302
303 #define NEON_FN(dest, src1, src2) \
304 (dest = do_uqrshl_d(src1, (int8_t)src2, false, env->vfp.qc))
305 NEON_GVEC_VOP2_ENV(neon_uqshl_d, uint64_t)
306 NEON_GVEC_VOP2i_ENV(neon_uqshli_d, uint64_t)
307 #undef NEON_FN
308
309 uint32_t HELPER(neon_qshl_u32)(CPUARMState *env, uint32_t val, uint32_t shift)
310 {
311 return do_uqrshl_bhs(val, (int8_t)shift, 32, false, env->vfp.qc);
312 }
313
HELPER(neon_qshl_u64)314 uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shift)
315 {
316 return do_uqrshl_d(val, (int8_t)shift, false, env->vfp.qc);
317 }
318
319 #define NEON_FN(dest, src1, src2) \
320 (dest = do_sqrshl_bhs(src1, (int8_t)src2, 8, false, env->vfp.qc))
321 NEON_VOP_ENV(qshl_s8, neon_s8, 4)
NEON_GVEC_VOP2_ENV(neon_sqshl_b,int8_t)322 NEON_GVEC_VOP2_ENV(neon_sqshl_b, int8_t)
323 NEON_GVEC_VOP2i_ENV(neon_sqshli_b, int8_t)
324 #undef NEON_FN
325
326 #define NEON_FN(dest, src1, src2) \
327 (dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, false, env->vfp.qc))
328 NEON_VOP_ENV(qshl_s16, neon_s16, 2)
329 NEON_GVEC_VOP2_ENV(neon_sqshl_h, int16_t)
330 NEON_GVEC_VOP2i_ENV(neon_sqshli_h, int16_t)
331 #undef NEON_FN
332
333 #define NEON_FN(dest, src1, src2) \
334 (dest = do_sqrshl_bhs(src1, (int8_t)src2, 32, false, env->vfp.qc))
335 NEON_GVEC_VOP2_ENV(neon_sqshl_s, int32_t)
336 NEON_GVEC_VOP2i_ENV(neon_sqshli_s, int32_t)
337 #undef NEON_FN
338
339 #define NEON_FN(dest, src1, src2) \
340 (dest = do_sqrshl_d(src1, (int8_t)src2, false, env->vfp.qc))
341 NEON_GVEC_VOP2_ENV(neon_sqshl_d, int64_t)
342 NEON_GVEC_VOP2i_ENV(neon_sqshli_d, int64_t)
343 #undef NEON_FN
344
345 uint32_t HELPER(neon_qshl_s32)(CPUARMState *env, uint32_t val, uint32_t shift)
346 {
347 return do_sqrshl_bhs(val, (int8_t)shift, 32, false, env->vfp.qc);
348 }
349
HELPER(neon_qshl_s64)350 uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t val, uint64_t shift)
351 {
352 return do_sqrshl_d(val, (int8_t)shift, false, env->vfp.qc);
353 }
354
355 #define NEON_FN(dest, src1, src2) \
356 (dest = do_suqrshl_bhs(src1, (int8_t)src2, 8, false, env->vfp.qc))
357 NEON_VOP_ENV(qshlu_s8, neon_s8, 4)
NEON_GVEC_VOP2i_ENV(neon_sqshlui_b,int8_t)358 NEON_GVEC_VOP2i_ENV(neon_sqshlui_b, int8_t)
359 #undef NEON_FN
360
361 #define NEON_FN(dest, src1, src2) \
362 (dest = do_suqrshl_bhs(src1, (int8_t)src2, 16, false, env->vfp.qc))
363 NEON_VOP_ENV(qshlu_s16, neon_s16, 2)
364 NEON_GVEC_VOP2i_ENV(neon_sqshlui_h, int16_t)
365 #undef NEON_FN
366
367 uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t val, uint32_t shift)
368 {
369 return do_suqrshl_bhs(val, (int8_t)shift, 32, false, env->vfp.qc);
370 }
371
HELPER(neon_qshlu_s64)372 uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t val, uint64_t shift)
373 {
374 return do_suqrshl_d(val, (int8_t)shift, false, env->vfp.qc);
375 }
376
377 #define NEON_FN(dest, src1, src2) \
378 (dest = do_suqrshl_bhs(src1, (int8_t)src2, 32, false, env->vfp.qc))
NEON_GVEC_VOP2i_ENV(neon_sqshlui_s,int32_t)379 NEON_GVEC_VOP2i_ENV(neon_sqshlui_s, int32_t)
380 #undef NEON_FN
381
382 #define NEON_FN(dest, src1, src2) \
383 (dest = do_suqrshl_d(src1, (int8_t)src2, false, env->vfp.qc))
384 NEON_GVEC_VOP2i_ENV(neon_sqshlui_d, int64_t)
385 #undef NEON_FN
386
387 #define NEON_FN(dest, src1, src2) \
388 (dest = do_uqrshl_bhs(src1, (int8_t)src2, 8, true, env->vfp.qc))
389 NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
390 NEON_GVEC_VOP2_ENV(neon_uqrshl_b, uint8_t)
391 #undef NEON_FN
392
393 #define NEON_FN(dest, src1, src2) \
394 (dest = do_uqrshl_bhs(src1, (int8_t)src2, 16, true, env->vfp.qc))
395 NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
396 NEON_GVEC_VOP2_ENV(neon_uqrshl_h, uint16_t)
397 #undef NEON_FN
398
399 #define NEON_FN(dest, src1, src2) \
400 (dest = do_uqrshl_bhs(src1, (int8_t)src2, 32, true, env->vfp.qc))
401 NEON_GVEC_VOP2_ENV(neon_uqrshl_s, uint32_t)
402 #undef NEON_FN
403
404 #define NEON_FN(dest, src1, src2) \
405 (dest = do_uqrshl_d(src1, (int8_t)src2, true, env->vfp.qc))
406 NEON_GVEC_VOP2_ENV(neon_uqrshl_d, uint64_t)
407 #undef NEON_FN
408
409 uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shift)
410 {
411 return do_uqrshl_bhs(val, (int8_t)shift, 32, true, env->vfp.qc);
412 }
413
HELPER(neon_qrshl_u64)414 uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shift)
415 {
416 return do_uqrshl_d(val, (int8_t)shift, true, env->vfp.qc);
417 }
418
419 #define NEON_FN(dest, src1, src2) \
420 (dest = do_sqrshl_bhs(src1, (int8_t)src2, 8, true, env->vfp.qc))
421 NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
NEON_GVEC_VOP2_ENV(neon_sqrshl_b,int8_t)422 NEON_GVEC_VOP2_ENV(neon_sqrshl_b, int8_t)
423 #undef NEON_FN
424
425 #define NEON_FN(dest, src1, src2) \
426 (dest = do_sqrshl_bhs(src1, (int8_t)src2, 16, true, env->vfp.qc))
427 NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
428 NEON_GVEC_VOP2_ENV(neon_sqrshl_h, int16_t)
429 #undef NEON_FN
430
431 #define NEON_FN(dest, src1, src2) \
432 (dest = do_sqrshl_bhs(src1, (int8_t)src2, 32, true, env->vfp.qc))
433 NEON_GVEC_VOP2_ENV(neon_sqrshl_s, int32_t)
434 #undef NEON_FN
435
436 #define NEON_FN(dest, src1, src2) \
437 (dest = do_sqrshl_d(src1, (int8_t)src2, true, env->vfp.qc))
438 NEON_GVEC_VOP2_ENV(neon_sqrshl_d, int64_t)
439 #undef NEON_FN
440
441 uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t val, uint32_t shift)
442 {
443 return do_sqrshl_bhs(val, (int8_t)shift, 32, true, env->vfp.qc);
444 }
445
HELPER(neon_qrshl_s64)446 uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t val, uint64_t shift)
447 {
448 return do_sqrshl_d(val, (int8_t)shift, true, env->vfp.qc);
449 }
450
HELPER(neon_add_u8)451 uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
452 {
453 uint32_t mask;
454 mask = (a ^ b) & 0x80808080u;
455 a &= ~0x80808080u;
456 b &= ~0x80808080u;
457 return (a + b) ^ mask;
458 }
459
HELPER(neon_add_u16)460 uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
461 {
462 uint32_t mask;
463 mask = (a ^ b) & 0x80008000u;
464 a &= ~0x80008000u;
465 b &= ~0x80008000u;
466 return (a + b) ^ mask;
467 }
468
469 #define NEON_FN(dest, src1, src2) dest = src1 - src2
470 NEON_VOP(sub_u8, neon_u8, 4)
471 NEON_VOP(sub_u16, neon_u16, 2)
472 #undef NEON_FN
473
474 #define NEON_FN(dest, src1, src2) dest = src1 * src2
475 NEON_VOP(mul_u8, neon_u8, 4)
476 NEON_VOP(mul_u16, neon_u16, 2)
477 #undef NEON_FN
478
479 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
480 NEON_VOP(tst_u8, neon_u8, 4)
481 NEON_VOP(tst_u16, neon_u16, 2)
482 NEON_VOP(tst_u32, neon_u32, 1)
483 #undef NEON_FN
484
485 /* Count Leading Sign/Zero Bits. */
do_clz8(uint8_t x)486 static inline int do_clz8(uint8_t x)
487 {
488 int n;
489 for (n = 8; x; n--)
490 x >>= 1;
491 return n;
492 }
493
do_clz16(uint16_t x)494 static inline int do_clz16(uint16_t x)
495 {
496 int n;
497 for (n = 16; x; n--)
498 x >>= 1;
499 return n;
500 }
501
502 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
503 NEON_VOP1(clz_u8, neon_u8, 4)
504 #undef NEON_FN
505
506 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
507 NEON_VOP1(clz_u16, neon_u16, 2)
508 #undef NEON_FN
509
510 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
511 NEON_VOP1(cls_s8, neon_s8, 4)
512 #undef NEON_FN
513
514 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
515 NEON_VOP1(cls_s16, neon_s16, 2)
516 #undef NEON_FN
517
HELPER(neon_cls_s32)518 uint32_t HELPER(neon_cls_s32)(uint32_t x)
519 {
520 int count;
521 if ((int32_t)x < 0)
522 x = ~x;
523 for (count = 32; x; count--)
524 x = x >> 1;
525 return count - 1;
526 }
527
528 /* Bit count. */
HELPER(neon_cnt_u8)529 uint32_t HELPER(neon_cnt_u8)(uint32_t x)
530 {
531 x = (x & 0x55555555) + ((x >> 1) & 0x55555555);
532 x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
533 x = (x & 0x0f0f0f0f) + ((x >> 4) & 0x0f0f0f0f);
534 return x;
535 }
536
537 /* Reverse bits in each 8 bit word */
HELPER(neon_rbit_u8)538 uint32_t HELPER(neon_rbit_u8)(uint32_t x)
539 {
540 x = ((x & 0xf0f0f0f0) >> 4)
541 | ((x & 0x0f0f0f0f) << 4);
542 x = ((x & 0x88888888) >> 3)
543 | ((x & 0x44444444) >> 1)
544 | ((x & 0x22222222) << 1)
545 | ((x & 0x11111111) << 3);
546 return x;
547 }
548
549 #define NEON_QDMULH16(dest, src1, src2, round) do { \
550 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
551 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
552 SET_QC(); \
553 tmp = (tmp >> 31) ^ ~SIGNBIT; \
554 } else { \
555 tmp <<= 1; \
556 } \
557 if (round) { \
558 int32_t old = tmp; \
559 tmp += 1 << 15; \
560 if ((int32_t)tmp < old) { \
561 SET_QC(); \
562 tmp = SIGNBIT - 1; \
563 } \
564 } \
565 dest = tmp >> 16; \
566 } while(0)
567 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
568 NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
569 #undef NEON_FN
570 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
571 NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
572 #undef NEON_FN
573 #undef NEON_QDMULH16
574
575 #define NEON_QDMULH32(dest, src1, src2, round) do { \
576 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
577 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
578 SET_QC(); \
579 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
580 } else { \
581 tmp <<= 1; \
582 } \
583 if (round) { \
584 int64_t old = tmp; \
585 tmp += (int64_t)1 << 31; \
586 if ((int64_t)tmp < old) { \
587 SET_QC(); \
588 tmp = SIGNBIT64 - 1; \
589 } \
590 } \
591 dest = tmp >> 32; \
592 } while(0)
593 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
594 NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
595 #undef NEON_FN
596 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
597 NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
598 #undef NEON_FN
599 #undef NEON_QDMULH32
600
601 /* Only the low 32-bits of output are significant. */
HELPER(neon_narrow_u8)602 uint64_t HELPER(neon_narrow_u8)(uint64_t x)
603 {
604 return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
605 | ((x >> 24) & 0xff000000u);
606 }
607
608 /* Only the low 32-bits of output are significant. */
HELPER(neon_narrow_u16)609 uint64_t HELPER(neon_narrow_u16)(uint64_t x)
610 {
611 return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
612 }
613
HELPER(neon_narrow_high_u8)614 uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
615 {
616 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
617 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
618 }
619
HELPER(neon_narrow_high_u16)620 uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
621 {
622 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
623 }
624
HELPER(neon_narrow_round_high_u8)625 uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
626 {
627 x &= 0xff80ff80ff80ff80ull;
628 x += 0x0080008000800080ull;
629 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
630 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
631 }
632
HELPER(neon_narrow_round_high_u16)633 uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
634 {
635 x &= 0xffff8000ffff8000ull;
636 x += 0x0000800000008000ull;
637 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
638 }
639
640 /* Only the low 32-bits of output are significant. */
HELPER(neon_unarrow_sat8)641 uint64_t HELPER(neon_unarrow_sat8)(CPUARMState *env, uint64_t x)
642 {
643 uint16_t s;
644 uint8_t d;
645 uint32_t res = 0;
646 #define SAT8(n) \
647 s = x >> n; \
648 if (s & 0x8000) { \
649 SET_QC(); \
650 } else { \
651 if (s > 0xff) { \
652 d = 0xff; \
653 SET_QC(); \
654 } else { \
655 d = s; \
656 } \
657 res |= (uint32_t)d << (n / 2); \
658 }
659
660 SAT8(0);
661 SAT8(16);
662 SAT8(32);
663 SAT8(48);
664 #undef SAT8
665 return res;
666 }
667
668 /* Only the low 32-bits of output are significant. */
HELPER(neon_narrow_sat_u8)669 uint64_t HELPER(neon_narrow_sat_u8)(CPUARMState *env, uint64_t x)
670 {
671 uint16_t s;
672 uint8_t d;
673 uint32_t res = 0;
674 #define SAT8(n) \
675 s = x >> n; \
676 if (s > 0xff) { \
677 d = 0xff; \
678 SET_QC(); \
679 } else { \
680 d = s; \
681 } \
682 res |= (uint32_t)d << (n / 2);
683
684 SAT8(0);
685 SAT8(16);
686 SAT8(32);
687 SAT8(48);
688 #undef SAT8
689 return res;
690 }
691
692 /* Only the low 32-bits of output are significant. */
HELPER(neon_narrow_sat_s8)693 uint64_t HELPER(neon_narrow_sat_s8)(CPUARMState *env, uint64_t x)
694 {
695 int16_t s;
696 uint8_t d;
697 uint32_t res = 0;
698 #define SAT8(n) \
699 s = x >> n; \
700 if (s != (int8_t)s) { \
701 d = (s >> 15) ^ 0x7f; \
702 SET_QC(); \
703 } else { \
704 d = s; \
705 } \
706 res |= (uint32_t)d << (n / 2);
707
708 SAT8(0);
709 SAT8(16);
710 SAT8(32);
711 SAT8(48);
712 #undef SAT8
713 return res;
714 }
715
716 /* Only the low 32-bits of output are significant. */
HELPER(neon_unarrow_sat16)717 uint64_t HELPER(neon_unarrow_sat16)(CPUARMState *env, uint64_t x)
718 {
719 uint32_t high;
720 uint32_t low;
721 low = x;
722 if (low & 0x80000000) {
723 low = 0;
724 SET_QC();
725 } else if (low > 0xffff) {
726 low = 0xffff;
727 SET_QC();
728 }
729 high = x >> 32;
730 if (high & 0x80000000) {
731 high = 0;
732 SET_QC();
733 } else if (high > 0xffff) {
734 high = 0xffff;
735 SET_QC();
736 }
737 return deposit32(low, 16, 16, high);
738 }
739
740 /* Only the low 32-bits of output are significant. */
HELPER(neon_narrow_sat_u16)741 uint64_t HELPER(neon_narrow_sat_u16)(CPUARMState *env, uint64_t x)
742 {
743 uint32_t high;
744 uint32_t low;
745 low = x;
746 if (low > 0xffff) {
747 low = 0xffff;
748 SET_QC();
749 }
750 high = x >> 32;
751 if (high > 0xffff) {
752 high = 0xffff;
753 SET_QC();
754 }
755 return deposit32(low, 16, 16, high);
756 }
757
758 /* Only the low 32-bits of output are significant. */
HELPER(neon_narrow_sat_s16)759 uint64_t HELPER(neon_narrow_sat_s16)(CPUARMState *env, uint64_t x)
760 {
761 int32_t low;
762 int32_t high;
763 low = x;
764 if (low != (int16_t)low) {
765 low = (low >> 31) ^ 0x7fff;
766 SET_QC();
767 }
768 high = x >> 32;
769 if (high != (int16_t)high) {
770 high = (high >> 31) ^ 0x7fff;
771 SET_QC();
772 }
773 return deposit32(low, 16, 16, high);
774 }
775
776 /* Only the low 32-bits of output are significant. */
HELPER(neon_unarrow_sat32)777 uint64_t HELPER(neon_unarrow_sat32)(CPUARMState *env, uint64_t x)
778 {
779 if (x & 0x8000000000000000ull) {
780 SET_QC();
781 return 0;
782 }
783 if (x > 0xffffffffu) {
784 SET_QC();
785 return 0xffffffffu;
786 }
787 return x;
788 }
789
790 /* Only the low 32-bits of output are significant. */
HELPER(neon_narrow_sat_u32)791 uint64_t HELPER(neon_narrow_sat_u32)(CPUARMState *env, uint64_t x)
792 {
793 if (x > 0xffffffffu) {
794 SET_QC();
795 return 0xffffffffu;
796 }
797 return x;
798 }
799
800 /* Only the low 32-bits of output are significant. */
HELPER(neon_narrow_sat_s32)801 uint64_t HELPER(neon_narrow_sat_s32)(CPUARMState *env, uint64_t x)
802 {
803 if ((int64_t)x != (int32_t)x) {
804 SET_QC();
805 return (uint32_t)((int64_t)x >> 63) ^ 0x7fffffff;
806 }
807 return (uint32_t)x;
808 }
809
HELPER(neon_widen_u8)810 uint64_t HELPER(neon_widen_u8)(uint32_t x)
811 {
812 uint64_t tmp;
813 uint64_t ret;
814 ret = (uint8_t)x;
815 tmp = (uint8_t)(x >> 8);
816 ret |= tmp << 16;
817 tmp = (uint8_t)(x >> 16);
818 ret |= tmp << 32;
819 tmp = (uint8_t)(x >> 24);
820 ret |= tmp << 48;
821 return ret;
822 }
823
HELPER(neon_widen_s8)824 uint64_t HELPER(neon_widen_s8)(uint32_t x)
825 {
826 uint64_t tmp;
827 uint64_t ret;
828 ret = (uint16_t)(int8_t)x;
829 tmp = (uint16_t)(int8_t)(x >> 8);
830 ret |= tmp << 16;
831 tmp = (uint16_t)(int8_t)(x >> 16);
832 ret |= tmp << 32;
833 tmp = (uint16_t)(int8_t)(x >> 24);
834 ret |= tmp << 48;
835 return ret;
836 }
837
HELPER(neon_widen_u16)838 uint64_t HELPER(neon_widen_u16)(uint32_t x)
839 {
840 uint64_t high = (uint16_t)(x >> 16);
841 return ((uint16_t)x) | (high << 32);
842 }
843
HELPER(neon_widen_s16)844 uint64_t HELPER(neon_widen_s16)(uint32_t x)
845 {
846 uint64_t high = (int16_t)(x >> 16);
847 return ((uint32_t)(int16_t)x) | (high << 32);
848 }
849
HELPER(neon_addl_u16)850 uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
851 {
852 uint64_t mask;
853 mask = (a ^ b) & 0x8000800080008000ull;
854 a &= ~0x8000800080008000ull;
855 b &= ~0x8000800080008000ull;
856 return (a + b) ^ mask;
857 }
858
HELPER(neon_addl_u32)859 uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
860 {
861 uint64_t mask;
862 mask = (a ^ b) & 0x8000000080000000ull;
863 a &= ~0x8000000080000000ull;
864 b &= ~0x8000000080000000ull;
865 return (a + b) ^ mask;
866 }
867
HELPER(neon_paddl_u16)868 uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
869 {
870 uint64_t tmp;
871 uint64_t tmp2;
872
873 tmp = a & 0x0000ffff0000ffffull;
874 tmp += (a >> 16) & 0x0000ffff0000ffffull;
875 tmp2 = b & 0xffff0000ffff0000ull;
876 tmp2 += (b << 16) & 0xffff0000ffff0000ull;
877 return ( tmp & 0xffff)
878 | ((tmp >> 16) & 0xffff0000ull)
879 | ((tmp2 << 16) & 0xffff00000000ull)
880 | ( tmp2 & 0xffff000000000000ull);
881 }
882
HELPER(neon_paddl_u32)883 uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
884 {
885 uint32_t low = a + (a >> 32);
886 uint32_t high = b + (b >> 32);
887 return low + ((uint64_t)high << 32);
888 }
889
HELPER(neon_subl_u16)890 uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
891 {
892 uint64_t mask;
893 mask = (a ^ ~b) & 0x8000800080008000ull;
894 a |= 0x8000800080008000ull;
895 b &= ~0x8000800080008000ull;
896 return (a - b) ^ mask;
897 }
898
HELPER(neon_subl_u32)899 uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
900 {
901 uint64_t mask;
902 mask = (a ^ ~b) & 0x8000000080000000ull;
903 a |= 0x8000000080000000ull;
904 b &= ~0x8000000080000000ull;
905 return (a - b) ^ mask;
906 }
907
HELPER(neon_addl_saturate_s32)908 uint64_t HELPER(neon_addl_saturate_s32)(CPUARMState *env, uint64_t a, uint64_t b)
909 {
910 uint32_t x, y;
911 uint32_t low, high;
912
913 x = a;
914 y = b;
915 low = x + y;
916 if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
917 SET_QC();
918 low = ((int32_t)x >> 31) ^ ~SIGNBIT;
919 }
920 x = a >> 32;
921 y = b >> 32;
922 high = x + y;
923 if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
924 SET_QC();
925 high = ((int32_t)x >> 31) ^ ~SIGNBIT;
926 }
927 return low | ((uint64_t)high << 32);
928 }
929
HELPER(neon_addl_saturate_s64)930 uint64_t HELPER(neon_addl_saturate_s64)(CPUARMState *env, uint64_t a, uint64_t b)
931 {
932 uint64_t result;
933
934 result = a + b;
935 if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
936 SET_QC();
937 result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
938 }
939 return result;
940 }
941
942 /* We have to do the arithmetic in a larger type than
943 * the input type, because for example with a signed 32 bit
944 * op the absolute difference can overflow a signed 32 bit value.
945 */
946 #define DO_ABD(dest, x, y, intype, arithtype) do { \
947 arithtype tmp_x = (intype)(x); \
948 arithtype tmp_y = (intype)(y); \
949 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
950 } while(0)
951
HELPER(neon_abdl_u16)952 uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
953 {
954 uint64_t tmp;
955 uint64_t result;
956 DO_ABD(result, a, b, uint8_t, uint32_t);
957 DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
958 result |= tmp << 16;
959 DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
960 result |= tmp << 32;
961 DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
962 result |= tmp << 48;
963 return result;
964 }
965
HELPER(neon_abdl_s16)966 uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
967 {
968 uint64_t tmp;
969 uint64_t result;
970 DO_ABD(result, a, b, int8_t, int32_t);
971 DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
972 result |= tmp << 16;
973 DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
974 result |= tmp << 32;
975 DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
976 result |= tmp << 48;
977 return result;
978 }
979
HELPER(neon_abdl_u32)980 uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
981 {
982 uint64_t tmp;
983 uint64_t result;
984 DO_ABD(result, a, b, uint16_t, uint32_t);
985 DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
986 return result | (tmp << 32);
987 }
988
HELPER(neon_abdl_s32)989 uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
990 {
991 uint64_t tmp;
992 uint64_t result;
993 DO_ABD(result, a, b, int16_t, int32_t);
994 DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
995 return result | (tmp << 32);
996 }
997
HELPER(neon_abdl_u64)998 uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
999 {
1000 uint64_t result;
1001 DO_ABD(result, a, b, uint32_t, uint64_t);
1002 return result;
1003 }
1004
HELPER(neon_abdl_s64)1005 uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1006 {
1007 uint64_t result;
1008 DO_ABD(result, a, b, int32_t, int64_t);
1009 return result;
1010 }
1011 #undef DO_ABD
1012
1013 /* Widening multiply. Named type is the source type. */
1014 #define DO_MULL(dest, x, y, type1, type2) do { \
1015 type1 tmp_x = x; \
1016 type1 tmp_y = y; \
1017 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1018 } while(0)
1019
HELPER(neon_mull_u8)1020 uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1021 {
1022 uint64_t tmp;
1023 uint64_t result;
1024
1025 DO_MULL(result, a, b, uint8_t, uint16_t);
1026 DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1027 result |= tmp << 16;
1028 DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1029 result |= tmp << 32;
1030 DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1031 result |= tmp << 48;
1032 return result;
1033 }
1034
HELPER(neon_mull_s8)1035 uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1036 {
1037 uint64_t tmp;
1038 uint64_t result;
1039
1040 DO_MULL(result, a, b, int8_t, uint16_t);
1041 DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1042 result |= tmp << 16;
1043 DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1044 result |= tmp << 32;
1045 DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1046 result |= tmp << 48;
1047 return result;
1048 }
1049
HELPER(neon_mull_u16)1050 uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1051 {
1052 uint64_t tmp;
1053 uint64_t result;
1054
1055 DO_MULL(result, a, b, uint16_t, uint32_t);
1056 DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1057 return result | (tmp << 32);
1058 }
1059
HELPER(neon_mull_s16)1060 uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1061 {
1062 uint64_t tmp;
1063 uint64_t result;
1064
1065 DO_MULL(result, a, b, int16_t, uint32_t);
1066 DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1067 return result | (tmp << 32);
1068 }
1069
HELPER(neon_negl_u16)1070 uint64_t HELPER(neon_negl_u16)(uint64_t x)
1071 {
1072 uint16_t tmp;
1073 uint64_t result;
1074 result = (uint16_t)-x;
1075 tmp = -(x >> 16);
1076 result |= (uint64_t)tmp << 16;
1077 tmp = -(x >> 32);
1078 result |= (uint64_t)tmp << 32;
1079 tmp = -(x >> 48);
1080 result |= (uint64_t)tmp << 48;
1081 return result;
1082 }
1083
HELPER(neon_negl_u32)1084 uint64_t HELPER(neon_negl_u32)(uint64_t x)
1085 {
1086 uint32_t low = -x;
1087 uint32_t high = -(x >> 32);
1088 return low | ((uint64_t)high << 32);
1089 }
1090
1091 /* Saturating sign manipulation. */
1092 /* ??? Make these use NEON_VOP1 */
1093 #define DO_QABS8(x) do { \
1094 if (x == (int8_t)0x80) { \
1095 x = 0x7f; \
1096 SET_QC(); \
1097 } else if (x < 0) { \
1098 x = -x; \
1099 }} while (0)
HELPER(neon_qabs_s8)1100 uint32_t HELPER(neon_qabs_s8)(CPUARMState *env, uint32_t x)
1101 {
1102 neon_s8 vec;
1103 NEON_UNPACK(neon_s8, vec, x);
1104 DO_QABS8(vec.v1);
1105 DO_QABS8(vec.v2);
1106 DO_QABS8(vec.v3);
1107 DO_QABS8(vec.v4);
1108 NEON_PACK(neon_s8, x, vec);
1109 return x;
1110 }
1111 #undef DO_QABS8
1112
1113 #define DO_QNEG8(x) do { \
1114 if (x == (int8_t)0x80) { \
1115 x = 0x7f; \
1116 SET_QC(); \
1117 } else { \
1118 x = -x; \
1119 }} while (0)
HELPER(neon_qneg_s8)1120 uint32_t HELPER(neon_qneg_s8)(CPUARMState *env, uint32_t x)
1121 {
1122 neon_s8 vec;
1123 NEON_UNPACK(neon_s8, vec, x);
1124 DO_QNEG8(vec.v1);
1125 DO_QNEG8(vec.v2);
1126 DO_QNEG8(vec.v3);
1127 DO_QNEG8(vec.v4);
1128 NEON_PACK(neon_s8, x, vec);
1129 return x;
1130 }
1131 #undef DO_QNEG8
1132
1133 #define DO_QABS16(x) do { \
1134 if (x == (int16_t)0x8000) { \
1135 x = 0x7fff; \
1136 SET_QC(); \
1137 } else if (x < 0) { \
1138 x = -x; \
1139 }} while (0)
HELPER(neon_qabs_s16)1140 uint32_t HELPER(neon_qabs_s16)(CPUARMState *env, uint32_t x)
1141 {
1142 neon_s16 vec;
1143 NEON_UNPACK(neon_s16, vec, x);
1144 DO_QABS16(vec.v1);
1145 DO_QABS16(vec.v2);
1146 NEON_PACK(neon_s16, x, vec);
1147 return x;
1148 }
1149 #undef DO_QABS16
1150
1151 #define DO_QNEG16(x) do { \
1152 if (x == (int16_t)0x8000) { \
1153 x = 0x7fff; \
1154 SET_QC(); \
1155 } else { \
1156 x = -x; \
1157 }} while (0)
HELPER(neon_qneg_s16)1158 uint32_t HELPER(neon_qneg_s16)(CPUARMState *env, uint32_t x)
1159 {
1160 neon_s16 vec;
1161 NEON_UNPACK(neon_s16, vec, x);
1162 DO_QNEG16(vec.v1);
1163 DO_QNEG16(vec.v2);
1164 NEON_PACK(neon_s16, x, vec);
1165 return x;
1166 }
1167 #undef DO_QNEG16
1168
HELPER(neon_qabs_s32)1169 uint32_t HELPER(neon_qabs_s32)(CPUARMState *env, uint32_t x)
1170 {
1171 if (x == SIGNBIT) {
1172 SET_QC();
1173 x = ~SIGNBIT;
1174 } else if ((int32_t)x < 0) {
1175 x = -x;
1176 }
1177 return x;
1178 }
1179
HELPER(neon_qneg_s32)1180 uint32_t HELPER(neon_qneg_s32)(CPUARMState *env, uint32_t x)
1181 {
1182 if (x == SIGNBIT) {
1183 SET_QC();
1184 x = ~SIGNBIT;
1185 } else {
1186 x = -x;
1187 }
1188 return x;
1189 }
1190
HELPER(neon_qabs_s64)1191 uint64_t HELPER(neon_qabs_s64)(CPUARMState *env, uint64_t x)
1192 {
1193 if (x == SIGNBIT64) {
1194 SET_QC();
1195 x = ~SIGNBIT64;
1196 } else if ((int64_t)x < 0) {
1197 x = -x;
1198 }
1199 return x;
1200 }
1201
HELPER(neon_qneg_s64)1202 uint64_t HELPER(neon_qneg_s64)(CPUARMState *env, uint64_t x)
1203 {
1204 if (x == SIGNBIT64) {
1205 SET_QC();
1206 x = ~SIGNBIT64;
1207 } else {
1208 x = -x;
1209 }
1210 return x;
1211 }
1212
1213 /* NEON Float helpers. */
1214
1215 /* Floating point comparisons produce an integer result.
1216 * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1217 * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1218 */
HELPER(neon_ceq_f32)1219 uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b, void *fpstp)
1220 {
1221 float_status *fpst = fpstp;
1222 return -float32_eq_quiet(make_float32(a), make_float32(b), fpst);
1223 }
1224
HELPER(neon_cge_f32)1225 uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b, void *fpstp)
1226 {
1227 float_status *fpst = fpstp;
1228 return -float32_le(make_float32(b), make_float32(a), fpst);
1229 }
1230
HELPER(neon_cgt_f32)1231 uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1232 {
1233 float_status *fpst = fpstp;
1234 return -float32_lt(make_float32(b), make_float32(a), fpst);
1235 }
1236
HELPER(neon_acge_f32)1237 uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b, void *fpstp)
1238 {
1239 float_status *fpst = fpstp;
1240 float32 f0 = float32_abs(make_float32(a));
1241 float32 f1 = float32_abs(make_float32(b));
1242 return -float32_le(f1, f0, fpst);
1243 }
1244
HELPER(neon_acgt_f32)1245 uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1246 {
1247 float_status *fpst = fpstp;
1248 float32 f0 = float32_abs(make_float32(a));
1249 float32 f1 = float32_abs(make_float32(b));
1250 return -float32_lt(f1, f0, fpst);
1251 }
1252
HELPER(neon_acge_f64)1253 uint64_t HELPER(neon_acge_f64)(uint64_t a, uint64_t b, void *fpstp)
1254 {
1255 float_status *fpst = fpstp;
1256 float64 f0 = float64_abs(make_float64(a));
1257 float64 f1 = float64_abs(make_float64(b));
1258 return -float64_le(f1, f0, fpst);
1259 }
1260
HELPER(neon_acgt_f64)1261 uint64_t HELPER(neon_acgt_f64)(uint64_t a, uint64_t b, void *fpstp)
1262 {
1263 float_status *fpst = fpstp;
1264 float64 f0 = float64_abs(make_float64(a));
1265 float64 f1 = float64_abs(make_float64(b));
1266 return -float64_lt(f1, f0, fpst);
1267 }
1268
1269 #define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
1270
HELPER(neon_qunzip8)1271 void HELPER(neon_qunzip8)(void *vd, void *vm)
1272 {
1273 uint64_t *rd = vd, *rm = vm;
1274 uint64_t zd0 = rd[0], zd1 = rd[1];
1275 uint64_t zm0 = rm[0], zm1 = rm[1];
1276
1277 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
1278 | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
1279 | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
1280 | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
1281 uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
1282 | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
1283 | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1284 | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
1285 uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
1286 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
1287 | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
1288 | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
1289 uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
1290 | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
1291 | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
1292 | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1293
1294 rm[0] = m0;
1295 rm[1] = m1;
1296 rd[0] = d0;
1297 rd[1] = d1;
1298 }
1299
HELPER(neon_qunzip16)1300 void HELPER(neon_qunzip16)(void *vd, void *vm)
1301 {
1302 uint64_t *rd = vd, *rm = vm;
1303 uint64_t zd0 = rd[0], zd1 = rd[1];
1304 uint64_t zm0 = rm[0], zm1 = rm[1];
1305
1306 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
1307 | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
1308 uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
1309 | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
1310 uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
1311 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
1312 uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
1313 | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1314
1315 rm[0] = m0;
1316 rm[1] = m1;
1317 rd[0] = d0;
1318 rd[1] = d1;
1319 }
1320
HELPER(neon_qunzip32)1321 void HELPER(neon_qunzip32)(void *vd, void *vm)
1322 {
1323 uint64_t *rd = vd, *rm = vm;
1324 uint64_t zd0 = rd[0], zd1 = rd[1];
1325 uint64_t zm0 = rm[0], zm1 = rm[1];
1326
1327 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
1328 uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1329 uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
1330 uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1331
1332 rm[0] = m0;
1333 rm[1] = m1;
1334 rd[0] = d0;
1335 rd[1] = d1;
1336 }
1337
HELPER(neon_unzip8)1338 void HELPER(neon_unzip8)(void *vd, void *vm)
1339 {
1340 uint64_t *rd = vd, *rm = vm;
1341 uint64_t zd = rd[0], zm = rm[0];
1342
1343 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
1344 | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
1345 | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
1346 | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
1347 uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
1348 | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
1349 | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
1350 | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
1351
1352 rm[0] = m0;
1353 rd[0] = d0;
1354 }
1355
HELPER(neon_unzip16)1356 void HELPER(neon_unzip16)(void *vd, void *vm)
1357 {
1358 uint64_t *rd = vd, *rm = vm;
1359 uint64_t zd = rd[0], zm = rm[0];
1360
1361 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
1362 | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
1363 uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
1364 | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
1365
1366 rm[0] = m0;
1367 rd[0] = d0;
1368 }
1369
HELPER(neon_qzip8)1370 void HELPER(neon_qzip8)(void *vd, void *vm)
1371 {
1372 uint64_t *rd = vd, *rm = vm;
1373 uint64_t zd0 = rd[0], zd1 = rd[1];
1374 uint64_t zm0 = rm[0], zm1 = rm[1];
1375
1376 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
1377 | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
1378 | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
1379 | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
1380 uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
1381 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
1382 | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
1383 | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
1384 uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
1385 | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
1386 | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1387 | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
1388 uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
1389 | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
1390 | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
1391 | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1392
1393 rm[0] = m0;
1394 rm[1] = m1;
1395 rd[0] = d0;
1396 rd[1] = d1;
1397 }
1398
HELPER(neon_qzip16)1399 void HELPER(neon_qzip16)(void *vd, void *vm)
1400 {
1401 uint64_t *rd = vd, *rm = vm;
1402 uint64_t zd0 = rd[0], zd1 = rd[1];
1403 uint64_t zm0 = rm[0], zm1 = rm[1];
1404
1405 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
1406 | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
1407 uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
1408 | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
1409 uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
1410 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
1411 uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
1412 | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1413
1414 rm[0] = m0;
1415 rm[1] = m1;
1416 rd[0] = d0;
1417 rd[1] = d1;
1418 }
1419
HELPER(neon_qzip32)1420 void HELPER(neon_qzip32)(void *vd, void *vm)
1421 {
1422 uint64_t *rd = vd, *rm = vm;
1423 uint64_t zd0 = rd[0], zd1 = rd[1];
1424 uint64_t zm0 = rm[0], zm1 = rm[1];
1425
1426 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
1427 uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
1428 uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1429 uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1430
1431 rm[0] = m0;
1432 rm[1] = m1;
1433 rd[0] = d0;
1434 rd[1] = d1;
1435 }
1436
HELPER(neon_zip8)1437 void HELPER(neon_zip8)(void *vd, void *vm)
1438 {
1439 uint64_t *rd = vd, *rm = vm;
1440 uint64_t zd = rd[0], zm = rm[0];
1441
1442 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
1443 | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
1444 | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
1445 | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
1446 uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
1447 | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
1448 | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
1449 | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
1450
1451 rm[0] = m0;
1452 rd[0] = d0;
1453 }
1454
HELPER(neon_zip16)1455 void HELPER(neon_zip16)(void *vd, void *vm)
1456 {
1457 uint64_t *rd = vd, *rm = vm;
1458 uint64_t zd = rd[0], zm = rm[0];
1459
1460 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
1461 | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
1462 uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
1463 | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
1464
1465 rm[0] = m0;
1466 rd[0] = d0;
1467 }
1468