xref: /openbmc/qemu/tcg/tcg-op-gvec.c (revision dcb32f1d)
1 /*
2  * Generic vector operation expansion
3  *
4  * Copyright (c) 2018 Linaro
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 #include "tcg/tcg.h"
22 #include "tcg/tcg-op.h"
23 #include "tcg/tcg-op-gvec.h"
24 #include "qemu/main-loop.h"
25 #include "tcg/tcg-gvec-desc.h"
26 
27 #define MAX_UNROLL  4
28 
29 #ifdef CONFIG_DEBUG_TCG
30 static const TCGOpcode vecop_list_empty[1] = { 0 };
31 #else
32 #define vecop_list_empty NULL
33 #endif
34 
35 
36 /* Verify vector size and alignment rules.  OFS should be the OR of all
37    of the operand offsets so that we can check them all at once.  */
38 static void check_size_align(uint32_t oprsz, uint32_t maxsz, uint32_t ofs)
39 {
40     uint32_t opr_align = oprsz >= 16 ? 15 : 7;
41     uint32_t max_align = maxsz >= 16 || oprsz >= 16 ? 15 : 7;
42     tcg_debug_assert(oprsz > 0);
43     tcg_debug_assert(oprsz <= maxsz);
44     tcg_debug_assert((oprsz & opr_align) == 0);
45     tcg_debug_assert((maxsz & max_align) == 0);
46     tcg_debug_assert((ofs & max_align) == 0);
47 }
48 
49 /* Verify vector overlap rules for two operands.  */
50 static void check_overlap_2(uint32_t d, uint32_t a, uint32_t s)
51 {
52     tcg_debug_assert(d == a || d + s <= a || a + s <= d);
53 }
54 
55 /* Verify vector overlap rules for three operands.  */
56 static void check_overlap_3(uint32_t d, uint32_t a, uint32_t b, uint32_t s)
57 {
58     check_overlap_2(d, a, s);
59     check_overlap_2(d, b, s);
60     check_overlap_2(a, b, s);
61 }
62 
63 /* Verify vector overlap rules for four operands.  */
64 static void check_overlap_4(uint32_t d, uint32_t a, uint32_t b,
65                             uint32_t c, uint32_t s)
66 {
67     check_overlap_2(d, a, s);
68     check_overlap_2(d, b, s);
69     check_overlap_2(d, c, s);
70     check_overlap_2(a, b, s);
71     check_overlap_2(a, c, s);
72     check_overlap_2(b, c, s);
73 }
74 
75 /* Create a descriptor from components.  */
76 uint32_t simd_desc(uint32_t oprsz, uint32_t maxsz, int32_t data)
77 {
78     uint32_t desc = 0;
79 
80     assert(oprsz % 8 == 0 && oprsz <= (8 << SIMD_OPRSZ_BITS));
81     assert(maxsz % 8 == 0 && maxsz <= (8 << SIMD_MAXSZ_BITS));
82     assert(data == sextract32(data, 0, SIMD_DATA_BITS));
83 
84     oprsz = (oprsz / 8) - 1;
85     maxsz = (maxsz / 8) - 1;
86     desc = deposit32(desc, SIMD_OPRSZ_SHIFT, SIMD_OPRSZ_BITS, oprsz);
87     desc = deposit32(desc, SIMD_MAXSZ_SHIFT, SIMD_MAXSZ_BITS, maxsz);
88     desc = deposit32(desc, SIMD_DATA_SHIFT, SIMD_DATA_BITS, data);
89 
90     return desc;
91 }
92 
93 /* Generate a call to a gvec-style helper with two vector operands.  */
94 void tcg_gen_gvec_2_ool(uint32_t dofs, uint32_t aofs,
95                         uint32_t oprsz, uint32_t maxsz, int32_t data,
96                         gen_helper_gvec_2 *fn)
97 {
98     TCGv_ptr a0, a1;
99     TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
100 
101     a0 = tcg_temp_new_ptr();
102     a1 = tcg_temp_new_ptr();
103 
104     tcg_gen_addi_ptr(a0, cpu_env, dofs);
105     tcg_gen_addi_ptr(a1, cpu_env, aofs);
106 
107     fn(a0, a1, desc);
108 
109     tcg_temp_free_ptr(a0);
110     tcg_temp_free_ptr(a1);
111     tcg_temp_free_i32(desc);
112 }
113 
114 /* Generate a call to a gvec-style helper with two vector operands
115    and one scalar operand.  */
116 void tcg_gen_gvec_2i_ool(uint32_t dofs, uint32_t aofs, TCGv_i64 c,
117                          uint32_t oprsz, uint32_t maxsz, int32_t data,
118                          gen_helper_gvec_2i *fn)
119 {
120     TCGv_ptr a0, a1;
121     TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
122 
123     a0 = tcg_temp_new_ptr();
124     a1 = tcg_temp_new_ptr();
125 
126     tcg_gen_addi_ptr(a0, cpu_env, dofs);
127     tcg_gen_addi_ptr(a1, cpu_env, aofs);
128 
129     fn(a0, a1, c, desc);
130 
131     tcg_temp_free_ptr(a0);
132     tcg_temp_free_ptr(a1);
133     tcg_temp_free_i32(desc);
134 }
135 
136 /* Generate a call to a gvec-style helper with three vector operands.  */
137 void tcg_gen_gvec_3_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
138                         uint32_t oprsz, uint32_t maxsz, int32_t data,
139                         gen_helper_gvec_3 *fn)
140 {
141     TCGv_ptr a0, a1, a2;
142     TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
143 
144     a0 = tcg_temp_new_ptr();
145     a1 = tcg_temp_new_ptr();
146     a2 = tcg_temp_new_ptr();
147 
148     tcg_gen_addi_ptr(a0, cpu_env, dofs);
149     tcg_gen_addi_ptr(a1, cpu_env, aofs);
150     tcg_gen_addi_ptr(a2, cpu_env, bofs);
151 
152     fn(a0, a1, a2, desc);
153 
154     tcg_temp_free_ptr(a0);
155     tcg_temp_free_ptr(a1);
156     tcg_temp_free_ptr(a2);
157     tcg_temp_free_i32(desc);
158 }
159 
160 /* Generate a call to a gvec-style helper with four vector operands.  */
161 void tcg_gen_gvec_4_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
162                         uint32_t cofs, uint32_t oprsz, uint32_t maxsz,
163                         int32_t data, gen_helper_gvec_4 *fn)
164 {
165     TCGv_ptr a0, a1, a2, a3;
166     TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
167 
168     a0 = tcg_temp_new_ptr();
169     a1 = tcg_temp_new_ptr();
170     a2 = tcg_temp_new_ptr();
171     a3 = tcg_temp_new_ptr();
172 
173     tcg_gen_addi_ptr(a0, cpu_env, dofs);
174     tcg_gen_addi_ptr(a1, cpu_env, aofs);
175     tcg_gen_addi_ptr(a2, cpu_env, bofs);
176     tcg_gen_addi_ptr(a3, cpu_env, cofs);
177 
178     fn(a0, a1, a2, a3, desc);
179 
180     tcg_temp_free_ptr(a0);
181     tcg_temp_free_ptr(a1);
182     tcg_temp_free_ptr(a2);
183     tcg_temp_free_ptr(a3);
184     tcg_temp_free_i32(desc);
185 }
186 
187 /* Generate a call to a gvec-style helper with five vector operands.  */
188 void tcg_gen_gvec_5_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
189                         uint32_t cofs, uint32_t xofs, uint32_t oprsz,
190                         uint32_t maxsz, int32_t data, gen_helper_gvec_5 *fn)
191 {
192     TCGv_ptr a0, a1, a2, a3, a4;
193     TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
194 
195     a0 = tcg_temp_new_ptr();
196     a1 = tcg_temp_new_ptr();
197     a2 = tcg_temp_new_ptr();
198     a3 = tcg_temp_new_ptr();
199     a4 = tcg_temp_new_ptr();
200 
201     tcg_gen_addi_ptr(a0, cpu_env, dofs);
202     tcg_gen_addi_ptr(a1, cpu_env, aofs);
203     tcg_gen_addi_ptr(a2, cpu_env, bofs);
204     tcg_gen_addi_ptr(a3, cpu_env, cofs);
205     tcg_gen_addi_ptr(a4, cpu_env, xofs);
206 
207     fn(a0, a1, a2, a3, a4, desc);
208 
209     tcg_temp_free_ptr(a0);
210     tcg_temp_free_ptr(a1);
211     tcg_temp_free_ptr(a2);
212     tcg_temp_free_ptr(a3);
213     tcg_temp_free_ptr(a4);
214     tcg_temp_free_i32(desc);
215 }
216 
217 /* Generate a call to a gvec-style helper with three vector operands
218    and an extra pointer operand.  */
219 void tcg_gen_gvec_2_ptr(uint32_t dofs, uint32_t aofs,
220                         TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
221                         int32_t data, gen_helper_gvec_2_ptr *fn)
222 {
223     TCGv_ptr a0, a1;
224     TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
225 
226     a0 = tcg_temp_new_ptr();
227     a1 = tcg_temp_new_ptr();
228 
229     tcg_gen_addi_ptr(a0, cpu_env, dofs);
230     tcg_gen_addi_ptr(a1, cpu_env, aofs);
231 
232     fn(a0, a1, ptr, desc);
233 
234     tcg_temp_free_ptr(a0);
235     tcg_temp_free_ptr(a1);
236     tcg_temp_free_i32(desc);
237 }
238 
239 /* Generate a call to a gvec-style helper with three vector operands
240    and an extra pointer operand.  */
241 void tcg_gen_gvec_3_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
242                         TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
243                         int32_t data, gen_helper_gvec_3_ptr *fn)
244 {
245     TCGv_ptr a0, a1, a2;
246     TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
247 
248     a0 = tcg_temp_new_ptr();
249     a1 = tcg_temp_new_ptr();
250     a2 = tcg_temp_new_ptr();
251 
252     tcg_gen_addi_ptr(a0, cpu_env, dofs);
253     tcg_gen_addi_ptr(a1, cpu_env, aofs);
254     tcg_gen_addi_ptr(a2, cpu_env, bofs);
255 
256     fn(a0, a1, a2, ptr, desc);
257 
258     tcg_temp_free_ptr(a0);
259     tcg_temp_free_ptr(a1);
260     tcg_temp_free_ptr(a2);
261     tcg_temp_free_i32(desc);
262 }
263 
264 /* Generate a call to a gvec-style helper with four vector operands
265    and an extra pointer operand.  */
266 void tcg_gen_gvec_4_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
267                         uint32_t cofs, TCGv_ptr ptr, uint32_t oprsz,
268                         uint32_t maxsz, int32_t data,
269                         gen_helper_gvec_4_ptr *fn)
270 {
271     TCGv_ptr a0, a1, a2, a3;
272     TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
273 
274     a0 = tcg_temp_new_ptr();
275     a1 = tcg_temp_new_ptr();
276     a2 = tcg_temp_new_ptr();
277     a3 = tcg_temp_new_ptr();
278 
279     tcg_gen_addi_ptr(a0, cpu_env, dofs);
280     tcg_gen_addi_ptr(a1, cpu_env, aofs);
281     tcg_gen_addi_ptr(a2, cpu_env, bofs);
282     tcg_gen_addi_ptr(a3, cpu_env, cofs);
283 
284     fn(a0, a1, a2, a3, ptr, desc);
285 
286     tcg_temp_free_ptr(a0);
287     tcg_temp_free_ptr(a1);
288     tcg_temp_free_ptr(a2);
289     tcg_temp_free_ptr(a3);
290     tcg_temp_free_i32(desc);
291 }
292 
293 /* Return true if we want to implement something of OPRSZ bytes
294    in units of LNSZ.  This limits the expansion of inline code.  */
295 static inline bool check_size_impl(uint32_t oprsz, uint32_t lnsz)
296 {
297     if (oprsz % lnsz == 0) {
298         uint32_t lnct = oprsz / lnsz;
299         return lnct >= 1 && lnct <= MAX_UNROLL;
300     }
301     return false;
302 }
303 
304 static void expand_clr(uint32_t dofs, uint32_t maxsz);
305 
306 /* Duplicate C as per VECE.  */
307 uint64_t (dup_const)(unsigned vece, uint64_t c)
308 {
309     switch (vece) {
310     case MO_8:
311         return 0x0101010101010101ull * (uint8_t)c;
312     case MO_16:
313         return 0x0001000100010001ull * (uint16_t)c;
314     case MO_32:
315         return 0x0000000100000001ull * (uint32_t)c;
316     case MO_64:
317         return c;
318     default:
319         g_assert_not_reached();
320     }
321 }
322 
323 /* Duplicate IN into OUT as per VECE.  */
324 static void gen_dup_i32(unsigned vece, TCGv_i32 out, TCGv_i32 in)
325 {
326     switch (vece) {
327     case MO_8:
328         tcg_gen_ext8u_i32(out, in);
329         tcg_gen_muli_i32(out, out, 0x01010101);
330         break;
331     case MO_16:
332         tcg_gen_deposit_i32(out, in, in, 16, 16);
333         break;
334     case MO_32:
335         tcg_gen_mov_i32(out, in);
336         break;
337     default:
338         g_assert_not_reached();
339     }
340 }
341 
342 static void gen_dup_i64(unsigned vece, TCGv_i64 out, TCGv_i64 in)
343 {
344     switch (vece) {
345     case MO_8:
346         tcg_gen_ext8u_i64(out, in);
347         tcg_gen_muli_i64(out, out, 0x0101010101010101ull);
348         break;
349     case MO_16:
350         tcg_gen_ext16u_i64(out, in);
351         tcg_gen_muli_i64(out, out, 0x0001000100010001ull);
352         break;
353     case MO_32:
354         tcg_gen_deposit_i64(out, in, in, 32, 32);
355         break;
356     case MO_64:
357         tcg_gen_mov_i64(out, in);
358         break;
359     default:
360         g_assert_not_reached();
361     }
362 }
363 
364 /* Select a supported vector type for implementing an operation on SIZE
365  * bytes.  If OP is 0, assume that the real operation to be performed is
366  * required by all backends.  Otherwise, make sure than OP can be performed
367  * on elements of size VECE in the selected type.  Do not select V64 if
368  * PREFER_I64 is true.  Return 0 if no vector type is selected.
369  */
370 static TCGType choose_vector_type(const TCGOpcode *list, unsigned vece,
371                                   uint32_t size, bool prefer_i64)
372 {
373     if (TCG_TARGET_HAS_v256 && check_size_impl(size, 32)) {
374         /*
375          * Recall that ARM SVE allows vector sizes that are not a
376          * power of 2, but always a multiple of 16.  The intent is
377          * that e.g. size == 80 would be expanded with 2x32 + 1x16.
378          * It is hard to imagine a case in which v256 is supported
379          * but v128 is not, but check anyway.
380          */
381         if (tcg_can_emit_vecop_list(list, TCG_TYPE_V256, vece)
382             && (size % 32 == 0
383                 || tcg_can_emit_vecop_list(list, TCG_TYPE_V128, vece))) {
384             return TCG_TYPE_V256;
385         }
386     }
387     if (TCG_TARGET_HAS_v128 && check_size_impl(size, 16)
388         && tcg_can_emit_vecop_list(list, TCG_TYPE_V128, vece)) {
389         return TCG_TYPE_V128;
390     }
391     if (TCG_TARGET_HAS_v64 && !prefer_i64 && check_size_impl(size, 8)
392         && tcg_can_emit_vecop_list(list, TCG_TYPE_V64, vece)) {
393         return TCG_TYPE_V64;
394     }
395     return 0;
396 }
397 
398 static void do_dup_store(TCGType type, uint32_t dofs, uint32_t oprsz,
399                          uint32_t maxsz, TCGv_vec t_vec)
400 {
401     uint32_t i = 0;
402 
403     switch (type) {
404     case TCG_TYPE_V256:
405         /*
406          * Recall that ARM SVE allows vector sizes that are not a
407          * power of 2, but always a multiple of 16.  The intent is
408          * that e.g. size == 80 would be expanded with 2x32 + 1x16.
409          */
410         for (; i + 32 <= oprsz; i += 32) {
411             tcg_gen_stl_vec(t_vec, cpu_env, dofs + i, TCG_TYPE_V256);
412         }
413         /* fallthru */
414     case TCG_TYPE_V128:
415         for (; i + 16 <= oprsz; i += 16) {
416             tcg_gen_stl_vec(t_vec, cpu_env, dofs + i, TCG_TYPE_V128);
417         }
418         break;
419     case TCG_TYPE_V64:
420         for (; i < oprsz; i += 8) {
421             tcg_gen_stl_vec(t_vec, cpu_env, dofs + i, TCG_TYPE_V64);
422         }
423         break;
424     default:
425         g_assert_not_reached();
426     }
427 
428     if (oprsz < maxsz) {
429         expand_clr(dofs + oprsz, maxsz - oprsz);
430     }
431 }
432 
433 /* Set OPRSZ bytes at DOFS to replications of IN_32, IN_64 or IN_C.
434  * Only one of IN_32 or IN_64 may be set;
435  * IN_C is used if IN_32 and IN_64 are unset.
436  */
437 static void do_dup(unsigned vece, uint32_t dofs, uint32_t oprsz,
438                    uint32_t maxsz, TCGv_i32 in_32, TCGv_i64 in_64,
439                    uint64_t in_c)
440 {
441     TCGType type;
442     TCGv_i64 t_64;
443     TCGv_i32 t_32, t_desc;
444     TCGv_ptr t_ptr;
445     uint32_t i;
446 
447     assert(vece <= (in_32 ? MO_32 : MO_64));
448     assert(in_32 == NULL || in_64 == NULL);
449 
450     /* If we're storing 0, expand oprsz to maxsz.  */
451     if (in_32 == NULL && in_64 == NULL) {
452         in_c = dup_const(vece, in_c);
453         if (in_c == 0) {
454             oprsz = maxsz;
455         }
456     }
457 
458     /* Implement inline with a vector type, if possible.
459      * Prefer integer when 64-bit host and no variable dup.
460      */
461     type = choose_vector_type(NULL, vece, oprsz,
462                               (TCG_TARGET_REG_BITS == 64 && in_32 == NULL
463                                && (in_64 == NULL || vece == MO_64)));
464     if (type != 0) {
465         TCGv_vec t_vec = tcg_temp_new_vec(type);
466 
467         if (in_32) {
468             tcg_gen_dup_i32_vec(vece, t_vec, in_32);
469         } else if (in_64) {
470             tcg_gen_dup_i64_vec(vece, t_vec, in_64);
471         } else {
472             tcg_gen_dupi_vec(vece, t_vec, in_c);
473         }
474         do_dup_store(type, dofs, oprsz, maxsz, t_vec);
475         tcg_temp_free_vec(t_vec);
476         return;
477     }
478 
479     /* Otherwise, inline with an integer type, unless "large".  */
480     if (check_size_impl(oprsz, TCG_TARGET_REG_BITS / 8)) {
481         t_64 = NULL;
482         t_32 = NULL;
483 
484         if (in_32) {
485             /* We are given a 32-bit variable input.  For a 64-bit host,
486                use a 64-bit operation unless the 32-bit operation would
487                be simple enough.  */
488             if (TCG_TARGET_REG_BITS == 64
489                 && (vece != MO_32 || !check_size_impl(oprsz, 4))) {
490                 t_64 = tcg_temp_new_i64();
491                 tcg_gen_extu_i32_i64(t_64, in_32);
492                 gen_dup_i64(vece, t_64, t_64);
493             } else {
494                 t_32 = tcg_temp_new_i32();
495                 gen_dup_i32(vece, t_32, in_32);
496             }
497         } else if (in_64) {
498             /* We are given a 64-bit variable input.  */
499             t_64 = tcg_temp_new_i64();
500             gen_dup_i64(vece, t_64, in_64);
501         } else {
502             /* We are given a constant input.  */
503             /* For 64-bit hosts, use 64-bit constants for "simple" constants
504                or when we'd need too many 32-bit stores, or when a 64-bit
505                constant is really required.  */
506             if (vece == MO_64
507                 || (TCG_TARGET_REG_BITS == 64
508                     && (in_c == 0 || in_c == -1
509                         || !check_size_impl(oprsz, 4)))) {
510                 t_64 = tcg_const_i64(in_c);
511             } else {
512                 t_32 = tcg_const_i32(in_c);
513             }
514         }
515 
516         /* Implement inline if we picked an implementation size above.  */
517         if (t_32) {
518             for (i = 0; i < oprsz; i += 4) {
519                 tcg_gen_st_i32(t_32, cpu_env, dofs + i);
520             }
521             tcg_temp_free_i32(t_32);
522             goto done;
523         }
524         if (t_64) {
525             for (i = 0; i < oprsz; i += 8) {
526                 tcg_gen_st_i64(t_64, cpu_env, dofs + i);
527             }
528             tcg_temp_free_i64(t_64);
529             goto done;
530         }
531     }
532 
533     /* Otherwise implement out of line.  */
534     t_ptr = tcg_temp_new_ptr();
535     tcg_gen_addi_ptr(t_ptr, cpu_env, dofs);
536     t_desc = tcg_const_i32(simd_desc(oprsz, maxsz, 0));
537 
538     if (vece == MO_64) {
539         if (in_64) {
540             gen_helper_gvec_dup64(t_ptr, t_desc, in_64);
541         } else {
542             t_64 = tcg_const_i64(in_c);
543             gen_helper_gvec_dup64(t_ptr, t_desc, t_64);
544             tcg_temp_free_i64(t_64);
545         }
546     } else {
547         typedef void dup_fn(TCGv_ptr, TCGv_i32, TCGv_i32);
548         static dup_fn * const fns[3] = {
549             gen_helper_gvec_dup8,
550             gen_helper_gvec_dup16,
551             gen_helper_gvec_dup32
552         };
553 
554         if (in_32) {
555             fns[vece](t_ptr, t_desc, in_32);
556         } else {
557             t_32 = tcg_temp_new_i32();
558             if (in_64) {
559                 tcg_gen_extrl_i64_i32(t_32, in_64);
560             } else if (vece == MO_8) {
561                 tcg_gen_movi_i32(t_32, in_c & 0xff);
562             } else if (vece == MO_16) {
563                 tcg_gen_movi_i32(t_32, in_c & 0xffff);
564             } else {
565                 tcg_gen_movi_i32(t_32, in_c);
566             }
567             fns[vece](t_ptr, t_desc, t_32);
568             tcg_temp_free_i32(t_32);
569         }
570     }
571 
572     tcg_temp_free_ptr(t_ptr);
573     tcg_temp_free_i32(t_desc);
574     return;
575 
576  done:
577     if (oprsz < maxsz) {
578         expand_clr(dofs + oprsz, maxsz - oprsz);
579     }
580 }
581 
582 /* Likewise, but with zero.  */
583 static void expand_clr(uint32_t dofs, uint32_t maxsz)
584 {
585     do_dup(MO_8, dofs, maxsz, maxsz, NULL, NULL, 0);
586 }
587 
588 /* Expand OPSZ bytes worth of two-operand operations using i32 elements.  */
589 static void expand_2_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
590                          void (*fni)(TCGv_i32, TCGv_i32))
591 {
592     TCGv_i32 t0 = tcg_temp_new_i32();
593     uint32_t i;
594 
595     for (i = 0; i < oprsz; i += 4) {
596         tcg_gen_ld_i32(t0, cpu_env, aofs + i);
597         fni(t0, t0);
598         tcg_gen_st_i32(t0, cpu_env, dofs + i);
599     }
600     tcg_temp_free_i32(t0);
601 }
602 
603 static void expand_2i_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
604                           int32_t c, bool load_dest,
605                           void (*fni)(TCGv_i32, TCGv_i32, int32_t))
606 {
607     TCGv_i32 t0 = tcg_temp_new_i32();
608     TCGv_i32 t1 = tcg_temp_new_i32();
609     uint32_t i;
610 
611     for (i = 0; i < oprsz; i += 4) {
612         tcg_gen_ld_i32(t0, cpu_env, aofs + i);
613         if (load_dest) {
614             tcg_gen_ld_i32(t1, cpu_env, dofs + i);
615         }
616         fni(t1, t0, c);
617         tcg_gen_st_i32(t1, cpu_env, dofs + i);
618     }
619     tcg_temp_free_i32(t0);
620     tcg_temp_free_i32(t1);
621 }
622 
623 static void expand_2s_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
624                           TCGv_i32 c, bool scalar_first,
625                           void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32))
626 {
627     TCGv_i32 t0 = tcg_temp_new_i32();
628     TCGv_i32 t1 = tcg_temp_new_i32();
629     uint32_t i;
630 
631     for (i = 0; i < oprsz; i += 4) {
632         tcg_gen_ld_i32(t0, cpu_env, aofs + i);
633         if (scalar_first) {
634             fni(t1, c, t0);
635         } else {
636             fni(t1, t0, c);
637         }
638         tcg_gen_st_i32(t1, cpu_env, dofs + i);
639     }
640     tcg_temp_free_i32(t0);
641     tcg_temp_free_i32(t1);
642 }
643 
644 /* Expand OPSZ bytes worth of three-operand operations using i32 elements.  */
645 static void expand_3_i32(uint32_t dofs, uint32_t aofs,
646                          uint32_t bofs, uint32_t oprsz, bool load_dest,
647                          void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32))
648 {
649     TCGv_i32 t0 = tcg_temp_new_i32();
650     TCGv_i32 t1 = tcg_temp_new_i32();
651     TCGv_i32 t2 = tcg_temp_new_i32();
652     uint32_t i;
653 
654     for (i = 0; i < oprsz; i += 4) {
655         tcg_gen_ld_i32(t0, cpu_env, aofs + i);
656         tcg_gen_ld_i32(t1, cpu_env, bofs + i);
657         if (load_dest) {
658             tcg_gen_ld_i32(t2, cpu_env, dofs + i);
659         }
660         fni(t2, t0, t1);
661         tcg_gen_st_i32(t2, cpu_env, dofs + i);
662     }
663     tcg_temp_free_i32(t2);
664     tcg_temp_free_i32(t1);
665     tcg_temp_free_i32(t0);
666 }
667 
668 static void expand_3i_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
669                           uint32_t oprsz, int32_t c, bool load_dest,
670                           void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32, int32_t))
671 {
672     TCGv_i32 t0 = tcg_temp_new_i32();
673     TCGv_i32 t1 = tcg_temp_new_i32();
674     TCGv_i32 t2 = tcg_temp_new_i32();
675     uint32_t i;
676 
677     for (i = 0; i < oprsz; i += 4) {
678         tcg_gen_ld_i32(t0, cpu_env, aofs + i);
679         tcg_gen_ld_i32(t1, cpu_env, bofs + i);
680         if (load_dest) {
681             tcg_gen_ld_i32(t2, cpu_env, dofs + i);
682         }
683         fni(t2, t0, t1, c);
684         tcg_gen_st_i32(t2, cpu_env, dofs + i);
685     }
686     tcg_temp_free_i32(t0);
687     tcg_temp_free_i32(t1);
688     tcg_temp_free_i32(t2);
689 }
690 
691 /* Expand OPSZ bytes worth of three-operand operations using i32 elements.  */
692 static void expand_4_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
693                          uint32_t cofs, uint32_t oprsz, bool write_aofs,
694                          void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32))
695 {
696     TCGv_i32 t0 = tcg_temp_new_i32();
697     TCGv_i32 t1 = tcg_temp_new_i32();
698     TCGv_i32 t2 = tcg_temp_new_i32();
699     TCGv_i32 t3 = tcg_temp_new_i32();
700     uint32_t i;
701 
702     for (i = 0; i < oprsz; i += 4) {
703         tcg_gen_ld_i32(t1, cpu_env, aofs + i);
704         tcg_gen_ld_i32(t2, cpu_env, bofs + i);
705         tcg_gen_ld_i32(t3, cpu_env, cofs + i);
706         fni(t0, t1, t2, t3);
707         tcg_gen_st_i32(t0, cpu_env, dofs + i);
708         if (write_aofs) {
709             tcg_gen_st_i32(t1, cpu_env, aofs + i);
710         }
711     }
712     tcg_temp_free_i32(t3);
713     tcg_temp_free_i32(t2);
714     tcg_temp_free_i32(t1);
715     tcg_temp_free_i32(t0);
716 }
717 
718 /* Expand OPSZ bytes worth of two-operand operations using i64 elements.  */
719 static void expand_2_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
720                          void (*fni)(TCGv_i64, TCGv_i64))
721 {
722     TCGv_i64 t0 = tcg_temp_new_i64();
723     uint32_t i;
724 
725     for (i = 0; i < oprsz; i += 8) {
726         tcg_gen_ld_i64(t0, cpu_env, aofs + i);
727         fni(t0, t0);
728         tcg_gen_st_i64(t0, cpu_env, dofs + i);
729     }
730     tcg_temp_free_i64(t0);
731 }
732 
733 static void expand_2i_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
734                           int64_t c, bool load_dest,
735                           void (*fni)(TCGv_i64, TCGv_i64, int64_t))
736 {
737     TCGv_i64 t0 = tcg_temp_new_i64();
738     TCGv_i64 t1 = tcg_temp_new_i64();
739     uint32_t i;
740 
741     for (i = 0; i < oprsz; i += 8) {
742         tcg_gen_ld_i64(t0, cpu_env, aofs + i);
743         if (load_dest) {
744             tcg_gen_ld_i64(t1, cpu_env, dofs + i);
745         }
746         fni(t1, t0, c);
747         tcg_gen_st_i64(t1, cpu_env, dofs + i);
748     }
749     tcg_temp_free_i64(t0);
750     tcg_temp_free_i64(t1);
751 }
752 
753 static void expand_2s_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
754                           TCGv_i64 c, bool scalar_first,
755                           void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64))
756 {
757     TCGv_i64 t0 = tcg_temp_new_i64();
758     TCGv_i64 t1 = tcg_temp_new_i64();
759     uint32_t i;
760 
761     for (i = 0; i < oprsz; i += 8) {
762         tcg_gen_ld_i64(t0, cpu_env, aofs + i);
763         if (scalar_first) {
764             fni(t1, c, t0);
765         } else {
766             fni(t1, t0, c);
767         }
768         tcg_gen_st_i64(t1, cpu_env, dofs + i);
769     }
770     tcg_temp_free_i64(t0);
771     tcg_temp_free_i64(t1);
772 }
773 
774 /* Expand OPSZ bytes worth of three-operand operations using i64 elements.  */
775 static void expand_3_i64(uint32_t dofs, uint32_t aofs,
776                          uint32_t bofs, uint32_t oprsz, bool load_dest,
777                          void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64))
778 {
779     TCGv_i64 t0 = tcg_temp_new_i64();
780     TCGv_i64 t1 = tcg_temp_new_i64();
781     TCGv_i64 t2 = tcg_temp_new_i64();
782     uint32_t i;
783 
784     for (i = 0; i < oprsz; i += 8) {
785         tcg_gen_ld_i64(t0, cpu_env, aofs + i);
786         tcg_gen_ld_i64(t1, cpu_env, bofs + i);
787         if (load_dest) {
788             tcg_gen_ld_i64(t2, cpu_env, dofs + i);
789         }
790         fni(t2, t0, t1);
791         tcg_gen_st_i64(t2, cpu_env, dofs + i);
792     }
793     tcg_temp_free_i64(t2);
794     tcg_temp_free_i64(t1);
795     tcg_temp_free_i64(t0);
796 }
797 
798 static void expand_3i_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
799                           uint32_t oprsz, int64_t c, bool load_dest,
800                           void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64, int64_t))
801 {
802     TCGv_i64 t0 = tcg_temp_new_i64();
803     TCGv_i64 t1 = tcg_temp_new_i64();
804     TCGv_i64 t2 = tcg_temp_new_i64();
805     uint32_t i;
806 
807     for (i = 0; i < oprsz; i += 8) {
808         tcg_gen_ld_i64(t0, cpu_env, aofs + i);
809         tcg_gen_ld_i64(t1, cpu_env, bofs + i);
810         if (load_dest) {
811             tcg_gen_ld_i64(t2, cpu_env, dofs + i);
812         }
813         fni(t2, t0, t1, c);
814         tcg_gen_st_i64(t2, cpu_env, dofs + i);
815     }
816     tcg_temp_free_i64(t0);
817     tcg_temp_free_i64(t1);
818     tcg_temp_free_i64(t2);
819 }
820 
821 /* Expand OPSZ bytes worth of three-operand operations using i64 elements.  */
822 static void expand_4_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
823                          uint32_t cofs, uint32_t oprsz, bool write_aofs,
824                          void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64))
825 {
826     TCGv_i64 t0 = tcg_temp_new_i64();
827     TCGv_i64 t1 = tcg_temp_new_i64();
828     TCGv_i64 t2 = tcg_temp_new_i64();
829     TCGv_i64 t3 = tcg_temp_new_i64();
830     uint32_t i;
831 
832     for (i = 0; i < oprsz; i += 8) {
833         tcg_gen_ld_i64(t1, cpu_env, aofs + i);
834         tcg_gen_ld_i64(t2, cpu_env, bofs + i);
835         tcg_gen_ld_i64(t3, cpu_env, cofs + i);
836         fni(t0, t1, t2, t3);
837         tcg_gen_st_i64(t0, cpu_env, dofs + i);
838         if (write_aofs) {
839             tcg_gen_st_i64(t1, cpu_env, aofs + i);
840         }
841     }
842     tcg_temp_free_i64(t3);
843     tcg_temp_free_i64(t2);
844     tcg_temp_free_i64(t1);
845     tcg_temp_free_i64(t0);
846 }
847 
848 /* Expand OPSZ bytes worth of two-operand operations using host vectors.  */
849 static void expand_2_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
850                          uint32_t oprsz, uint32_t tysz, TCGType type,
851                          void (*fni)(unsigned, TCGv_vec, TCGv_vec))
852 {
853     TCGv_vec t0 = tcg_temp_new_vec(type);
854     uint32_t i;
855 
856     for (i = 0; i < oprsz; i += tysz) {
857         tcg_gen_ld_vec(t0, cpu_env, aofs + i);
858         fni(vece, t0, t0);
859         tcg_gen_st_vec(t0, cpu_env, dofs + i);
860     }
861     tcg_temp_free_vec(t0);
862 }
863 
864 /* Expand OPSZ bytes worth of two-vector operands and an immediate operand
865    using host vectors.  */
866 static void expand_2i_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
867                           uint32_t oprsz, uint32_t tysz, TCGType type,
868                           int64_t c, bool load_dest,
869                           void (*fni)(unsigned, TCGv_vec, TCGv_vec, int64_t))
870 {
871     TCGv_vec t0 = tcg_temp_new_vec(type);
872     TCGv_vec t1 = tcg_temp_new_vec(type);
873     uint32_t i;
874 
875     for (i = 0; i < oprsz; i += tysz) {
876         tcg_gen_ld_vec(t0, cpu_env, aofs + i);
877         if (load_dest) {
878             tcg_gen_ld_vec(t1, cpu_env, dofs + i);
879         }
880         fni(vece, t1, t0, c);
881         tcg_gen_st_vec(t1, cpu_env, dofs + i);
882     }
883     tcg_temp_free_vec(t0);
884     tcg_temp_free_vec(t1);
885 }
886 
887 static void expand_2s_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
888                           uint32_t oprsz, uint32_t tysz, TCGType type,
889                           TCGv_vec c, bool scalar_first,
890                           void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec))
891 {
892     TCGv_vec t0 = tcg_temp_new_vec(type);
893     TCGv_vec t1 = tcg_temp_new_vec(type);
894     uint32_t i;
895 
896     for (i = 0; i < oprsz; i += tysz) {
897         tcg_gen_ld_vec(t0, cpu_env, aofs + i);
898         if (scalar_first) {
899             fni(vece, t1, c, t0);
900         } else {
901             fni(vece, t1, t0, c);
902         }
903         tcg_gen_st_vec(t1, cpu_env, dofs + i);
904     }
905     tcg_temp_free_vec(t0);
906     tcg_temp_free_vec(t1);
907 }
908 
909 /* Expand OPSZ bytes worth of three-operand operations using host vectors.  */
910 static void expand_3_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
911                          uint32_t bofs, uint32_t oprsz,
912                          uint32_t tysz, TCGType type, bool load_dest,
913                          void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec))
914 {
915     TCGv_vec t0 = tcg_temp_new_vec(type);
916     TCGv_vec t1 = tcg_temp_new_vec(type);
917     TCGv_vec t2 = tcg_temp_new_vec(type);
918     uint32_t i;
919 
920     for (i = 0; i < oprsz; i += tysz) {
921         tcg_gen_ld_vec(t0, cpu_env, aofs + i);
922         tcg_gen_ld_vec(t1, cpu_env, bofs + i);
923         if (load_dest) {
924             tcg_gen_ld_vec(t2, cpu_env, dofs + i);
925         }
926         fni(vece, t2, t0, t1);
927         tcg_gen_st_vec(t2, cpu_env, dofs + i);
928     }
929     tcg_temp_free_vec(t2);
930     tcg_temp_free_vec(t1);
931     tcg_temp_free_vec(t0);
932 }
933 
934 /*
935  * Expand OPSZ bytes worth of three-vector operands and an immediate operand
936  * using host vectors.
937  */
938 static void expand_3i_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
939                           uint32_t bofs, uint32_t oprsz, uint32_t tysz,
940                           TCGType type, int64_t c, bool load_dest,
941                           void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec,
942                                       int64_t))
943 {
944     TCGv_vec t0 = tcg_temp_new_vec(type);
945     TCGv_vec t1 = tcg_temp_new_vec(type);
946     TCGv_vec t2 = tcg_temp_new_vec(type);
947     uint32_t i;
948 
949     for (i = 0; i < oprsz; i += tysz) {
950         tcg_gen_ld_vec(t0, cpu_env, aofs + i);
951         tcg_gen_ld_vec(t1, cpu_env, bofs + i);
952         if (load_dest) {
953             tcg_gen_ld_vec(t2, cpu_env, dofs + i);
954         }
955         fni(vece, t2, t0, t1, c);
956         tcg_gen_st_vec(t2, cpu_env, dofs + i);
957     }
958     tcg_temp_free_vec(t0);
959     tcg_temp_free_vec(t1);
960     tcg_temp_free_vec(t2);
961 }
962 
963 /* Expand OPSZ bytes worth of four-operand operations using host vectors.  */
964 static void expand_4_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
965                          uint32_t bofs, uint32_t cofs, uint32_t oprsz,
966                          uint32_t tysz, TCGType type, bool write_aofs,
967                          void (*fni)(unsigned, TCGv_vec, TCGv_vec,
968                                      TCGv_vec, TCGv_vec))
969 {
970     TCGv_vec t0 = tcg_temp_new_vec(type);
971     TCGv_vec t1 = tcg_temp_new_vec(type);
972     TCGv_vec t2 = tcg_temp_new_vec(type);
973     TCGv_vec t3 = tcg_temp_new_vec(type);
974     uint32_t i;
975 
976     for (i = 0; i < oprsz; i += tysz) {
977         tcg_gen_ld_vec(t1, cpu_env, aofs + i);
978         tcg_gen_ld_vec(t2, cpu_env, bofs + i);
979         tcg_gen_ld_vec(t3, cpu_env, cofs + i);
980         fni(vece, t0, t1, t2, t3);
981         tcg_gen_st_vec(t0, cpu_env, dofs + i);
982         if (write_aofs) {
983             tcg_gen_st_vec(t1, cpu_env, aofs + i);
984         }
985     }
986     tcg_temp_free_vec(t3);
987     tcg_temp_free_vec(t2);
988     tcg_temp_free_vec(t1);
989     tcg_temp_free_vec(t0);
990 }
991 
992 /* Expand a vector two-operand operation.  */
993 void tcg_gen_gvec_2(uint32_t dofs, uint32_t aofs,
994                     uint32_t oprsz, uint32_t maxsz, const GVecGen2 *g)
995 {
996     const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
997     const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
998     TCGType type;
999     uint32_t some;
1000 
1001     check_size_align(oprsz, maxsz, dofs | aofs);
1002     check_overlap_2(dofs, aofs, maxsz);
1003 
1004     type = 0;
1005     if (g->fniv) {
1006         type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1007     }
1008     switch (type) {
1009     case TCG_TYPE_V256:
1010         /* Recall that ARM SVE allows vector sizes that are not a
1011          * power of 2, but always a multiple of 16.  The intent is
1012          * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1013          */
1014         some = QEMU_ALIGN_DOWN(oprsz, 32);
1015         expand_2_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256, g->fniv);
1016         if (some == oprsz) {
1017             break;
1018         }
1019         dofs += some;
1020         aofs += some;
1021         oprsz -= some;
1022         maxsz -= some;
1023         /* fallthru */
1024     case TCG_TYPE_V128:
1025         expand_2_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128, g->fniv);
1026         break;
1027     case TCG_TYPE_V64:
1028         expand_2_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64, g->fniv);
1029         break;
1030 
1031     case 0:
1032         if (g->fni8 && check_size_impl(oprsz, 8)) {
1033             expand_2_i64(dofs, aofs, oprsz, g->fni8);
1034         } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1035             expand_2_i32(dofs, aofs, oprsz, g->fni4);
1036         } else {
1037             assert(g->fno != NULL);
1038             tcg_gen_gvec_2_ool(dofs, aofs, oprsz, maxsz, g->data, g->fno);
1039             oprsz = maxsz;
1040         }
1041         break;
1042 
1043     default:
1044         g_assert_not_reached();
1045     }
1046     tcg_swap_vecop_list(hold_list);
1047 
1048     if (oprsz < maxsz) {
1049         expand_clr(dofs + oprsz, maxsz - oprsz);
1050     }
1051 }
1052 
1053 /* Expand a vector operation with two vectors and an immediate.  */
1054 void tcg_gen_gvec_2i(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
1055                      uint32_t maxsz, int64_t c, const GVecGen2i *g)
1056 {
1057     const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1058     const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1059     TCGType type;
1060     uint32_t some;
1061 
1062     check_size_align(oprsz, maxsz, dofs | aofs);
1063     check_overlap_2(dofs, aofs, maxsz);
1064 
1065     type = 0;
1066     if (g->fniv) {
1067         type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1068     }
1069     switch (type) {
1070     case TCG_TYPE_V256:
1071         /* Recall that ARM SVE allows vector sizes that are not a
1072          * power of 2, but always a multiple of 16.  The intent is
1073          * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1074          */
1075         some = QEMU_ALIGN_DOWN(oprsz, 32);
1076         expand_2i_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,
1077                       c, g->load_dest, g->fniv);
1078         if (some == oprsz) {
1079             break;
1080         }
1081         dofs += some;
1082         aofs += some;
1083         oprsz -= some;
1084         maxsz -= some;
1085         /* fallthru */
1086     case TCG_TYPE_V128:
1087         expand_2i_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
1088                       c, g->load_dest, g->fniv);
1089         break;
1090     case TCG_TYPE_V64:
1091         expand_2i_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
1092                       c, g->load_dest, g->fniv);
1093         break;
1094 
1095     case 0:
1096         if (g->fni8 && check_size_impl(oprsz, 8)) {
1097             expand_2i_i64(dofs, aofs, oprsz, c, g->load_dest, g->fni8);
1098         } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1099             expand_2i_i32(dofs, aofs, oprsz, c, g->load_dest, g->fni4);
1100         } else {
1101             if (g->fno) {
1102                 tcg_gen_gvec_2_ool(dofs, aofs, oprsz, maxsz, c, g->fno);
1103             } else {
1104                 TCGv_i64 tcg_c = tcg_const_i64(c);
1105                 tcg_gen_gvec_2i_ool(dofs, aofs, tcg_c, oprsz,
1106                                     maxsz, c, g->fnoi);
1107                 tcg_temp_free_i64(tcg_c);
1108             }
1109             oprsz = maxsz;
1110         }
1111         break;
1112 
1113     default:
1114         g_assert_not_reached();
1115     }
1116     tcg_swap_vecop_list(hold_list);
1117 
1118     if (oprsz < maxsz) {
1119         expand_clr(dofs + oprsz, maxsz - oprsz);
1120     }
1121 }
1122 
1123 /* Expand a vector operation with two vectors and a scalar.  */
1124 void tcg_gen_gvec_2s(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
1125                      uint32_t maxsz, TCGv_i64 c, const GVecGen2s *g)
1126 {
1127     TCGType type;
1128 
1129     check_size_align(oprsz, maxsz, dofs | aofs);
1130     check_overlap_2(dofs, aofs, maxsz);
1131 
1132     type = 0;
1133     if (g->fniv) {
1134         type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1135     }
1136     if (type != 0) {
1137         const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1138         const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1139         TCGv_vec t_vec = tcg_temp_new_vec(type);
1140         uint32_t some;
1141 
1142         tcg_gen_dup_i64_vec(g->vece, t_vec, c);
1143 
1144         switch (type) {
1145         case TCG_TYPE_V256:
1146             /* Recall that ARM SVE allows vector sizes that are not a
1147              * power of 2, but always a multiple of 16.  The intent is
1148              * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1149              */
1150             some = QEMU_ALIGN_DOWN(oprsz, 32);
1151             expand_2s_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,
1152                           t_vec, g->scalar_first, g->fniv);
1153             if (some == oprsz) {
1154                 break;
1155             }
1156             dofs += some;
1157             aofs += some;
1158             oprsz -= some;
1159             maxsz -= some;
1160             /* fallthru */
1161 
1162         case TCG_TYPE_V128:
1163             expand_2s_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
1164                           t_vec, g->scalar_first, g->fniv);
1165             break;
1166 
1167         case TCG_TYPE_V64:
1168             expand_2s_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
1169                           t_vec, g->scalar_first, g->fniv);
1170             break;
1171 
1172         default:
1173             g_assert_not_reached();
1174         }
1175         tcg_temp_free_vec(t_vec);
1176         tcg_swap_vecop_list(hold_list);
1177     } else if (g->fni8 && check_size_impl(oprsz, 8)) {
1178         TCGv_i64 t64 = tcg_temp_new_i64();
1179 
1180         gen_dup_i64(g->vece, t64, c);
1181         expand_2s_i64(dofs, aofs, oprsz, t64, g->scalar_first, g->fni8);
1182         tcg_temp_free_i64(t64);
1183     } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1184         TCGv_i32 t32 = tcg_temp_new_i32();
1185 
1186         tcg_gen_extrl_i64_i32(t32, c);
1187         gen_dup_i32(g->vece, t32, t32);
1188         expand_2s_i32(dofs, aofs, oprsz, t32, g->scalar_first, g->fni4);
1189         tcg_temp_free_i32(t32);
1190     } else {
1191         tcg_gen_gvec_2i_ool(dofs, aofs, c, oprsz, maxsz, 0, g->fno);
1192         return;
1193     }
1194 
1195     if (oprsz < maxsz) {
1196         expand_clr(dofs + oprsz, maxsz - oprsz);
1197     }
1198 }
1199 
1200 /* Expand a vector three-operand operation.  */
1201 void tcg_gen_gvec_3(uint32_t dofs, uint32_t aofs, uint32_t bofs,
1202                     uint32_t oprsz, uint32_t maxsz, const GVecGen3 *g)
1203 {
1204     const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1205     const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1206     TCGType type;
1207     uint32_t some;
1208 
1209     check_size_align(oprsz, maxsz, dofs | aofs | bofs);
1210     check_overlap_3(dofs, aofs, bofs, maxsz);
1211 
1212     type = 0;
1213     if (g->fniv) {
1214         type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1215     }
1216     switch (type) {
1217     case TCG_TYPE_V256:
1218         /* Recall that ARM SVE allows vector sizes that are not a
1219          * power of 2, but always a multiple of 16.  The intent is
1220          * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1221          */
1222         some = QEMU_ALIGN_DOWN(oprsz, 32);
1223         expand_3_vec(g->vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256,
1224                      g->load_dest, g->fniv);
1225         if (some == oprsz) {
1226             break;
1227         }
1228         dofs += some;
1229         aofs += some;
1230         bofs += some;
1231         oprsz -= some;
1232         maxsz -= some;
1233         /* fallthru */
1234     case TCG_TYPE_V128:
1235         expand_3_vec(g->vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128,
1236                      g->load_dest, g->fniv);
1237         break;
1238     case TCG_TYPE_V64:
1239         expand_3_vec(g->vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64,
1240                      g->load_dest, g->fniv);
1241         break;
1242 
1243     case 0:
1244         if (g->fni8 && check_size_impl(oprsz, 8)) {
1245             expand_3_i64(dofs, aofs, bofs, oprsz, g->load_dest, g->fni8);
1246         } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1247             expand_3_i32(dofs, aofs, bofs, oprsz, g->load_dest, g->fni4);
1248         } else {
1249             assert(g->fno != NULL);
1250             tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz,
1251                                maxsz, g->data, g->fno);
1252             oprsz = maxsz;
1253         }
1254         break;
1255 
1256     default:
1257         g_assert_not_reached();
1258     }
1259     tcg_swap_vecop_list(hold_list);
1260 
1261     if (oprsz < maxsz) {
1262         expand_clr(dofs + oprsz, maxsz - oprsz);
1263     }
1264 }
1265 
1266 /* Expand a vector operation with three vectors and an immediate.  */
1267 void tcg_gen_gvec_3i(uint32_t dofs, uint32_t aofs, uint32_t bofs,
1268                      uint32_t oprsz, uint32_t maxsz, int64_t c,
1269                      const GVecGen3i *g)
1270 {
1271     const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1272     const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1273     TCGType type;
1274     uint32_t some;
1275 
1276     check_size_align(oprsz, maxsz, dofs | aofs | bofs);
1277     check_overlap_3(dofs, aofs, bofs, maxsz);
1278 
1279     type = 0;
1280     if (g->fniv) {
1281         type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1282     }
1283     switch (type) {
1284     case TCG_TYPE_V256:
1285         /*
1286          * Recall that ARM SVE allows vector sizes that are not a
1287          * power of 2, but always a multiple of 16.  The intent is
1288          * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1289          */
1290         some = QEMU_ALIGN_DOWN(oprsz, 32);
1291         expand_3i_vec(g->vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256,
1292                       c, g->load_dest, g->fniv);
1293         if (some == oprsz) {
1294             break;
1295         }
1296         dofs += some;
1297         aofs += some;
1298         bofs += some;
1299         oprsz -= some;
1300         maxsz -= some;
1301         /* fallthru */
1302     case TCG_TYPE_V128:
1303         expand_3i_vec(g->vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128,
1304                       c, g->load_dest, g->fniv);
1305         break;
1306     case TCG_TYPE_V64:
1307         expand_3i_vec(g->vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64,
1308                       c, g->load_dest, g->fniv);
1309         break;
1310 
1311     case 0:
1312         if (g->fni8 && check_size_impl(oprsz, 8)) {
1313             expand_3i_i64(dofs, aofs, bofs, oprsz, c, g->load_dest, g->fni8);
1314         } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1315             expand_3i_i32(dofs, aofs, bofs, oprsz, c, g->load_dest, g->fni4);
1316         } else {
1317             assert(g->fno != NULL);
1318             tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz, maxsz, c, g->fno);
1319             oprsz = maxsz;
1320         }
1321         break;
1322 
1323     default:
1324         g_assert_not_reached();
1325     }
1326     tcg_swap_vecop_list(hold_list);
1327 
1328     if (oprsz < maxsz) {
1329         expand_clr(dofs + oprsz, maxsz - oprsz);
1330     }
1331 }
1332 
1333 /* Expand a vector four-operand operation.  */
1334 void tcg_gen_gvec_4(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t cofs,
1335                     uint32_t oprsz, uint32_t maxsz, const GVecGen4 *g)
1336 {
1337     const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1338     const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1339     TCGType type;
1340     uint32_t some;
1341 
1342     check_size_align(oprsz, maxsz, dofs | aofs | bofs | cofs);
1343     check_overlap_4(dofs, aofs, bofs, cofs, maxsz);
1344 
1345     type = 0;
1346     if (g->fniv) {
1347         type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1348     }
1349     switch (type) {
1350     case TCG_TYPE_V256:
1351         /* Recall that ARM SVE allows vector sizes that are not a
1352          * power of 2, but always a multiple of 16.  The intent is
1353          * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1354          */
1355         some = QEMU_ALIGN_DOWN(oprsz, 32);
1356         expand_4_vec(g->vece, dofs, aofs, bofs, cofs, some,
1357                      32, TCG_TYPE_V256, g->write_aofs, g->fniv);
1358         if (some == oprsz) {
1359             break;
1360         }
1361         dofs += some;
1362         aofs += some;
1363         bofs += some;
1364         cofs += some;
1365         oprsz -= some;
1366         maxsz -= some;
1367         /* fallthru */
1368     case TCG_TYPE_V128:
1369         expand_4_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,
1370                      16, TCG_TYPE_V128, g->write_aofs, g->fniv);
1371         break;
1372     case TCG_TYPE_V64:
1373         expand_4_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,
1374                      8, TCG_TYPE_V64, g->write_aofs, g->fniv);
1375         break;
1376 
1377     case 0:
1378         if (g->fni8 && check_size_impl(oprsz, 8)) {
1379             expand_4_i64(dofs, aofs, bofs, cofs, oprsz,
1380                          g->write_aofs, g->fni8);
1381         } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1382             expand_4_i32(dofs, aofs, bofs, cofs, oprsz,
1383                          g->write_aofs, g->fni4);
1384         } else {
1385             assert(g->fno != NULL);
1386             tcg_gen_gvec_4_ool(dofs, aofs, bofs, cofs,
1387                                oprsz, maxsz, g->data, g->fno);
1388             oprsz = maxsz;
1389         }
1390         break;
1391 
1392     default:
1393         g_assert_not_reached();
1394     }
1395     tcg_swap_vecop_list(hold_list);
1396 
1397     if (oprsz < maxsz) {
1398         expand_clr(dofs + oprsz, maxsz - oprsz);
1399     }
1400 }
1401 
1402 /*
1403  * Expand specific vector operations.
1404  */
1405 
1406 static void vec_mov2(unsigned vece, TCGv_vec a, TCGv_vec b)
1407 {
1408     tcg_gen_mov_vec(a, b);
1409 }
1410 
1411 void tcg_gen_gvec_mov(unsigned vece, uint32_t dofs, uint32_t aofs,
1412                       uint32_t oprsz, uint32_t maxsz)
1413 {
1414     static const GVecGen2 g = {
1415         .fni8 = tcg_gen_mov_i64,
1416         .fniv = vec_mov2,
1417         .fno = gen_helper_gvec_mov,
1418         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1419     };
1420     if (dofs != aofs) {
1421         tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g);
1422     } else {
1423         check_size_align(oprsz, maxsz, dofs);
1424         if (oprsz < maxsz) {
1425             expand_clr(dofs + oprsz, maxsz - oprsz);
1426         }
1427     }
1428 }
1429 
1430 void tcg_gen_gvec_dup_i32(unsigned vece, uint32_t dofs, uint32_t oprsz,
1431                           uint32_t maxsz, TCGv_i32 in)
1432 {
1433     check_size_align(oprsz, maxsz, dofs);
1434     tcg_debug_assert(vece <= MO_32);
1435     do_dup(vece, dofs, oprsz, maxsz, in, NULL, 0);
1436 }
1437 
1438 void tcg_gen_gvec_dup_i64(unsigned vece, uint32_t dofs, uint32_t oprsz,
1439                           uint32_t maxsz, TCGv_i64 in)
1440 {
1441     check_size_align(oprsz, maxsz, dofs);
1442     tcg_debug_assert(vece <= MO_64);
1443     do_dup(vece, dofs, oprsz, maxsz, NULL, in, 0);
1444 }
1445 
1446 void tcg_gen_gvec_dup_mem(unsigned vece, uint32_t dofs, uint32_t aofs,
1447                           uint32_t oprsz, uint32_t maxsz)
1448 {
1449     check_size_align(oprsz, maxsz, dofs);
1450     if (vece <= MO_64) {
1451         TCGType type = choose_vector_type(NULL, vece, oprsz, 0);
1452         if (type != 0) {
1453             TCGv_vec t_vec = tcg_temp_new_vec(type);
1454             tcg_gen_dup_mem_vec(vece, t_vec, cpu_env, aofs);
1455             do_dup_store(type, dofs, oprsz, maxsz, t_vec);
1456             tcg_temp_free_vec(t_vec);
1457         } else if (vece <= MO_32) {
1458             TCGv_i32 in = tcg_temp_new_i32();
1459             switch (vece) {
1460             case MO_8:
1461                 tcg_gen_ld8u_i32(in, cpu_env, aofs);
1462                 break;
1463             case MO_16:
1464                 tcg_gen_ld16u_i32(in, cpu_env, aofs);
1465                 break;
1466             default:
1467                 tcg_gen_ld_i32(in, cpu_env, aofs);
1468                 break;
1469             }
1470             do_dup(vece, dofs, oprsz, maxsz, in, NULL, 0);
1471             tcg_temp_free_i32(in);
1472         } else {
1473             TCGv_i64 in = tcg_temp_new_i64();
1474             tcg_gen_ld_i64(in, cpu_env, aofs);
1475             do_dup(vece, dofs, oprsz, maxsz, NULL, in, 0);
1476             tcg_temp_free_i64(in);
1477         }
1478     } else {
1479         /* 128-bit duplicate.  */
1480         /* ??? Dup to 256-bit vector.  */
1481         int i;
1482 
1483         tcg_debug_assert(vece == 4);
1484         tcg_debug_assert(oprsz >= 16);
1485         if (TCG_TARGET_HAS_v128) {
1486             TCGv_vec in = tcg_temp_new_vec(TCG_TYPE_V128);
1487 
1488             tcg_gen_ld_vec(in, cpu_env, aofs);
1489             for (i = 0; i < oprsz; i += 16) {
1490                 tcg_gen_st_vec(in, cpu_env, dofs + i);
1491             }
1492             tcg_temp_free_vec(in);
1493         } else {
1494             TCGv_i64 in0 = tcg_temp_new_i64();
1495             TCGv_i64 in1 = tcg_temp_new_i64();
1496 
1497             tcg_gen_ld_i64(in0, cpu_env, aofs);
1498             tcg_gen_ld_i64(in1, cpu_env, aofs + 8);
1499             for (i = 0; i < oprsz; i += 16) {
1500                 tcg_gen_st_i64(in0, cpu_env, dofs + i);
1501                 tcg_gen_st_i64(in1, cpu_env, dofs + i + 8);
1502             }
1503             tcg_temp_free_i64(in0);
1504             tcg_temp_free_i64(in1);
1505         }
1506         if (oprsz < maxsz) {
1507             expand_clr(dofs + oprsz, maxsz - oprsz);
1508         }
1509     }
1510 }
1511 
1512 void tcg_gen_gvec_dup64i(uint32_t dofs, uint32_t oprsz,
1513                          uint32_t maxsz, uint64_t x)
1514 {
1515     check_size_align(oprsz, maxsz, dofs);
1516     do_dup(MO_64, dofs, oprsz, maxsz, NULL, NULL, x);
1517 }
1518 
1519 void tcg_gen_gvec_dup32i(uint32_t dofs, uint32_t oprsz,
1520                          uint32_t maxsz, uint32_t x)
1521 {
1522     check_size_align(oprsz, maxsz, dofs);
1523     do_dup(MO_32, dofs, oprsz, maxsz, NULL, NULL, x);
1524 }
1525 
1526 void tcg_gen_gvec_dup16i(uint32_t dofs, uint32_t oprsz,
1527                          uint32_t maxsz, uint16_t x)
1528 {
1529     check_size_align(oprsz, maxsz, dofs);
1530     do_dup(MO_16, dofs, oprsz, maxsz, NULL, NULL, x);
1531 }
1532 
1533 void tcg_gen_gvec_dup8i(uint32_t dofs, uint32_t oprsz,
1534                          uint32_t maxsz, uint8_t x)
1535 {
1536     check_size_align(oprsz, maxsz, dofs);
1537     do_dup(MO_8, dofs, oprsz, maxsz, NULL, NULL, x);
1538 }
1539 
1540 void tcg_gen_gvec_not(unsigned vece, uint32_t dofs, uint32_t aofs,
1541                       uint32_t oprsz, uint32_t maxsz)
1542 {
1543     static const GVecGen2 g = {
1544         .fni8 = tcg_gen_not_i64,
1545         .fniv = tcg_gen_not_vec,
1546         .fno = gen_helper_gvec_not,
1547         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1548     };
1549     tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g);
1550 }
1551 
1552 /* Perform a vector addition using normal addition and a mask.  The mask
1553    should be the sign bit of each lane.  This 6-operation form is more
1554    efficient than separate additions when there are 4 or more lanes in
1555    the 64-bit operation.  */
1556 static void gen_addv_mask(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 m)
1557 {
1558     TCGv_i64 t1 = tcg_temp_new_i64();
1559     TCGv_i64 t2 = tcg_temp_new_i64();
1560     TCGv_i64 t3 = tcg_temp_new_i64();
1561 
1562     tcg_gen_andc_i64(t1, a, m);
1563     tcg_gen_andc_i64(t2, b, m);
1564     tcg_gen_xor_i64(t3, a, b);
1565     tcg_gen_add_i64(d, t1, t2);
1566     tcg_gen_and_i64(t3, t3, m);
1567     tcg_gen_xor_i64(d, d, t3);
1568 
1569     tcg_temp_free_i64(t1);
1570     tcg_temp_free_i64(t2);
1571     tcg_temp_free_i64(t3);
1572 }
1573 
1574 void tcg_gen_vec_add8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1575 {
1576     TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80));
1577     gen_addv_mask(d, a, b, m);
1578     tcg_temp_free_i64(m);
1579 }
1580 
1581 void tcg_gen_vec_add16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1582 {
1583     TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000));
1584     gen_addv_mask(d, a, b, m);
1585     tcg_temp_free_i64(m);
1586 }
1587 
1588 void tcg_gen_vec_add32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1589 {
1590     TCGv_i64 t1 = tcg_temp_new_i64();
1591     TCGv_i64 t2 = tcg_temp_new_i64();
1592 
1593     tcg_gen_andi_i64(t1, a, ~0xffffffffull);
1594     tcg_gen_add_i64(t2, a, b);
1595     tcg_gen_add_i64(t1, t1, b);
1596     tcg_gen_deposit_i64(d, t1, t2, 0, 32);
1597 
1598     tcg_temp_free_i64(t1);
1599     tcg_temp_free_i64(t2);
1600 }
1601 
1602 static const TCGOpcode vecop_list_add[] = { INDEX_op_add_vec, 0 };
1603 
1604 void tcg_gen_gvec_add(unsigned vece, uint32_t dofs, uint32_t aofs,
1605                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1606 {
1607     static const GVecGen3 g[4] = {
1608         { .fni8 = tcg_gen_vec_add8_i64,
1609           .fniv = tcg_gen_add_vec,
1610           .fno = gen_helper_gvec_add8,
1611           .opt_opc = vecop_list_add,
1612           .vece = MO_8 },
1613         { .fni8 = tcg_gen_vec_add16_i64,
1614           .fniv = tcg_gen_add_vec,
1615           .fno = gen_helper_gvec_add16,
1616           .opt_opc = vecop_list_add,
1617           .vece = MO_16 },
1618         { .fni4 = tcg_gen_add_i32,
1619           .fniv = tcg_gen_add_vec,
1620           .fno = gen_helper_gvec_add32,
1621           .opt_opc = vecop_list_add,
1622           .vece = MO_32 },
1623         { .fni8 = tcg_gen_add_i64,
1624           .fniv = tcg_gen_add_vec,
1625           .fno = gen_helper_gvec_add64,
1626           .opt_opc = vecop_list_add,
1627           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1628           .vece = MO_64 },
1629     };
1630 
1631     tcg_debug_assert(vece <= MO_64);
1632     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1633 }
1634 
1635 void tcg_gen_gvec_adds(unsigned vece, uint32_t dofs, uint32_t aofs,
1636                        TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1637 {
1638     static const GVecGen2s g[4] = {
1639         { .fni8 = tcg_gen_vec_add8_i64,
1640           .fniv = tcg_gen_add_vec,
1641           .fno = gen_helper_gvec_adds8,
1642           .opt_opc = vecop_list_add,
1643           .vece = MO_8 },
1644         { .fni8 = tcg_gen_vec_add16_i64,
1645           .fniv = tcg_gen_add_vec,
1646           .fno = gen_helper_gvec_adds16,
1647           .opt_opc = vecop_list_add,
1648           .vece = MO_16 },
1649         { .fni4 = tcg_gen_add_i32,
1650           .fniv = tcg_gen_add_vec,
1651           .fno = gen_helper_gvec_adds32,
1652           .opt_opc = vecop_list_add,
1653           .vece = MO_32 },
1654         { .fni8 = tcg_gen_add_i64,
1655           .fniv = tcg_gen_add_vec,
1656           .fno = gen_helper_gvec_adds64,
1657           .opt_opc = vecop_list_add,
1658           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1659           .vece = MO_64 },
1660     };
1661 
1662     tcg_debug_assert(vece <= MO_64);
1663     tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
1664 }
1665 
1666 void tcg_gen_gvec_addi(unsigned vece, uint32_t dofs, uint32_t aofs,
1667                        int64_t c, uint32_t oprsz, uint32_t maxsz)
1668 {
1669     TCGv_i64 tmp = tcg_const_i64(c);
1670     tcg_gen_gvec_adds(vece, dofs, aofs, tmp, oprsz, maxsz);
1671     tcg_temp_free_i64(tmp);
1672 }
1673 
1674 static const TCGOpcode vecop_list_sub[] = { INDEX_op_sub_vec, 0 };
1675 
1676 void tcg_gen_gvec_subs(unsigned vece, uint32_t dofs, uint32_t aofs,
1677                        TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1678 {
1679     static const GVecGen2s g[4] = {
1680         { .fni8 = tcg_gen_vec_sub8_i64,
1681           .fniv = tcg_gen_sub_vec,
1682           .fno = gen_helper_gvec_subs8,
1683           .opt_opc = vecop_list_sub,
1684           .vece = MO_8 },
1685         { .fni8 = tcg_gen_vec_sub16_i64,
1686           .fniv = tcg_gen_sub_vec,
1687           .fno = gen_helper_gvec_subs16,
1688           .opt_opc = vecop_list_sub,
1689           .vece = MO_16 },
1690         { .fni4 = tcg_gen_sub_i32,
1691           .fniv = tcg_gen_sub_vec,
1692           .fno = gen_helper_gvec_subs32,
1693           .opt_opc = vecop_list_sub,
1694           .vece = MO_32 },
1695         { .fni8 = tcg_gen_sub_i64,
1696           .fniv = tcg_gen_sub_vec,
1697           .fno = gen_helper_gvec_subs64,
1698           .opt_opc = vecop_list_sub,
1699           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1700           .vece = MO_64 },
1701     };
1702 
1703     tcg_debug_assert(vece <= MO_64);
1704     tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
1705 }
1706 
1707 /* Perform a vector subtraction using normal subtraction and a mask.
1708    Compare gen_addv_mask above.  */
1709 static void gen_subv_mask(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 m)
1710 {
1711     TCGv_i64 t1 = tcg_temp_new_i64();
1712     TCGv_i64 t2 = tcg_temp_new_i64();
1713     TCGv_i64 t3 = tcg_temp_new_i64();
1714 
1715     tcg_gen_or_i64(t1, a, m);
1716     tcg_gen_andc_i64(t2, b, m);
1717     tcg_gen_eqv_i64(t3, a, b);
1718     tcg_gen_sub_i64(d, t1, t2);
1719     tcg_gen_and_i64(t3, t3, m);
1720     tcg_gen_xor_i64(d, d, t3);
1721 
1722     tcg_temp_free_i64(t1);
1723     tcg_temp_free_i64(t2);
1724     tcg_temp_free_i64(t3);
1725 }
1726 
1727 void tcg_gen_vec_sub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1728 {
1729     TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80));
1730     gen_subv_mask(d, a, b, m);
1731     tcg_temp_free_i64(m);
1732 }
1733 
1734 void tcg_gen_vec_sub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1735 {
1736     TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000));
1737     gen_subv_mask(d, a, b, m);
1738     tcg_temp_free_i64(m);
1739 }
1740 
1741 void tcg_gen_vec_sub32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1742 {
1743     TCGv_i64 t1 = tcg_temp_new_i64();
1744     TCGv_i64 t2 = tcg_temp_new_i64();
1745 
1746     tcg_gen_andi_i64(t1, b, ~0xffffffffull);
1747     tcg_gen_sub_i64(t2, a, b);
1748     tcg_gen_sub_i64(t1, a, t1);
1749     tcg_gen_deposit_i64(d, t1, t2, 0, 32);
1750 
1751     tcg_temp_free_i64(t1);
1752     tcg_temp_free_i64(t2);
1753 }
1754 
1755 void tcg_gen_gvec_sub(unsigned vece, uint32_t dofs, uint32_t aofs,
1756                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1757 {
1758     static const GVecGen3 g[4] = {
1759         { .fni8 = tcg_gen_vec_sub8_i64,
1760           .fniv = tcg_gen_sub_vec,
1761           .fno = gen_helper_gvec_sub8,
1762           .opt_opc = vecop_list_sub,
1763           .vece = MO_8 },
1764         { .fni8 = tcg_gen_vec_sub16_i64,
1765           .fniv = tcg_gen_sub_vec,
1766           .fno = gen_helper_gvec_sub16,
1767           .opt_opc = vecop_list_sub,
1768           .vece = MO_16 },
1769         { .fni4 = tcg_gen_sub_i32,
1770           .fniv = tcg_gen_sub_vec,
1771           .fno = gen_helper_gvec_sub32,
1772           .opt_opc = vecop_list_sub,
1773           .vece = MO_32 },
1774         { .fni8 = tcg_gen_sub_i64,
1775           .fniv = tcg_gen_sub_vec,
1776           .fno = gen_helper_gvec_sub64,
1777           .opt_opc = vecop_list_sub,
1778           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1779           .vece = MO_64 },
1780     };
1781 
1782     tcg_debug_assert(vece <= MO_64);
1783     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1784 }
1785 
1786 static const TCGOpcode vecop_list_mul[] = { INDEX_op_mul_vec, 0 };
1787 
1788 void tcg_gen_gvec_mul(unsigned vece, uint32_t dofs, uint32_t aofs,
1789                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1790 {
1791     static const GVecGen3 g[4] = {
1792         { .fniv = tcg_gen_mul_vec,
1793           .fno = gen_helper_gvec_mul8,
1794           .opt_opc = vecop_list_mul,
1795           .vece = MO_8 },
1796         { .fniv = tcg_gen_mul_vec,
1797           .fno = gen_helper_gvec_mul16,
1798           .opt_opc = vecop_list_mul,
1799           .vece = MO_16 },
1800         { .fni4 = tcg_gen_mul_i32,
1801           .fniv = tcg_gen_mul_vec,
1802           .fno = gen_helper_gvec_mul32,
1803           .opt_opc = vecop_list_mul,
1804           .vece = MO_32 },
1805         { .fni8 = tcg_gen_mul_i64,
1806           .fniv = tcg_gen_mul_vec,
1807           .fno = gen_helper_gvec_mul64,
1808           .opt_opc = vecop_list_mul,
1809           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1810           .vece = MO_64 },
1811     };
1812 
1813     tcg_debug_assert(vece <= MO_64);
1814     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1815 }
1816 
1817 void tcg_gen_gvec_muls(unsigned vece, uint32_t dofs, uint32_t aofs,
1818                        TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1819 {
1820     static const GVecGen2s g[4] = {
1821         { .fniv = tcg_gen_mul_vec,
1822           .fno = gen_helper_gvec_muls8,
1823           .opt_opc = vecop_list_mul,
1824           .vece = MO_8 },
1825         { .fniv = tcg_gen_mul_vec,
1826           .fno = gen_helper_gvec_muls16,
1827           .opt_opc = vecop_list_mul,
1828           .vece = MO_16 },
1829         { .fni4 = tcg_gen_mul_i32,
1830           .fniv = tcg_gen_mul_vec,
1831           .fno = gen_helper_gvec_muls32,
1832           .opt_opc = vecop_list_mul,
1833           .vece = MO_32 },
1834         { .fni8 = tcg_gen_mul_i64,
1835           .fniv = tcg_gen_mul_vec,
1836           .fno = gen_helper_gvec_muls64,
1837           .opt_opc = vecop_list_mul,
1838           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1839           .vece = MO_64 },
1840     };
1841 
1842     tcg_debug_assert(vece <= MO_64);
1843     tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
1844 }
1845 
1846 void tcg_gen_gvec_muli(unsigned vece, uint32_t dofs, uint32_t aofs,
1847                        int64_t c, uint32_t oprsz, uint32_t maxsz)
1848 {
1849     TCGv_i64 tmp = tcg_const_i64(c);
1850     tcg_gen_gvec_muls(vece, dofs, aofs, tmp, oprsz, maxsz);
1851     tcg_temp_free_i64(tmp);
1852 }
1853 
1854 void tcg_gen_gvec_ssadd(unsigned vece, uint32_t dofs, uint32_t aofs,
1855                         uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1856 {
1857     static const TCGOpcode vecop_list[] = { INDEX_op_ssadd_vec, 0 };
1858     static const GVecGen3 g[4] = {
1859         { .fniv = tcg_gen_ssadd_vec,
1860           .fno = gen_helper_gvec_ssadd8,
1861           .opt_opc = vecop_list,
1862           .vece = MO_8 },
1863         { .fniv = tcg_gen_ssadd_vec,
1864           .fno = gen_helper_gvec_ssadd16,
1865           .opt_opc = vecop_list,
1866           .vece = MO_16 },
1867         { .fniv = tcg_gen_ssadd_vec,
1868           .fno = gen_helper_gvec_ssadd32,
1869           .opt_opc = vecop_list,
1870           .vece = MO_32 },
1871         { .fniv = tcg_gen_ssadd_vec,
1872           .fno = gen_helper_gvec_ssadd64,
1873           .opt_opc = vecop_list,
1874           .vece = MO_64 },
1875     };
1876     tcg_debug_assert(vece <= MO_64);
1877     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1878 }
1879 
1880 void tcg_gen_gvec_sssub(unsigned vece, uint32_t dofs, uint32_t aofs,
1881                         uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1882 {
1883     static const TCGOpcode vecop_list[] = { INDEX_op_sssub_vec, 0 };
1884     static const GVecGen3 g[4] = {
1885         { .fniv = tcg_gen_sssub_vec,
1886           .fno = gen_helper_gvec_sssub8,
1887           .opt_opc = vecop_list,
1888           .vece = MO_8 },
1889         { .fniv = tcg_gen_sssub_vec,
1890           .fno = gen_helper_gvec_sssub16,
1891           .opt_opc = vecop_list,
1892           .vece = MO_16 },
1893         { .fniv = tcg_gen_sssub_vec,
1894           .fno = gen_helper_gvec_sssub32,
1895           .opt_opc = vecop_list,
1896           .vece = MO_32 },
1897         { .fniv = tcg_gen_sssub_vec,
1898           .fno = gen_helper_gvec_sssub64,
1899           .opt_opc = vecop_list,
1900           .vece = MO_64 },
1901     };
1902     tcg_debug_assert(vece <= MO_64);
1903     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1904 }
1905 
1906 static void tcg_gen_usadd_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1907 {
1908     TCGv_i32 max = tcg_const_i32(-1);
1909     tcg_gen_add_i32(d, a, b);
1910     tcg_gen_movcond_i32(TCG_COND_LTU, d, d, a, max, d);
1911     tcg_temp_free_i32(max);
1912 }
1913 
1914 static void tcg_gen_usadd_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1915 {
1916     TCGv_i64 max = tcg_const_i64(-1);
1917     tcg_gen_add_i64(d, a, b);
1918     tcg_gen_movcond_i64(TCG_COND_LTU, d, d, a, max, d);
1919     tcg_temp_free_i64(max);
1920 }
1921 
1922 void tcg_gen_gvec_usadd(unsigned vece, uint32_t dofs, uint32_t aofs,
1923                         uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1924 {
1925     static const TCGOpcode vecop_list[] = { INDEX_op_usadd_vec, 0 };
1926     static const GVecGen3 g[4] = {
1927         { .fniv = tcg_gen_usadd_vec,
1928           .fno = gen_helper_gvec_usadd8,
1929           .opt_opc = vecop_list,
1930           .vece = MO_8 },
1931         { .fniv = tcg_gen_usadd_vec,
1932           .fno = gen_helper_gvec_usadd16,
1933           .opt_opc = vecop_list,
1934           .vece = MO_16 },
1935         { .fni4 = tcg_gen_usadd_i32,
1936           .fniv = tcg_gen_usadd_vec,
1937           .fno = gen_helper_gvec_usadd32,
1938           .opt_opc = vecop_list,
1939           .vece = MO_32 },
1940         { .fni8 = tcg_gen_usadd_i64,
1941           .fniv = tcg_gen_usadd_vec,
1942           .fno = gen_helper_gvec_usadd64,
1943           .opt_opc = vecop_list,
1944           .vece = MO_64 }
1945     };
1946     tcg_debug_assert(vece <= MO_64);
1947     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1948 }
1949 
1950 static void tcg_gen_ussub_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1951 {
1952     TCGv_i32 min = tcg_const_i32(0);
1953     tcg_gen_sub_i32(d, a, b);
1954     tcg_gen_movcond_i32(TCG_COND_LTU, d, a, b, min, d);
1955     tcg_temp_free_i32(min);
1956 }
1957 
1958 static void tcg_gen_ussub_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1959 {
1960     TCGv_i64 min = tcg_const_i64(0);
1961     tcg_gen_sub_i64(d, a, b);
1962     tcg_gen_movcond_i64(TCG_COND_LTU, d, a, b, min, d);
1963     tcg_temp_free_i64(min);
1964 }
1965 
1966 void tcg_gen_gvec_ussub(unsigned vece, uint32_t dofs, uint32_t aofs,
1967                         uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1968 {
1969     static const TCGOpcode vecop_list[] = { INDEX_op_ussub_vec, 0 };
1970     static const GVecGen3 g[4] = {
1971         { .fniv = tcg_gen_ussub_vec,
1972           .fno = gen_helper_gvec_ussub8,
1973           .opt_opc = vecop_list,
1974           .vece = MO_8 },
1975         { .fniv = tcg_gen_ussub_vec,
1976           .fno = gen_helper_gvec_ussub16,
1977           .opt_opc = vecop_list,
1978           .vece = MO_16 },
1979         { .fni4 = tcg_gen_ussub_i32,
1980           .fniv = tcg_gen_ussub_vec,
1981           .fno = gen_helper_gvec_ussub32,
1982           .opt_opc = vecop_list,
1983           .vece = MO_32 },
1984         { .fni8 = tcg_gen_ussub_i64,
1985           .fniv = tcg_gen_ussub_vec,
1986           .fno = gen_helper_gvec_ussub64,
1987           .opt_opc = vecop_list,
1988           .vece = MO_64 }
1989     };
1990     tcg_debug_assert(vece <= MO_64);
1991     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1992 }
1993 
1994 void tcg_gen_gvec_smin(unsigned vece, uint32_t dofs, uint32_t aofs,
1995                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1996 {
1997     static const TCGOpcode vecop_list[] = { INDEX_op_smin_vec, 0 };
1998     static const GVecGen3 g[4] = {
1999         { .fniv = tcg_gen_smin_vec,
2000           .fno = gen_helper_gvec_smin8,
2001           .opt_opc = vecop_list,
2002           .vece = MO_8 },
2003         { .fniv = tcg_gen_smin_vec,
2004           .fno = gen_helper_gvec_smin16,
2005           .opt_opc = vecop_list,
2006           .vece = MO_16 },
2007         { .fni4 = tcg_gen_smin_i32,
2008           .fniv = tcg_gen_smin_vec,
2009           .fno = gen_helper_gvec_smin32,
2010           .opt_opc = vecop_list,
2011           .vece = MO_32 },
2012         { .fni8 = tcg_gen_smin_i64,
2013           .fniv = tcg_gen_smin_vec,
2014           .fno = gen_helper_gvec_smin64,
2015           .opt_opc = vecop_list,
2016           .vece = MO_64 }
2017     };
2018     tcg_debug_assert(vece <= MO_64);
2019     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2020 }
2021 
2022 void tcg_gen_gvec_umin(unsigned vece, uint32_t dofs, uint32_t aofs,
2023                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2024 {
2025     static const TCGOpcode vecop_list[] = { INDEX_op_umin_vec, 0 };
2026     static const GVecGen3 g[4] = {
2027         { .fniv = tcg_gen_umin_vec,
2028           .fno = gen_helper_gvec_umin8,
2029           .opt_opc = vecop_list,
2030           .vece = MO_8 },
2031         { .fniv = tcg_gen_umin_vec,
2032           .fno = gen_helper_gvec_umin16,
2033           .opt_opc = vecop_list,
2034           .vece = MO_16 },
2035         { .fni4 = tcg_gen_umin_i32,
2036           .fniv = tcg_gen_umin_vec,
2037           .fno = gen_helper_gvec_umin32,
2038           .opt_opc = vecop_list,
2039           .vece = MO_32 },
2040         { .fni8 = tcg_gen_umin_i64,
2041           .fniv = tcg_gen_umin_vec,
2042           .fno = gen_helper_gvec_umin64,
2043           .opt_opc = vecop_list,
2044           .vece = MO_64 }
2045     };
2046     tcg_debug_assert(vece <= MO_64);
2047     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2048 }
2049 
2050 void tcg_gen_gvec_smax(unsigned vece, uint32_t dofs, uint32_t aofs,
2051                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2052 {
2053     static const TCGOpcode vecop_list[] = { INDEX_op_smax_vec, 0 };
2054     static const GVecGen3 g[4] = {
2055         { .fniv = tcg_gen_smax_vec,
2056           .fno = gen_helper_gvec_smax8,
2057           .opt_opc = vecop_list,
2058           .vece = MO_8 },
2059         { .fniv = tcg_gen_smax_vec,
2060           .fno = gen_helper_gvec_smax16,
2061           .opt_opc = vecop_list,
2062           .vece = MO_16 },
2063         { .fni4 = tcg_gen_smax_i32,
2064           .fniv = tcg_gen_smax_vec,
2065           .fno = gen_helper_gvec_smax32,
2066           .opt_opc = vecop_list,
2067           .vece = MO_32 },
2068         { .fni8 = tcg_gen_smax_i64,
2069           .fniv = tcg_gen_smax_vec,
2070           .fno = gen_helper_gvec_smax64,
2071           .opt_opc = vecop_list,
2072           .vece = MO_64 }
2073     };
2074     tcg_debug_assert(vece <= MO_64);
2075     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2076 }
2077 
2078 void tcg_gen_gvec_umax(unsigned vece, uint32_t dofs, uint32_t aofs,
2079                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2080 {
2081     static const TCGOpcode vecop_list[] = { INDEX_op_umax_vec, 0 };
2082     static const GVecGen3 g[4] = {
2083         { .fniv = tcg_gen_umax_vec,
2084           .fno = gen_helper_gvec_umax8,
2085           .opt_opc = vecop_list,
2086           .vece = MO_8 },
2087         { .fniv = tcg_gen_umax_vec,
2088           .fno = gen_helper_gvec_umax16,
2089           .opt_opc = vecop_list,
2090           .vece = MO_16 },
2091         { .fni4 = tcg_gen_umax_i32,
2092           .fniv = tcg_gen_umax_vec,
2093           .fno = gen_helper_gvec_umax32,
2094           .opt_opc = vecop_list,
2095           .vece = MO_32 },
2096         { .fni8 = tcg_gen_umax_i64,
2097           .fniv = tcg_gen_umax_vec,
2098           .fno = gen_helper_gvec_umax64,
2099           .opt_opc = vecop_list,
2100           .vece = MO_64 }
2101     };
2102     tcg_debug_assert(vece <= MO_64);
2103     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2104 }
2105 
2106 /* Perform a vector negation using normal negation and a mask.
2107    Compare gen_subv_mask above.  */
2108 static void gen_negv_mask(TCGv_i64 d, TCGv_i64 b, TCGv_i64 m)
2109 {
2110     TCGv_i64 t2 = tcg_temp_new_i64();
2111     TCGv_i64 t3 = tcg_temp_new_i64();
2112 
2113     tcg_gen_andc_i64(t3, m, b);
2114     tcg_gen_andc_i64(t2, b, m);
2115     tcg_gen_sub_i64(d, m, t2);
2116     tcg_gen_xor_i64(d, d, t3);
2117 
2118     tcg_temp_free_i64(t2);
2119     tcg_temp_free_i64(t3);
2120 }
2121 
2122 void tcg_gen_vec_neg8_i64(TCGv_i64 d, TCGv_i64 b)
2123 {
2124     TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80));
2125     gen_negv_mask(d, b, m);
2126     tcg_temp_free_i64(m);
2127 }
2128 
2129 void tcg_gen_vec_neg16_i64(TCGv_i64 d, TCGv_i64 b)
2130 {
2131     TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000));
2132     gen_negv_mask(d, b, m);
2133     tcg_temp_free_i64(m);
2134 }
2135 
2136 void tcg_gen_vec_neg32_i64(TCGv_i64 d, TCGv_i64 b)
2137 {
2138     TCGv_i64 t1 = tcg_temp_new_i64();
2139     TCGv_i64 t2 = tcg_temp_new_i64();
2140 
2141     tcg_gen_andi_i64(t1, b, ~0xffffffffull);
2142     tcg_gen_neg_i64(t2, b);
2143     tcg_gen_neg_i64(t1, t1);
2144     tcg_gen_deposit_i64(d, t1, t2, 0, 32);
2145 
2146     tcg_temp_free_i64(t1);
2147     tcg_temp_free_i64(t2);
2148 }
2149 
2150 void tcg_gen_gvec_neg(unsigned vece, uint32_t dofs, uint32_t aofs,
2151                       uint32_t oprsz, uint32_t maxsz)
2152 {
2153     static const TCGOpcode vecop_list[] = { INDEX_op_neg_vec, 0 };
2154     static const GVecGen2 g[4] = {
2155         { .fni8 = tcg_gen_vec_neg8_i64,
2156           .fniv = tcg_gen_neg_vec,
2157           .fno = gen_helper_gvec_neg8,
2158           .opt_opc = vecop_list,
2159           .vece = MO_8 },
2160         { .fni8 = tcg_gen_vec_neg16_i64,
2161           .fniv = tcg_gen_neg_vec,
2162           .fno = gen_helper_gvec_neg16,
2163           .opt_opc = vecop_list,
2164           .vece = MO_16 },
2165         { .fni4 = tcg_gen_neg_i32,
2166           .fniv = tcg_gen_neg_vec,
2167           .fno = gen_helper_gvec_neg32,
2168           .opt_opc = vecop_list,
2169           .vece = MO_32 },
2170         { .fni8 = tcg_gen_neg_i64,
2171           .fniv = tcg_gen_neg_vec,
2172           .fno = gen_helper_gvec_neg64,
2173           .opt_opc = vecop_list,
2174           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2175           .vece = MO_64 },
2176     };
2177 
2178     tcg_debug_assert(vece <= MO_64);
2179     tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g[vece]);
2180 }
2181 
2182 static void gen_absv_mask(TCGv_i64 d, TCGv_i64 b, unsigned vece)
2183 {
2184     TCGv_i64 t = tcg_temp_new_i64();
2185     int nbit = 8 << vece;
2186 
2187     /* Create -1 for each negative element.  */
2188     tcg_gen_shri_i64(t, b, nbit - 1);
2189     tcg_gen_andi_i64(t, t, dup_const(vece, 1));
2190     tcg_gen_muli_i64(t, t, (1 << nbit) - 1);
2191 
2192     /*
2193      * Invert (via xor -1) and add one (via sub -1).
2194      * Because of the ordering the msb is cleared,
2195      * so we never have carry into the next element.
2196      */
2197     tcg_gen_xor_i64(d, b, t);
2198     tcg_gen_sub_i64(d, d, t);
2199 
2200     tcg_temp_free_i64(t);
2201 }
2202 
2203 static void tcg_gen_vec_abs8_i64(TCGv_i64 d, TCGv_i64 b)
2204 {
2205     gen_absv_mask(d, b, MO_8);
2206 }
2207 
2208 static void tcg_gen_vec_abs16_i64(TCGv_i64 d, TCGv_i64 b)
2209 {
2210     gen_absv_mask(d, b, MO_16);
2211 }
2212 
2213 void tcg_gen_gvec_abs(unsigned vece, uint32_t dofs, uint32_t aofs,
2214                       uint32_t oprsz, uint32_t maxsz)
2215 {
2216     static const TCGOpcode vecop_list[] = { INDEX_op_abs_vec, 0 };
2217     static const GVecGen2 g[4] = {
2218         { .fni8 = tcg_gen_vec_abs8_i64,
2219           .fniv = tcg_gen_abs_vec,
2220           .fno = gen_helper_gvec_abs8,
2221           .opt_opc = vecop_list,
2222           .vece = MO_8 },
2223         { .fni8 = tcg_gen_vec_abs16_i64,
2224           .fniv = tcg_gen_abs_vec,
2225           .fno = gen_helper_gvec_abs16,
2226           .opt_opc = vecop_list,
2227           .vece = MO_16 },
2228         { .fni4 = tcg_gen_abs_i32,
2229           .fniv = tcg_gen_abs_vec,
2230           .fno = gen_helper_gvec_abs32,
2231           .opt_opc = vecop_list,
2232           .vece = MO_32 },
2233         { .fni8 = tcg_gen_abs_i64,
2234           .fniv = tcg_gen_abs_vec,
2235           .fno = gen_helper_gvec_abs64,
2236           .opt_opc = vecop_list,
2237           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2238           .vece = MO_64 },
2239     };
2240 
2241     tcg_debug_assert(vece <= MO_64);
2242     tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g[vece]);
2243 }
2244 
2245 void tcg_gen_gvec_and(unsigned vece, uint32_t dofs, uint32_t aofs,
2246                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2247 {
2248     static const GVecGen3 g = {
2249         .fni8 = tcg_gen_and_i64,
2250         .fniv = tcg_gen_and_vec,
2251         .fno = gen_helper_gvec_and,
2252         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2253     };
2254 
2255     if (aofs == bofs) {
2256         tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2257     } else {
2258         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2259     }
2260 }
2261 
2262 void tcg_gen_gvec_or(unsigned vece, uint32_t dofs, uint32_t aofs,
2263                      uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2264 {
2265     static const GVecGen3 g = {
2266         .fni8 = tcg_gen_or_i64,
2267         .fniv = tcg_gen_or_vec,
2268         .fno = gen_helper_gvec_or,
2269         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2270     };
2271 
2272     if (aofs == bofs) {
2273         tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2274     } else {
2275         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2276     }
2277 }
2278 
2279 void tcg_gen_gvec_xor(unsigned vece, uint32_t dofs, uint32_t aofs,
2280                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2281 {
2282     static const GVecGen3 g = {
2283         .fni8 = tcg_gen_xor_i64,
2284         .fniv = tcg_gen_xor_vec,
2285         .fno = gen_helper_gvec_xor,
2286         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2287     };
2288 
2289     if (aofs == bofs) {
2290         tcg_gen_gvec_dup8i(dofs, oprsz, maxsz, 0);
2291     } else {
2292         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2293     }
2294 }
2295 
2296 void tcg_gen_gvec_andc(unsigned vece, uint32_t dofs, uint32_t aofs,
2297                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2298 {
2299     static const GVecGen3 g = {
2300         .fni8 = tcg_gen_andc_i64,
2301         .fniv = tcg_gen_andc_vec,
2302         .fno = gen_helper_gvec_andc,
2303         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2304     };
2305 
2306     if (aofs == bofs) {
2307         tcg_gen_gvec_dup8i(dofs, oprsz, maxsz, 0);
2308     } else {
2309         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2310     }
2311 }
2312 
2313 void tcg_gen_gvec_orc(unsigned vece, uint32_t dofs, uint32_t aofs,
2314                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2315 {
2316     static const GVecGen3 g = {
2317         .fni8 = tcg_gen_orc_i64,
2318         .fniv = tcg_gen_orc_vec,
2319         .fno = gen_helper_gvec_orc,
2320         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2321     };
2322 
2323     if (aofs == bofs) {
2324         tcg_gen_gvec_dup8i(dofs, oprsz, maxsz, -1);
2325     } else {
2326         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2327     }
2328 }
2329 
2330 void tcg_gen_gvec_nand(unsigned vece, uint32_t dofs, uint32_t aofs,
2331                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2332 {
2333     static const GVecGen3 g = {
2334         .fni8 = tcg_gen_nand_i64,
2335         .fniv = tcg_gen_nand_vec,
2336         .fno = gen_helper_gvec_nand,
2337         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2338     };
2339 
2340     if (aofs == bofs) {
2341         tcg_gen_gvec_not(vece, dofs, aofs, oprsz, maxsz);
2342     } else {
2343         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2344     }
2345 }
2346 
2347 void tcg_gen_gvec_nor(unsigned vece, uint32_t dofs, uint32_t aofs,
2348                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2349 {
2350     static const GVecGen3 g = {
2351         .fni8 = tcg_gen_nor_i64,
2352         .fniv = tcg_gen_nor_vec,
2353         .fno = gen_helper_gvec_nor,
2354         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2355     };
2356 
2357     if (aofs == bofs) {
2358         tcg_gen_gvec_not(vece, dofs, aofs, oprsz, maxsz);
2359     } else {
2360         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2361     }
2362 }
2363 
2364 void tcg_gen_gvec_eqv(unsigned vece, uint32_t dofs, uint32_t aofs,
2365                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2366 {
2367     static const GVecGen3 g = {
2368         .fni8 = tcg_gen_eqv_i64,
2369         .fniv = tcg_gen_eqv_vec,
2370         .fno = gen_helper_gvec_eqv,
2371         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2372     };
2373 
2374     if (aofs == bofs) {
2375         tcg_gen_gvec_dup8i(dofs, oprsz, maxsz, -1);
2376     } else {
2377         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2378     }
2379 }
2380 
2381 static const GVecGen2s gop_ands = {
2382     .fni8 = tcg_gen_and_i64,
2383     .fniv = tcg_gen_and_vec,
2384     .fno = gen_helper_gvec_ands,
2385     .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2386     .vece = MO_64
2387 };
2388 
2389 void tcg_gen_gvec_ands(unsigned vece, uint32_t dofs, uint32_t aofs,
2390                        TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2391 {
2392     TCGv_i64 tmp = tcg_temp_new_i64();
2393     gen_dup_i64(vece, tmp, c);
2394     tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ands);
2395     tcg_temp_free_i64(tmp);
2396 }
2397 
2398 void tcg_gen_gvec_andi(unsigned vece, uint32_t dofs, uint32_t aofs,
2399                        int64_t c, uint32_t oprsz, uint32_t maxsz)
2400 {
2401     TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c));
2402     tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ands);
2403     tcg_temp_free_i64(tmp);
2404 }
2405 
2406 static const GVecGen2s gop_xors = {
2407     .fni8 = tcg_gen_xor_i64,
2408     .fniv = tcg_gen_xor_vec,
2409     .fno = gen_helper_gvec_xors,
2410     .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2411     .vece = MO_64
2412 };
2413 
2414 void tcg_gen_gvec_xors(unsigned vece, uint32_t dofs, uint32_t aofs,
2415                        TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2416 {
2417     TCGv_i64 tmp = tcg_temp_new_i64();
2418     gen_dup_i64(vece, tmp, c);
2419     tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_xors);
2420     tcg_temp_free_i64(tmp);
2421 }
2422 
2423 void tcg_gen_gvec_xori(unsigned vece, uint32_t dofs, uint32_t aofs,
2424                        int64_t c, uint32_t oprsz, uint32_t maxsz)
2425 {
2426     TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c));
2427     tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_xors);
2428     tcg_temp_free_i64(tmp);
2429 }
2430 
2431 static const GVecGen2s gop_ors = {
2432     .fni8 = tcg_gen_or_i64,
2433     .fniv = tcg_gen_or_vec,
2434     .fno = gen_helper_gvec_ors,
2435     .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2436     .vece = MO_64
2437 };
2438 
2439 void tcg_gen_gvec_ors(unsigned vece, uint32_t dofs, uint32_t aofs,
2440                       TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2441 {
2442     TCGv_i64 tmp = tcg_temp_new_i64();
2443     gen_dup_i64(vece, tmp, c);
2444     tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ors);
2445     tcg_temp_free_i64(tmp);
2446 }
2447 
2448 void tcg_gen_gvec_ori(unsigned vece, uint32_t dofs, uint32_t aofs,
2449                       int64_t c, uint32_t oprsz, uint32_t maxsz)
2450 {
2451     TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c));
2452     tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ors);
2453     tcg_temp_free_i64(tmp);
2454 }
2455 
2456 void tcg_gen_vec_shl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2457 {
2458     uint64_t mask = dup_const(MO_8, 0xff << c);
2459     tcg_gen_shli_i64(d, a, c);
2460     tcg_gen_andi_i64(d, d, mask);
2461 }
2462 
2463 void tcg_gen_vec_shl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2464 {
2465     uint64_t mask = dup_const(MO_16, 0xffff << c);
2466     tcg_gen_shli_i64(d, a, c);
2467     tcg_gen_andi_i64(d, d, mask);
2468 }
2469 
2470 void tcg_gen_gvec_shli(unsigned vece, uint32_t dofs, uint32_t aofs,
2471                        int64_t shift, uint32_t oprsz, uint32_t maxsz)
2472 {
2473     static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 };
2474     static const GVecGen2i g[4] = {
2475         { .fni8 = tcg_gen_vec_shl8i_i64,
2476           .fniv = tcg_gen_shli_vec,
2477           .fno = gen_helper_gvec_shl8i,
2478           .opt_opc = vecop_list,
2479           .vece = MO_8 },
2480         { .fni8 = tcg_gen_vec_shl16i_i64,
2481           .fniv = tcg_gen_shli_vec,
2482           .fno = gen_helper_gvec_shl16i,
2483           .opt_opc = vecop_list,
2484           .vece = MO_16 },
2485         { .fni4 = tcg_gen_shli_i32,
2486           .fniv = tcg_gen_shli_vec,
2487           .fno = gen_helper_gvec_shl32i,
2488           .opt_opc = vecop_list,
2489           .vece = MO_32 },
2490         { .fni8 = tcg_gen_shli_i64,
2491           .fniv = tcg_gen_shli_vec,
2492           .fno = gen_helper_gvec_shl64i,
2493           .opt_opc = vecop_list,
2494           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2495           .vece = MO_64 },
2496     };
2497 
2498     tcg_debug_assert(vece <= MO_64);
2499     tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2500     if (shift == 0) {
2501         tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2502     } else {
2503         tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2504     }
2505 }
2506 
2507 void tcg_gen_vec_shr8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2508 {
2509     uint64_t mask = dup_const(MO_8, 0xff >> c);
2510     tcg_gen_shri_i64(d, a, c);
2511     tcg_gen_andi_i64(d, d, mask);
2512 }
2513 
2514 void tcg_gen_vec_shr16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2515 {
2516     uint64_t mask = dup_const(MO_16, 0xffff >> c);
2517     tcg_gen_shri_i64(d, a, c);
2518     tcg_gen_andi_i64(d, d, mask);
2519 }
2520 
2521 void tcg_gen_gvec_shri(unsigned vece, uint32_t dofs, uint32_t aofs,
2522                        int64_t shift, uint32_t oprsz, uint32_t maxsz)
2523 {
2524     static const TCGOpcode vecop_list[] = { INDEX_op_shri_vec, 0 };
2525     static const GVecGen2i g[4] = {
2526         { .fni8 = tcg_gen_vec_shr8i_i64,
2527           .fniv = tcg_gen_shri_vec,
2528           .fno = gen_helper_gvec_shr8i,
2529           .opt_opc = vecop_list,
2530           .vece = MO_8 },
2531         { .fni8 = tcg_gen_vec_shr16i_i64,
2532           .fniv = tcg_gen_shri_vec,
2533           .fno = gen_helper_gvec_shr16i,
2534           .opt_opc = vecop_list,
2535           .vece = MO_16 },
2536         { .fni4 = tcg_gen_shri_i32,
2537           .fniv = tcg_gen_shri_vec,
2538           .fno = gen_helper_gvec_shr32i,
2539           .opt_opc = vecop_list,
2540           .vece = MO_32 },
2541         { .fni8 = tcg_gen_shri_i64,
2542           .fniv = tcg_gen_shri_vec,
2543           .fno = gen_helper_gvec_shr64i,
2544           .opt_opc = vecop_list,
2545           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2546           .vece = MO_64 },
2547     };
2548 
2549     tcg_debug_assert(vece <= MO_64);
2550     tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2551     if (shift == 0) {
2552         tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2553     } else {
2554         tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2555     }
2556 }
2557 
2558 void tcg_gen_vec_sar8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2559 {
2560     uint64_t s_mask = dup_const(MO_8, 0x80 >> c);
2561     uint64_t c_mask = dup_const(MO_8, 0xff >> c);
2562     TCGv_i64 s = tcg_temp_new_i64();
2563 
2564     tcg_gen_shri_i64(d, a, c);
2565     tcg_gen_andi_i64(s, d, s_mask);  /* isolate (shifted) sign bit */
2566     tcg_gen_muli_i64(s, s, (2 << c) - 2); /* replicate isolated signs */
2567     tcg_gen_andi_i64(d, d, c_mask);  /* clear out bits above sign  */
2568     tcg_gen_or_i64(d, d, s);         /* include sign extension */
2569     tcg_temp_free_i64(s);
2570 }
2571 
2572 void tcg_gen_vec_sar16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2573 {
2574     uint64_t s_mask = dup_const(MO_16, 0x8000 >> c);
2575     uint64_t c_mask = dup_const(MO_16, 0xffff >> c);
2576     TCGv_i64 s = tcg_temp_new_i64();
2577 
2578     tcg_gen_shri_i64(d, a, c);
2579     tcg_gen_andi_i64(s, d, s_mask);  /* isolate (shifted) sign bit */
2580     tcg_gen_andi_i64(d, d, c_mask);  /* clear out bits above sign  */
2581     tcg_gen_muli_i64(s, s, (2 << c) - 2); /* replicate isolated signs */
2582     tcg_gen_or_i64(d, d, s);         /* include sign extension */
2583     tcg_temp_free_i64(s);
2584 }
2585 
2586 void tcg_gen_gvec_sari(unsigned vece, uint32_t dofs, uint32_t aofs,
2587                        int64_t shift, uint32_t oprsz, uint32_t maxsz)
2588 {
2589     static const TCGOpcode vecop_list[] = { INDEX_op_sari_vec, 0 };
2590     static const GVecGen2i g[4] = {
2591         { .fni8 = tcg_gen_vec_sar8i_i64,
2592           .fniv = tcg_gen_sari_vec,
2593           .fno = gen_helper_gvec_sar8i,
2594           .opt_opc = vecop_list,
2595           .vece = MO_8 },
2596         { .fni8 = tcg_gen_vec_sar16i_i64,
2597           .fniv = tcg_gen_sari_vec,
2598           .fno = gen_helper_gvec_sar16i,
2599           .opt_opc = vecop_list,
2600           .vece = MO_16 },
2601         { .fni4 = tcg_gen_sari_i32,
2602           .fniv = tcg_gen_sari_vec,
2603           .fno = gen_helper_gvec_sar32i,
2604           .opt_opc = vecop_list,
2605           .vece = MO_32 },
2606         { .fni8 = tcg_gen_sari_i64,
2607           .fniv = tcg_gen_sari_vec,
2608           .fno = gen_helper_gvec_sar64i,
2609           .opt_opc = vecop_list,
2610           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2611           .vece = MO_64 },
2612     };
2613 
2614     tcg_debug_assert(vece <= MO_64);
2615     tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2616     if (shift == 0) {
2617         tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2618     } else {
2619         tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2620     }
2621 }
2622 
2623 /*
2624  * Specialized generation vector shifts by a non-constant scalar.
2625  */
2626 
2627 typedef struct {
2628     void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32);
2629     void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64);
2630     void (*fniv_s)(unsigned, TCGv_vec, TCGv_vec, TCGv_i32);
2631     void (*fniv_v)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec);
2632     gen_helper_gvec_2 *fno[4];
2633     TCGOpcode s_list[2];
2634     TCGOpcode v_list[2];
2635 } GVecGen2sh;
2636 
2637 static void expand_2sh_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
2638                            uint32_t oprsz, uint32_t tysz, TCGType type,
2639                            TCGv_i32 shift,
2640                            void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_i32))
2641 {
2642     TCGv_vec t0 = tcg_temp_new_vec(type);
2643     uint32_t i;
2644 
2645     for (i = 0; i < oprsz; i += tysz) {
2646         tcg_gen_ld_vec(t0, cpu_env, aofs + i);
2647         fni(vece, t0, t0, shift);
2648         tcg_gen_st_vec(t0, cpu_env, dofs + i);
2649     }
2650     tcg_temp_free_vec(t0);
2651 }
2652 
2653 static void
2654 do_gvec_shifts(unsigned vece, uint32_t dofs, uint32_t aofs, TCGv_i32 shift,
2655                uint32_t oprsz, uint32_t maxsz, const GVecGen2sh *g)
2656 {
2657     TCGType type;
2658     uint32_t some;
2659 
2660     check_size_align(oprsz, maxsz, dofs | aofs);
2661     check_overlap_2(dofs, aofs, maxsz);
2662 
2663     /* If the backend has a scalar expansion, great.  */
2664     type = choose_vector_type(g->s_list, vece, oprsz, vece == MO_64);
2665     if (type) {
2666         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
2667         switch (type) {
2668         case TCG_TYPE_V256:
2669             some = QEMU_ALIGN_DOWN(oprsz, 32);
2670             expand_2sh_vec(vece, dofs, aofs, some, 32,
2671                            TCG_TYPE_V256, shift, g->fniv_s);
2672             if (some == oprsz) {
2673                 break;
2674             }
2675             dofs += some;
2676             aofs += some;
2677             oprsz -= some;
2678             maxsz -= some;
2679             /* fallthru */
2680         case TCG_TYPE_V128:
2681             expand_2sh_vec(vece, dofs, aofs, oprsz, 16,
2682                            TCG_TYPE_V128, shift, g->fniv_s);
2683             break;
2684         case TCG_TYPE_V64:
2685             expand_2sh_vec(vece, dofs, aofs, oprsz, 8,
2686                            TCG_TYPE_V64, shift, g->fniv_s);
2687             break;
2688         default:
2689             g_assert_not_reached();
2690         }
2691         tcg_swap_vecop_list(hold_list);
2692         goto clear_tail;
2693     }
2694 
2695     /* If the backend supports variable vector shifts, also cool.  */
2696     type = choose_vector_type(g->v_list, vece, oprsz, vece == MO_64);
2697     if (type) {
2698         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
2699         TCGv_vec v_shift = tcg_temp_new_vec(type);
2700 
2701         if (vece == MO_64) {
2702             TCGv_i64 sh64 = tcg_temp_new_i64();
2703             tcg_gen_extu_i32_i64(sh64, shift);
2704             tcg_gen_dup_i64_vec(MO_64, v_shift, sh64);
2705             tcg_temp_free_i64(sh64);
2706         } else {
2707             tcg_gen_dup_i32_vec(vece, v_shift, shift);
2708         }
2709 
2710         switch (type) {
2711         case TCG_TYPE_V256:
2712             some = QEMU_ALIGN_DOWN(oprsz, 32);
2713             expand_2s_vec(vece, dofs, aofs, some, 32, TCG_TYPE_V256,
2714                           v_shift, false, g->fniv_v);
2715             if (some == oprsz) {
2716                 break;
2717             }
2718             dofs += some;
2719             aofs += some;
2720             oprsz -= some;
2721             maxsz -= some;
2722             /* fallthru */
2723         case TCG_TYPE_V128:
2724             expand_2s_vec(vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
2725                           v_shift, false, g->fniv_v);
2726             break;
2727         case TCG_TYPE_V64:
2728             expand_2s_vec(vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
2729                           v_shift, false, g->fniv_v);
2730             break;
2731         default:
2732             g_assert_not_reached();
2733         }
2734         tcg_temp_free_vec(v_shift);
2735         tcg_swap_vecop_list(hold_list);
2736         goto clear_tail;
2737     }
2738 
2739     /* Otherwise fall back to integral... */
2740     if (vece == MO_32 && check_size_impl(oprsz, 4)) {
2741         expand_2s_i32(dofs, aofs, oprsz, shift, false, g->fni4);
2742     } else if (vece == MO_64 && check_size_impl(oprsz, 8)) {
2743         TCGv_i64 sh64 = tcg_temp_new_i64();
2744         tcg_gen_extu_i32_i64(sh64, shift);
2745         expand_2s_i64(dofs, aofs, oprsz, sh64, false, g->fni8);
2746         tcg_temp_free_i64(sh64);
2747     } else {
2748         TCGv_ptr a0 = tcg_temp_new_ptr();
2749         TCGv_ptr a1 = tcg_temp_new_ptr();
2750         TCGv_i32 desc = tcg_temp_new_i32();
2751 
2752         tcg_gen_shli_i32(desc, shift, SIMD_DATA_SHIFT);
2753         tcg_gen_ori_i32(desc, desc, simd_desc(oprsz, maxsz, 0));
2754         tcg_gen_addi_ptr(a0, cpu_env, dofs);
2755         tcg_gen_addi_ptr(a1, cpu_env, aofs);
2756 
2757         g->fno[vece](a0, a1, desc);
2758 
2759         tcg_temp_free_ptr(a0);
2760         tcg_temp_free_ptr(a1);
2761         tcg_temp_free_i32(desc);
2762         return;
2763     }
2764 
2765  clear_tail:
2766     if (oprsz < maxsz) {
2767         expand_clr(dofs + oprsz, maxsz - oprsz);
2768     }
2769 }
2770 
2771 void tcg_gen_gvec_shls(unsigned vece, uint32_t dofs, uint32_t aofs,
2772                        TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
2773 {
2774     static const GVecGen2sh g = {
2775         .fni4 = tcg_gen_shl_i32,
2776         .fni8 = tcg_gen_shl_i64,
2777         .fniv_s = tcg_gen_shls_vec,
2778         .fniv_v = tcg_gen_shlv_vec,
2779         .fno = {
2780             gen_helper_gvec_shl8i,
2781             gen_helper_gvec_shl16i,
2782             gen_helper_gvec_shl32i,
2783             gen_helper_gvec_shl64i,
2784         },
2785         .s_list = { INDEX_op_shls_vec, 0 },
2786         .v_list = { INDEX_op_shlv_vec, 0 },
2787     };
2788 
2789     tcg_debug_assert(vece <= MO_64);
2790     do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
2791 }
2792 
2793 void tcg_gen_gvec_shrs(unsigned vece, uint32_t dofs, uint32_t aofs,
2794                        TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
2795 {
2796     static const GVecGen2sh g = {
2797         .fni4 = tcg_gen_shr_i32,
2798         .fni8 = tcg_gen_shr_i64,
2799         .fniv_s = tcg_gen_shrs_vec,
2800         .fniv_v = tcg_gen_shrv_vec,
2801         .fno = {
2802             gen_helper_gvec_shr8i,
2803             gen_helper_gvec_shr16i,
2804             gen_helper_gvec_shr32i,
2805             gen_helper_gvec_shr64i,
2806         },
2807         .s_list = { INDEX_op_shrs_vec, 0 },
2808         .v_list = { INDEX_op_shrv_vec, 0 },
2809     };
2810 
2811     tcg_debug_assert(vece <= MO_64);
2812     do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
2813 }
2814 
2815 void tcg_gen_gvec_sars(unsigned vece, uint32_t dofs, uint32_t aofs,
2816                        TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
2817 {
2818     static const GVecGen2sh g = {
2819         .fni4 = tcg_gen_sar_i32,
2820         .fni8 = tcg_gen_sar_i64,
2821         .fniv_s = tcg_gen_sars_vec,
2822         .fniv_v = tcg_gen_sarv_vec,
2823         .fno = {
2824             gen_helper_gvec_sar8i,
2825             gen_helper_gvec_sar16i,
2826             gen_helper_gvec_sar32i,
2827             gen_helper_gvec_sar64i,
2828         },
2829         .s_list = { INDEX_op_sars_vec, 0 },
2830         .v_list = { INDEX_op_sarv_vec, 0 },
2831     };
2832 
2833     tcg_debug_assert(vece <= MO_64);
2834     do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
2835 }
2836 
2837 /*
2838  * Expand D = A << (B % element bits)
2839  *
2840  * Unlike scalar shifts, where it is easy for the target front end
2841  * to include the modulo as part of the expansion.  If the target
2842  * naturally includes the modulo as part of the operation, great!
2843  * If the target has some other behaviour from out-of-range shifts,
2844  * then it could not use this function anyway, and would need to
2845  * do it's own expansion with custom functions.
2846  */
2847 static void tcg_gen_shlv_mod_vec(unsigned vece, TCGv_vec d,
2848                                  TCGv_vec a, TCGv_vec b)
2849 {
2850     TCGv_vec t = tcg_temp_new_vec_matching(d);
2851 
2852     tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
2853     tcg_gen_and_vec(vece, t, t, b);
2854     tcg_gen_shlv_vec(vece, d, a, t);
2855     tcg_temp_free_vec(t);
2856 }
2857 
2858 static void tcg_gen_shl_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2859 {
2860     TCGv_i32 t = tcg_temp_new_i32();
2861 
2862     tcg_gen_andi_i32(t, b, 31);
2863     tcg_gen_shl_i32(d, a, t);
2864     tcg_temp_free_i32(t);
2865 }
2866 
2867 static void tcg_gen_shl_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2868 {
2869     TCGv_i64 t = tcg_temp_new_i64();
2870 
2871     tcg_gen_andi_i64(t, b, 63);
2872     tcg_gen_shl_i64(d, a, t);
2873     tcg_temp_free_i64(t);
2874 }
2875 
2876 void tcg_gen_gvec_shlv(unsigned vece, uint32_t dofs, uint32_t aofs,
2877                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2878 {
2879     static const TCGOpcode vecop_list[] = { INDEX_op_shlv_vec, 0 };
2880     static const GVecGen3 g[4] = {
2881         { .fniv = tcg_gen_shlv_mod_vec,
2882           .fno = gen_helper_gvec_shl8v,
2883           .opt_opc = vecop_list,
2884           .vece = MO_8 },
2885         { .fniv = tcg_gen_shlv_mod_vec,
2886           .fno = gen_helper_gvec_shl16v,
2887           .opt_opc = vecop_list,
2888           .vece = MO_16 },
2889         { .fni4 = tcg_gen_shl_mod_i32,
2890           .fniv = tcg_gen_shlv_mod_vec,
2891           .fno = gen_helper_gvec_shl32v,
2892           .opt_opc = vecop_list,
2893           .vece = MO_32 },
2894         { .fni8 = tcg_gen_shl_mod_i64,
2895           .fniv = tcg_gen_shlv_mod_vec,
2896           .fno = gen_helper_gvec_shl64v,
2897           .opt_opc = vecop_list,
2898           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2899           .vece = MO_64 },
2900     };
2901 
2902     tcg_debug_assert(vece <= MO_64);
2903     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2904 }
2905 
2906 /*
2907  * Similarly for logical right shifts.
2908  */
2909 
2910 static void tcg_gen_shrv_mod_vec(unsigned vece, TCGv_vec d,
2911                                  TCGv_vec a, TCGv_vec b)
2912 {
2913     TCGv_vec t = tcg_temp_new_vec_matching(d);
2914 
2915     tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
2916     tcg_gen_and_vec(vece, t, t, b);
2917     tcg_gen_shrv_vec(vece, d, a, t);
2918     tcg_temp_free_vec(t);
2919 }
2920 
2921 static void tcg_gen_shr_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2922 {
2923     TCGv_i32 t = tcg_temp_new_i32();
2924 
2925     tcg_gen_andi_i32(t, b, 31);
2926     tcg_gen_shr_i32(d, a, t);
2927     tcg_temp_free_i32(t);
2928 }
2929 
2930 static void tcg_gen_shr_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2931 {
2932     TCGv_i64 t = tcg_temp_new_i64();
2933 
2934     tcg_gen_andi_i64(t, b, 63);
2935     tcg_gen_shr_i64(d, a, t);
2936     tcg_temp_free_i64(t);
2937 }
2938 
2939 void tcg_gen_gvec_shrv(unsigned vece, uint32_t dofs, uint32_t aofs,
2940                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2941 {
2942     static const TCGOpcode vecop_list[] = { INDEX_op_shrv_vec, 0 };
2943     static const GVecGen3 g[4] = {
2944         { .fniv = tcg_gen_shrv_mod_vec,
2945           .fno = gen_helper_gvec_shr8v,
2946           .opt_opc = vecop_list,
2947           .vece = MO_8 },
2948         { .fniv = tcg_gen_shrv_mod_vec,
2949           .fno = gen_helper_gvec_shr16v,
2950           .opt_opc = vecop_list,
2951           .vece = MO_16 },
2952         { .fni4 = tcg_gen_shr_mod_i32,
2953           .fniv = tcg_gen_shrv_mod_vec,
2954           .fno = gen_helper_gvec_shr32v,
2955           .opt_opc = vecop_list,
2956           .vece = MO_32 },
2957         { .fni8 = tcg_gen_shr_mod_i64,
2958           .fniv = tcg_gen_shrv_mod_vec,
2959           .fno = gen_helper_gvec_shr64v,
2960           .opt_opc = vecop_list,
2961           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2962           .vece = MO_64 },
2963     };
2964 
2965     tcg_debug_assert(vece <= MO_64);
2966     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2967 }
2968 
2969 /*
2970  * Similarly for arithmetic right shifts.
2971  */
2972 
2973 static void tcg_gen_sarv_mod_vec(unsigned vece, TCGv_vec d,
2974                                  TCGv_vec a, TCGv_vec b)
2975 {
2976     TCGv_vec t = tcg_temp_new_vec_matching(d);
2977 
2978     tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
2979     tcg_gen_and_vec(vece, t, t, b);
2980     tcg_gen_sarv_vec(vece, d, a, t);
2981     tcg_temp_free_vec(t);
2982 }
2983 
2984 static void tcg_gen_sar_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2985 {
2986     TCGv_i32 t = tcg_temp_new_i32();
2987 
2988     tcg_gen_andi_i32(t, b, 31);
2989     tcg_gen_sar_i32(d, a, t);
2990     tcg_temp_free_i32(t);
2991 }
2992 
2993 static void tcg_gen_sar_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2994 {
2995     TCGv_i64 t = tcg_temp_new_i64();
2996 
2997     tcg_gen_andi_i64(t, b, 63);
2998     tcg_gen_sar_i64(d, a, t);
2999     tcg_temp_free_i64(t);
3000 }
3001 
3002 void tcg_gen_gvec_sarv(unsigned vece, uint32_t dofs, uint32_t aofs,
3003                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
3004 {
3005     static const TCGOpcode vecop_list[] = { INDEX_op_sarv_vec, 0 };
3006     static const GVecGen3 g[4] = {
3007         { .fniv = tcg_gen_sarv_mod_vec,
3008           .fno = gen_helper_gvec_sar8v,
3009           .opt_opc = vecop_list,
3010           .vece = MO_8 },
3011         { .fniv = tcg_gen_sarv_mod_vec,
3012           .fno = gen_helper_gvec_sar16v,
3013           .opt_opc = vecop_list,
3014           .vece = MO_16 },
3015         { .fni4 = tcg_gen_sar_mod_i32,
3016           .fniv = tcg_gen_sarv_mod_vec,
3017           .fno = gen_helper_gvec_sar32v,
3018           .opt_opc = vecop_list,
3019           .vece = MO_32 },
3020         { .fni8 = tcg_gen_sar_mod_i64,
3021           .fniv = tcg_gen_sarv_mod_vec,
3022           .fno = gen_helper_gvec_sar64v,
3023           .opt_opc = vecop_list,
3024           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
3025           .vece = MO_64 },
3026     };
3027 
3028     tcg_debug_assert(vece <= MO_64);
3029     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
3030 }
3031 
3032 /* Expand OPSZ bytes worth of three-operand operations using i32 elements.  */
3033 static void expand_cmp_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
3034                            uint32_t oprsz, TCGCond cond)
3035 {
3036     TCGv_i32 t0 = tcg_temp_new_i32();
3037     TCGv_i32 t1 = tcg_temp_new_i32();
3038     uint32_t i;
3039 
3040     for (i = 0; i < oprsz; i += 4) {
3041         tcg_gen_ld_i32(t0, cpu_env, aofs + i);
3042         tcg_gen_ld_i32(t1, cpu_env, bofs + i);
3043         tcg_gen_setcond_i32(cond, t0, t0, t1);
3044         tcg_gen_neg_i32(t0, t0);
3045         tcg_gen_st_i32(t0, cpu_env, dofs + i);
3046     }
3047     tcg_temp_free_i32(t1);
3048     tcg_temp_free_i32(t0);
3049 }
3050 
3051 static void expand_cmp_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
3052                            uint32_t oprsz, TCGCond cond)
3053 {
3054     TCGv_i64 t0 = tcg_temp_new_i64();
3055     TCGv_i64 t1 = tcg_temp_new_i64();
3056     uint32_t i;
3057 
3058     for (i = 0; i < oprsz; i += 8) {
3059         tcg_gen_ld_i64(t0, cpu_env, aofs + i);
3060         tcg_gen_ld_i64(t1, cpu_env, bofs + i);
3061         tcg_gen_setcond_i64(cond, t0, t0, t1);
3062         tcg_gen_neg_i64(t0, t0);
3063         tcg_gen_st_i64(t0, cpu_env, dofs + i);
3064     }
3065     tcg_temp_free_i64(t1);
3066     tcg_temp_free_i64(t0);
3067 }
3068 
3069 static void expand_cmp_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
3070                            uint32_t bofs, uint32_t oprsz, uint32_t tysz,
3071                            TCGType type, TCGCond cond)
3072 {
3073     TCGv_vec t0 = tcg_temp_new_vec(type);
3074     TCGv_vec t1 = tcg_temp_new_vec(type);
3075     uint32_t i;
3076 
3077     for (i = 0; i < oprsz; i += tysz) {
3078         tcg_gen_ld_vec(t0, cpu_env, aofs + i);
3079         tcg_gen_ld_vec(t1, cpu_env, bofs + i);
3080         tcg_gen_cmp_vec(cond, vece, t0, t0, t1);
3081         tcg_gen_st_vec(t0, cpu_env, dofs + i);
3082     }
3083     tcg_temp_free_vec(t1);
3084     tcg_temp_free_vec(t0);
3085 }
3086 
3087 void tcg_gen_gvec_cmp(TCGCond cond, unsigned vece, uint32_t dofs,
3088                       uint32_t aofs, uint32_t bofs,
3089                       uint32_t oprsz, uint32_t maxsz)
3090 {
3091     static const TCGOpcode cmp_list[] = { INDEX_op_cmp_vec, 0 };
3092     static gen_helper_gvec_3 * const eq_fn[4] = {
3093         gen_helper_gvec_eq8, gen_helper_gvec_eq16,
3094         gen_helper_gvec_eq32, gen_helper_gvec_eq64
3095     };
3096     static gen_helper_gvec_3 * const ne_fn[4] = {
3097         gen_helper_gvec_ne8, gen_helper_gvec_ne16,
3098         gen_helper_gvec_ne32, gen_helper_gvec_ne64
3099     };
3100     static gen_helper_gvec_3 * const lt_fn[4] = {
3101         gen_helper_gvec_lt8, gen_helper_gvec_lt16,
3102         gen_helper_gvec_lt32, gen_helper_gvec_lt64
3103     };
3104     static gen_helper_gvec_3 * const le_fn[4] = {
3105         gen_helper_gvec_le8, gen_helper_gvec_le16,
3106         gen_helper_gvec_le32, gen_helper_gvec_le64
3107     };
3108     static gen_helper_gvec_3 * const ltu_fn[4] = {
3109         gen_helper_gvec_ltu8, gen_helper_gvec_ltu16,
3110         gen_helper_gvec_ltu32, gen_helper_gvec_ltu64
3111     };
3112     static gen_helper_gvec_3 * const leu_fn[4] = {
3113         gen_helper_gvec_leu8, gen_helper_gvec_leu16,
3114         gen_helper_gvec_leu32, gen_helper_gvec_leu64
3115     };
3116     static gen_helper_gvec_3 * const * const fns[16] = {
3117         [TCG_COND_EQ] = eq_fn,
3118         [TCG_COND_NE] = ne_fn,
3119         [TCG_COND_LT] = lt_fn,
3120         [TCG_COND_LE] = le_fn,
3121         [TCG_COND_LTU] = ltu_fn,
3122         [TCG_COND_LEU] = leu_fn,
3123     };
3124 
3125     const TCGOpcode *hold_list;
3126     TCGType type;
3127     uint32_t some;
3128 
3129     check_size_align(oprsz, maxsz, dofs | aofs | bofs);
3130     check_overlap_3(dofs, aofs, bofs, maxsz);
3131 
3132     if (cond == TCG_COND_NEVER || cond == TCG_COND_ALWAYS) {
3133         do_dup(MO_8, dofs, oprsz, maxsz,
3134                NULL, NULL, -(cond == TCG_COND_ALWAYS));
3135         return;
3136     }
3137 
3138     /*
3139      * Implement inline with a vector type, if possible.
3140      * Prefer integer when 64-bit host and 64-bit comparison.
3141      */
3142     hold_list = tcg_swap_vecop_list(cmp_list);
3143     type = choose_vector_type(cmp_list, vece, oprsz,
3144                               TCG_TARGET_REG_BITS == 64 && vece == MO_64);
3145     switch (type) {
3146     case TCG_TYPE_V256:
3147         /* Recall that ARM SVE allows vector sizes that are not a
3148          * power of 2, but always a multiple of 16.  The intent is
3149          * that e.g. size == 80 would be expanded with 2x32 + 1x16.
3150          */
3151         some = QEMU_ALIGN_DOWN(oprsz, 32);
3152         expand_cmp_vec(vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256, cond);
3153         if (some == oprsz) {
3154             break;
3155         }
3156         dofs += some;
3157         aofs += some;
3158         bofs += some;
3159         oprsz -= some;
3160         maxsz -= some;
3161         /* fallthru */
3162     case TCG_TYPE_V128:
3163         expand_cmp_vec(vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128, cond);
3164         break;
3165     case TCG_TYPE_V64:
3166         expand_cmp_vec(vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64, cond);
3167         break;
3168 
3169     case 0:
3170         if (vece == MO_64 && check_size_impl(oprsz, 8)) {
3171             expand_cmp_i64(dofs, aofs, bofs, oprsz, cond);
3172         } else if (vece == MO_32 && check_size_impl(oprsz, 4)) {
3173             expand_cmp_i32(dofs, aofs, bofs, oprsz, cond);
3174         } else {
3175             gen_helper_gvec_3 * const *fn = fns[cond];
3176 
3177             if (fn == NULL) {
3178                 uint32_t tmp;
3179                 tmp = aofs, aofs = bofs, bofs = tmp;
3180                 cond = tcg_swap_cond(cond);
3181                 fn = fns[cond];
3182                 assert(fn != NULL);
3183             }
3184             tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz, maxsz, 0, fn[vece]);
3185             oprsz = maxsz;
3186         }
3187         break;
3188 
3189     default:
3190         g_assert_not_reached();
3191     }
3192     tcg_swap_vecop_list(hold_list);
3193 
3194     if (oprsz < maxsz) {
3195         expand_clr(dofs + oprsz, maxsz - oprsz);
3196     }
3197 }
3198 
3199 static void tcg_gen_bitsel_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 c)
3200 {
3201     TCGv_i64 t = tcg_temp_new_i64();
3202 
3203     tcg_gen_and_i64(t, b, a);
3204     tcg_gen_andc_i64(d, c, a);
3205     tcg_gen_or_i64(d, d, t);
3206     tcg_temp_free_i64(t);
3207 }
3208 
3209 void tcg_gen_gvec_bitsel(unsigned vece, uint32_t dofs, uint32_t aofs,
3210                          uint32_t bofs, uint32_t cofs,
3211                          uint32_t oprsz, uint32_t maxsz)
3212 {
3213     static const GVecGen4 g = {
3214         .fni8 = tcg_gen_bitsel_i64,
3215         .fniv = tcg_gen_bitsel_vec,
3216         .fno = gen_helper_gvec_bitsel,
3217     };
3218 
3219     tcg_gen_gvec_4(dofs, aofs, bofs, cofs, oprsz, maxsz, &g);
3220 }
3221