xref: /openbmc/qemu/target/arm/tcg/translate-sve.c (revision 5d326db2)
1 /*
2  * AArch64 SVE translation
3  *
4  * Copyright (c) 2018 Linaro, Ltd
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 #include "translate.h"
22 #include "translate-a64.h"
23 #include "fpu/softfloat.h"
24 
25 
26 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
27                          TCGv_i64, uint32_t, uint32_t);
28 
29 typedef void gen_helper_gvec_flags_3(TCGv_i32, TCGv_ptr, TCGv_ptr,
30                                      TCGv_ptr, TCGv_i32);
31 typedef void gen_helper_gvec_flags_4(TCGv_i32, TCGv_ptr, TCGv_ptr,
32                                      TCGv_ptr, TCGv_ptr, TCGv_i32);
33 
34 typedef void gen_helper_gvec_mem(TCGv_env, TCGv_ptr, TCGv_i64, TCGv_i32);
35 typedef void gen_helper_gvec_mem_scatter(TCGv_env, TCGv_ptr, TCGv_ptr,
36                                          TCGv_ptr, TCGv_i64, TCGv_i32);
37 
38 /*
39  * Helpers for extracting complex instruction fields.
40  */
41 
42 /* See e.g. ASR (immediate, predicated).
43  * Returns -1 for unallocated encoding; diagnose later.
44  */
45 static int tszimm_esz(DisasContext *s, int x)
46 {
47     x >>= 3;  /* discard imm3 */
48     return 31 - clz32(x);
49 }
50 
51 static int tszimm_shr(DisasContext *s, int x)
52 {
53     return (16 << tszimm_esz(s, x)) - x;
54 }
55 
56 /* See e.g. LSL (immediate, predicated).  */
57 static int tszimm_shl(DisasContext *s, int x)
58 {
59     return x - (8 << tszimm_esz(s, x));
60 }
61 
62 /* The SH bit is in bit 8.  Extract the low 8 and shift.  */
63 static inline int expand_imm_sh8s(DisasContext *s, int x)
64 {
65     return (int8_t)x << (x & 0x100 ? 8 : 0);
66 }
67 
68 static inline int expand_imm_sh8u(DisasContext *s, int x)
69 {
70     return (uint8_t)x << (x & 0x100 ? 8 : 0);
71 }
72 
73 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
74  * with unsigned data.  C.f. SVE Memory Contiguous Load Group.
75  */
76 static inline int msz_dtype(DisasContext *s, int msz)
77 {
78     static const uint8_t dtype[4] = { 0, 5, 10, 15 };
79     return dtype[msz];
80 }
81 
82 /*
83  * Include the generated decoder.
84  */
85 
86 #include "decode-sve.c.inc"
87 
88 /*
89  * Implement all of the translator functions referenced by the decoder.
90  */
91 
92 /* Invoke an out-of-line helper on 2 Zregs. */
93 static bool gen_gvec_ool_zz(DisasContext *s, gen_helper_gvec_2 *fn,
94                             int rd, int rn, int data)
95 {
96     if (fn == NULL) {
97         return false;
98     }
99     if (sve_access_check(s)) {
100         unsigned vsz = vec_full_reg_size(s);
101         tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd),
102                            vec_full_reg_offset(s, rn),
103                            vsz, vsz, data, fn);
104     }
105     return true;
106 }
107 
108 static bool gen_gvec_fpst_zz(DisasContext *s, gen_helper_gvec_2_ptr *fn,
109                              int rd, int rn, int data,
110                              ARMFPStatusFlavour flavour)
111 {
112     if (fn == NULL) {
113         return false;
114     }
115     if (sve_access_check(s)) {
116         unsigned vsz = vec_full_reg_size(s);
117         TCGv_ptr status = fpstatus_ptr(flavour);
118 
119         tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, rd),
120                            vec_full_reg_offset(s, rn),
121                            status, vsz, vsz, data, fn);
122     }
123     return true;
124 }
125 
126 static bool gen_gvec_fpst_arg_zz(DisasContext *s, gen_helper_gvec_2_ptr *fn,
127                                  arg_rr_esz *a, int data)
128 {
129     return gen_gvec_fpst_zz(s, fn, a->rd, a->rn, data,
130                             a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
131 }
132 
133 /* Invoke an out-of-line helper on 3 Zregs. */
134 static bool gen_gvec_ool_zzz(DisasContext *s, gen_helper_gvec_3 *fn,
135                              int rd, int rn, int rm, int data)
136 {
137     if (fn == NULL) {
138         return false;
139     }
140     if (sve_access_check(s)) {
141         unsigned vsz = vec_full_reg_size(s);
142         tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
143                            vec_full_reg_offset(s, rn),
144                            vec_full_reg_offset(s, rm),
145                            vsz, vsz, data, fn);
146     }
147     return true;
148 }
149 
150 static bool gen_gvec_ool_arg_zzz(DisasContext *s, gen_helper_gvec_3 *fn,
151                                  arg_rrr_esz *a, int data)
152 {
153     return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, data);
154 }
155 
156 /* Invoke an out-of-line helper on 3 Zregs, plus float_status. */
157 static bool gen_gvec_fpst_zzz(DisasContext *s, gen_helper_gvec_3_ptr *fn,
158                               int rd, int rn, int rm,
159                               int data, ARMFPStatusFlavour flavour)
160 {
161     if (fn == NULL) {
162         return false;
163     }
164     if (sve_access_check(s)) {
165         unsigned vsz = vec_full_reg_size(s);
166         TCGv_ptr status = fpstatus_ptr(flavour);
167 
168         tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
169                            vec_full_reg_offset(s, rn),
170                            vec_full_reg_offset(s, rm),
171                            status, vsz, vsz, data, fn);
172     }
173     return true;
174 }
175 
176 static bool gen_gvec_fpst_arg_zzz(DisasContext *s, gen_helper_gvec_3_ptr *fn,
177                                   arg_rrr_esz *a, int data)
178 {
179     return gen_gvec_fpst_zzz(s, fn, a->rd, a->rn, a->rm, data,
180                              a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
181 }
182 
183 /* Invoke an out-of-line helper on 4 Zregs. */
184 static bool gen_gvec_ool_zzzz(DisasContext *s, gen_helper_gvec_4 *fn,
185                               int rd, int rn, int rm, int ra, int data)
186 {
187     if (fn == NULL) {
188         return false;
189     }
190     if (sve_access_check(s)) {
191         unsigned vsz = vec_full_reg_size(s);
192         tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
193                            vec_full_reg_offset(s, rn),
194                            vec_full_reg_offset(s, rm),
195                            vec_full_reg_offset(s, ra),
196                            vsz, vsz, data, fn);
197     }
198     return true;
199 }
200 
201 static bool gen_gvec_ool_arg_zzzz(DisasContext *s, gen_helper_gvec_4 *fn,
202                                   arg_rrrr_esz *a, int data)
203 {
204     return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data);
205 }
206 
207 static bool gen_gvec_ool_arg_zzxz(DisasContext *s, gen_helper_gvec_4 *fn,
208                                   arg_rrxr_esz *a)
209 {
210     return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index);
211 }
212 
213 /* Invoke an out-of-line helper on 4 Zregs, plus a pointer. */
214 static bool gen_gvec_ptr_zzzz(DisasContext *s, gen_helper_gvec_4_ptr *fn,
215                               int rd, int rn, int rm, int ra,
216                               int data, TCGv_ptr ptr)
217 {
218     if (fn == NULL) {
219         return false;
220     }
221     if (sve_access_check(s)) {
222         unsigned vsz = vec_full_reg_size(s);
223         tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
224                            vec_full_reg_offset(s, rn),
225                            vec_full_reg_offset(s, rm),
226                            vec_full_reg_offset(s, ra),
227                            ptr, vsz, vsz, data, fn);
228     }
229     return true;
230 }
231 
232 static bool gen_gvec_fpst_zzzz(DisasContext *s, gen_helper_gvec_4_ptr *fn,
233                                int rd, int rn, int rm, int ra,
234                                int data, ARMFPStatusFlavour flavour)
235 {
236     TCGv_ptr status = fpstatus_ptr(flavour);
237     bool ret = gen_gvec_ptr_zzzz(s, fn, rd, rn, rm, ra, data, status);
238     return ret;
239 }
240 
241 /* Invoke an out-of-line helper on 4 Zregs, 1 Preg, plus fpst. */
242 static bool gen_gvec_fpst_zzzzp(DisasContext *s, gen_helper_gvec_5_ptr *fn,
243                                 int rd, int rn, int rm, int ra, int pg,
244                                 int data, ARMFPStatusFlavour flavour)
245 {
246     if (fn == NULL) {
247         return false;
248     }
249     if (sve_access_check(s)) {
250         unsigned vsz = vec_full_reg_size(s);
251         TCGv_ptr status = fpstatus_ptr(flavour);
252 
253         tcg_gen_gvec_5_ptr(vec_full_reg_offset(s, rd),
254                            vec_full_reg_offset(s, rn),
255                            vec_full_reg_offset(s, rm),
256                            vec_full_reg_offset(s, ra),
257                            pred_full_reg_offset(s, pg),
258                            status, vsz, vsz, data, fn);
259     }
260     return true;
261 }
262 
263 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */
264 static bool gen_gvec_ool_zzp(DisasContext *s, gen_helper_gvec_3 *fn,
265                              int rd, int rn, int pg, int data)
266 {
267     if (fn == NULL) {
268         return false;
269     }
270     if (sve_access_check(s)) {
271         unsigned vsz = vec_full_reg_size(s);
272         tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
273                            vec_full_reg_offset(s, rn),
274                            pred_full_reg_offset(s, pg),
275                            vsz, vsz, data, fn);
276     }
277     return true;
278 }
279 
280 static bool gen_gvec_ool_arg_zpz(DisasContext *s, gen_helper_gvec_3 *fn,
281                                  arg_rpr_esz *a, int data)
282 {
283     return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, data);
284 }
285 
286 static bool gen_gvec_ool_arg_zpzi(DisasContext *s, gen_helper_gvec_3 *fn,
287                                   arg_rpri_esz *a)
288 {
289     return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, a->imm);
290 }
291 
292 static bool gen_gvec_fpst_zzp(DisasContext *s, gen_helper_gvec_3_ptr *fn,
293                               int rd, int rn, int pg, int data,
294                               ARMFPStatusFlavour flavour)
295 {
296     if (fn == NULL) {
297         return false;
298     }
299     if (sve_access_check(s)) {
300         unsigned vsz = vec_full_reg_size(s);
301         TCGv_ptr status = fpstatus_ptr(flavour);
302 
303         tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
304                            vec_full_reg_offset(s, rn),
305                            pred_full_reg_offset(s, pg),
306                            status, vsz, vsz, data, fn);
307     }
308     return true;
309 }
310 
311 static bool gen_gvec_fpst_arg_zpz(DisasContext *s, gen_helper_gvec_3_ptr *fn,
312                                   arg_rpr_esz *a, int data,
313                                   ARMFPStatusFlavour flavour)
314 {
315     return gen_gvec_fpst_zzp(s, fn, a->rd, a->rn, a->pg, data, flavour);
316 }
317 
318 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
319 static bool gen_gvec_ool_zzzp(DisasContext *s, gen_helper_gvec_4 *fn,
320                               int rd, int rn, int rm, int pg, int data)
321 {
322     if (fn == NULL) {
323         return false;
324     }
325     if (sve_access_check(s)) {
326         unsigned vsz = vec_full_reg_size(s);
327         tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
328                            vec_full_reg_offset(s, rn),
329                            vec_full_reg_offset(s, rm),
330                            pred_full_reg_offset(s, pg),
331                            vsz, vsz, data, fn);
332     }
333     return true;
334 }
335 
336 static bool gen_gvec_ool_arg_zpzz(DisasContext *s, gen_helper_gvec_4 *fn,
337                                   arg_rprr_esz *a, int data)
338 {
339     return gen_gvec_ool_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, data);
340 }
341 
342 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
343 static bool gen_gvec_fpst_zzzp(DisasContext *s, gen_helper_gvec_4_ptr *fn,
344                                int rd, int rn, int rm, int pg, int data,
345                                ARMFPStatusFlavour flavour)
346 {
347     if (fn == NULL) {
348         return false;
349     }
350     if (sve_access_check(s)) {
351         unsigned vsz = vec_full_reg_size(s);
352         TCGv_ptr status = fpstatus_ptr(flavour);
353 
354         tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
355                            vec_full_reg_offset(s, rn),
356                            vec_full_reg_offset(s, rm),
357                            pred_full_reg_offset(s, pg),
358                            status, vsz, vsz, data, fn);
359     }
360     return true;
361 }
362 
363 static bool gen_gvec_fpst_arg_zpzz(DisasContext *s, gen_helper_gvec_4_ptr *fn,
364                                    arg_rprr_esz *a)
365 {
366     return gen_gvec_fpst_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, 0,
367                               a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
368 }
369 
370 /* Invoke a vector expander on two Zregs and an immediate.  */
371 static bool gen_gvec_fn_zzi(DisasContext *s, GVecGen2iFn *gvec_fn,
372                             int esz, int rd, int rn, uint64_t imm)
373 {
374     if (gvec_fn == NULL) {
375         return false;
376     }
377     if (sve_access_check(s)) {
378         unsigned vsz = vec_full_reg_size(s);
379         gvec_fn(esz, vec_full_reg_offset(s, rd),
380                 vec_full_reg_offset(s, rn), imm, vsz, vsz);
381     }
382     return true;
383 }
384 
385 static bool gen_gvec_fn_arg_zzi(DisasContext *s, GVecGen2iFn *gvec_fn,
386                                 arg_rri_esz *a)
387 {
388     if (a->esz < 0) {
389         /* Invalid tsz encoding -- see tszimm_esz. */
390         return false;
391     }
392     return gen_gvec_fn_zzi(s, gvec_fn, a->esz, a->rd, a->rn, a->imm);
393 }
394 
395 /* Invoke a vector expander on three Zregs.  */
396 static bool gen_gvec_fn_zzz(DisasContext *s, GVecGen3Fn *gvec_fn,
397                             int esz, int rd, int rn, int rm)
398 {
399     if (gvec_fn == NULL) {
400         return false;
401     }
402     if (sve_access_check(s)) {
403         unsigned vsz = vec_full_reg_size(s);
404         gvec_fn(esz, vec_full_reg_offset(s, rd),
405                 vec_full_reg_offset(s, rn),
406                 vec_full_reg_offset(s, rm), vsz, vsz);
407     }
408     return true;
409 }
410 
411 static bool gen_gvec_fn_arg_zzz(DisasContext *s, GVecGen3Fn *fn,
412                                 arg_rrr_esz *a)
413 {
414     return gen_gvec_fn_zzz(s, fn, a->esz, a->rd, a->rn, a->rm);
415 }
416 
417 /* Invoke a vector expander on four Zregs.  */
418 static bool gen_gvec_fn_arg_zzzz(DisasContext *s, GVecGen4Fn *gvec_fn,
419                                  arg_rrrr_esz *a)
420 {
421     if (gvec_fn == NULL) {
422         return false;
423     }
424     if (sve_access_check(s)) {
425         unsigned vsz = vec_full_reg_size(s);
426         gvec_fn(a->esz, vec_full_reg_offset(s, a->rd),
427                 vec_full_reg_offset(s, a->rn),
428                 vec_full_reg_offset(s, a->rm),
429                 vec_full_reg_offset(s, a->ra), vsz, vsz);
430     }
431     return true;
432 }
433 
434 /* Invoke a vector move on two Zregs.  */
435 static bool do_mov_z(DisasContext *s, int rd, int rn)
436 {
437     if (sve_access_check(s)) {
438         unsigned vsz = vec_full_reg_size(s);
439         tcg_gen_gvec_mov(MO_8, vec_full_reg_offset(s, rd),
440                          vec_full_reg_offset(s, rn), vsz, vsz);
441     }
442     return true;
443 }
444 
445 /* Initialize a Zreg with replications of a 64-bit immediate.  */
446 static void do_dupi_z(DisasContext *s, int rd, uint64_t word)
447 {
448     unsigned vsz = vec_full_reg_size(s);
449     tcg_gen_gvec_dup_imm(MO_64, vec_full_reg_offset(s, rd), vsz, vsz, word);
450 }
451 
452 /* Invoke a vector expander on three Pregs.  */
453 static bool gen_gvec_fn_ppp(DisasContext *s, GVecGen3Fn *gvec_fn,
454                             int rd, int rn, int rm)
455 {
456     if (sve_access_check(s)) {
457         unsigned psz = pred_gvec_reg_size(s);
458         gvec_fn(MO_64, pred_full_reg_offset(s, rd),
459                 pred_full_reg_offset(s, rn),
460                 pred_full_reg_offset(s, rm), psz, psz);
461     }
462     return true;
463 }
464 
465 /* Invoke a vector move on two Pregs.  */
466 static bool do_mov_p(DisasContext *s, int rd, int rn)
467 {
468     if (sve_access_check(s)) {
469         unsigned psz = pred_gvec_reg_size(s);
470         tcg_gen_gvec_mov(MO_8, pred_full_reg_offset(s, rd),
471                          pred_full_reg_offset(s, rn), psz, psz);
472     }
473     return true;
474 }
475 
476 /* Set the cpu flags as per a return from an SVE helper.  */
477 static void do_pred_flags(TCGv_i32 t)
478 {
479     tcg_gen_mov_i32(cpu_NF, t);
480     tcg_gen_andi_i32(cpu_ZF, t, 2);
481     tcg_gen_andi_i32(cpu_CF, t, 1);
482     tcg_gen_movi_i32(cpu_VF, 0);
483 }
484 
485 /* Subroutines computing the ARM PredTest psuedofunction.  */
486 static void do_predtest1(TCGv_i64 d, TCGv_i64 g)
487 {
488     TCGv_i32 t = tcg_temp_new_i32();
489 
490     gen_helper_sve_predtest1(t, d, g);
491     do_pred_flags(t);
492 }
493 
494 static void do_predtest(DisasContext *s, int dofs, int gofs, int words)
495 {
496     TCGv_ptr dptr = tcg_temp_new_ptr();
497     TCGv_ptr gptr = tcg_temp_new_ptr();
498     TCGv_i32 t = tcg_temp_new_i32();
499 
500     tcg_gen_addi_ptr(dptr, cpu_env, dofs);
501     tcg_gen_addi_ptr(gptr, cpu_env, gofs);
502 
503     gen_helper_sve_predtest(t, dptr, gptr, tcg_constant_i32(words));
504 
505     do_pred_flags(t);
506 }
507 
508 /* For each element size, the bits within a predicate word that are active.  */
509 const uint64_t pred_esz_masks[5] = {
510     0xffffffffffffffffull, 0x5555555555555555ull,
511     0x1111111111111111ull, 0x0101010101010101ull,
512     0x0001000100010001ull,
513 };
514 
515 static bool trans_INVALID(DisasContext *s, arg_INVALID *a)
516 {
517     unallocated_encoding(s);
518     return true;
519 }
520 
521 /*
522  *** SVE Logical - Unpredicated Group
523  */
524 
525 TRANS_FEAT(AND_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_and, a)
526 TRANS_FEAT(ORR_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_or, a)
527 TRANS_FEAT(EOR_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_xor, a)
528 TRANS_FEAT(BIC_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_andc, a)
529 
530 static void gen_xar8_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh)
531 {
532     TCGv_i64 t = tcg_temp_new_i64();
533     uint64_t mask = dup_const(MO_8, 0xff >> sh);
534 
535     tcg_gen_xor_i64(t, n, m);
536     tcg_gen_shri_i64(d, t, sh);
537     tcg_gen_shli_i64(t, t, 8 - sh);
538     tcg_gen_andi_i64(d, d, mask);
539     tcg_gen_andi_i64(t, t, ~mask);
540     tcg_gen_or_i64(d, d, t);
541 }
542 
543 static void gen_xar16_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh)
544 {
545     TCGv_i64 t = tcg_temp_new_i64();
546     uint64_t mask = dup_const(MO_16, 0xffff >> sh);
547 
548     tcg_gen_xor_i64(t, n, m);
549     tcg_gen_shri_i64(d, t, sh);
550     tcg_gen_shli_i64(t, t, 16 - sh);
551     tcg_gen_andi_i64(d, d, mask);
552     tcg_gen_andi_i64(t, t, ~mask);
553     tcg_gen_or_i64(d, d, t);
554 }
555 
556 static void gen_xar_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, int32_t sh)
557 {
558     tcg_gen_xor_i32(d, n, m);
559     tcg_gen_rotri_i32(d, d, sh);
560 }
561 
562 static void gen_xar_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh)
563 {
564     tcg_gen_xor_i64(d, n, m);
565     tcg_gen_rotri_i64(d, d, sh);
566 }
567 
568 static void gen_xar_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
569                         TCGv_vec m, int64_t sh)
570 {
571     tcg_gen_xor_vec(vece, d, n, m);
572     tcg_gen_rotri_vec(vece, d, d, sh);
573 }
574 
575 void gen_gvec_xar(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
576                   uint32_t rm_ofs, int64_t shift,
577                   uint32_t opr_sz, uint32_t max_sz)
578 {
579     static const TCGOpcode vecop[] = { INDEX_op_rotli_vec, 0 };
580     static const GVecGen3i ops[4] = {
581         { .fni8 = gen_xar8_i64,
582           .fniv = gen_xar_vec,
583           .fno = gen_helper_sve2_xar_b,
584           .opt_opc = vecop,
585           .vece = MO_8 },
586         { .fni8 = gen_xar16_i64,
587           .fniv = gen_xar_vec,
588           .fno = gen_helper_sve2_xar_h,
589           .opt_opc = vecop,
590           .vece = MO_16 },
591         { .fni4 = gen_xar_i32,
592           .fniv = gen_xar_vec,
593           .fno = gen_helper_sve2_xar_s,
594           .opt_opc = vecop,
595           .vece = MO_32 },
596         { .fni8 = gen_xar_i64,
597           .fniv = gen_xar_vec,
598           .fno = gen_helper_gvec_xar_d,
599           .opt_opc = vecop,
600           .vece = MO_64 }
601     };
602     int esize = 8 << vece;
603 
604     /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */
605     tcg_debug_assert(shift >= 0);
606     tcg_debug_assert(shift <= esize);
607     shift &= esize - 1;
608 
609     if (shift == 0) {
610         /* xar with no rotate devolves to xor. */
611         tcg_gen_gvec_xor(vece, rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz);
612     } else {
613         tcg_gen_gvec_3i(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz,
614                         shift, &ops[vece]);
615     }
616 }
617 
618 static bool trans_XAR(DisasContext *s, arg_rrri_esz *a)
619 {
620     if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) {
621         return false;
622     }
623     if (sve_access_check(s)) {
624         unsigned vsz = vec_full_reg_size(s);
625         gen_gvec_xar(a->esz, vec_full_reg_offset(s, a->rd),
626                      vec_full_reg_offset(s, a->rn),
627                      vec_full_reg_offset(s, a->rm), a->imm, vsz, vsz);
628     }
629     return true;
630 }
631 
632 static void gen_eor3_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
633 {
634     tcg_gen_xor_i64(d, n, m);
635     tcg_gen_xor_i64(d, d, k);
636 }
637 
638 static void gen_eor3_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
639                          TCGv_vec m, TCGv_vec k)
640 {
641     tcg_gen_xor_vec(vece, d, n, m);
642     tcg_gen_xor_vec(vece, d, d, k);
643 }
644 
645 static void gen_eor3(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
646                      uint32_t a, uint32_t oprsz, uint32_t maxsz)
647 {
648     static const GVecGen4 op = {
649         .fni8 = gen_eor3_i64,
650         .fniv = gen_eor3_vec,
651         .fno = gen_helper_sve2_eor3,
652         .vece = MO_64,
653         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
654     };
655     tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
656 }
657 
658 TRANS_FEAT(EOR3, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_eor3, a)
659 
660 static void gen_bcax_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
661 {
662     tcg_gen_andc_i64(d, m, k);
663     tcg_gen_xor_i64(d, d, n);
664 }
665 
666 static void gen_bcax_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
667                          TCGv_vec m, TCGv_vec k)
668 {
669     tcg_gen_andc_vec(vece, d, m, k);
670     tcg_gen_xor_vec(vece, d, d, n);
671 }
672 
673 static void gen_bcax(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
674                      uint32_t a, uint32_t oprsz, uint32_t maxsz)
675 {
676     static const GVecGen4 op = {
677         .fni8 = gen_bcax_i64,
678         .fniv = gen_bcax_vec,
679         .fno = gen_helper_sve2_bcax,
680         .vece = MO_64,
681         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
682     };
683     tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
684 }
685 
686 TRANS_FEAT(BCAX, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bcax, a)
687 
688 static void gen_bsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
689                     uint32_t a, uint32_t oprsz, uint32_t maxsz)
690 {
691     /* BSL differs from the generic bitsel in argument ordering. */
692     tcg_gen_gvec_bitsel(vece, d, a, n, m, oprsz, maxsz);
693 }
694 
695 TRANS_FEAT(BSL, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl, a)
696 
697 static void gen_bsl1n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
698 {
699     tcg_gen_andc_i64(n, k, n);
700     tcg_gen_andc_i64(m, m, k);
701     tcg_gen_or_i64(d, n, m);
702 }
703 
704 static void gen_bsl1n_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
705                           TCGv_vec m, TCGv_vec k)
706 {
707     if (TCG_TARGET_HAS_bitsel_vec) {
708         tcg_gen_not_vec(vece, n, n);
709         tcg_gen_bitsel_vec(vece, d, k, n, m);
710     } else {
711         tcg_gen_andc_vec(vece, n, k, n);
712         tcg_gen_andc_vec(vece, m, m, k);
713         tcg_gen_or_vec(vece, d, n, m);
714     }
715 }
716 
717 static void gen_bsl1n(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
718                       uint32_t a, uint32_t oprsz, uint32_t maxsz)
719 {
720     static const GVecGen4 op = {
721         .fni8 = gen_bsl1n_i64,
722         .fniv = gen_bsl1n_vec,
723         .fno = gen_helper_sve2_bsl1n,
724         .vece = MO_64,
725         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
726     };
727     tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
728 }
729 
730 TRANS_FEAT(BSL1N, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl1n, a)
731 
732 static void gen_bsl2n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
733 {
734     /*
735      * Z[dn] = (n & k) | (~m & ~k)
736      *       =         | ~(m | k)
737      */
738     tcg_gen_and_i64(n, n, k);
739     if (TCG_TARGET_HAS_orc_i64) {
740         tcg_gen_or_i64(m, m, k);
741         tcg_gen_orc_i64(d, n, m);
742     } else {
743         tcg_gen_nor_i64(m, m, k);
744         tcg_gen_or_i64(d, n, m);
745     }
746 }
747 
748 static void gen_bsl2n_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
749                           TCGv_vec m, TCGv_vec k)
750 {
751     if (TCG_TARGET_HAS_bitsel_vec) {
752         tcg_gen_not_vec(vece, m, m);
753         tcg_gen_bitsel_vec(vece, d, k, n, m);
754     } else {
755         tcg_gen_and_vec(vece, n, n, k);
756         tcg_gen_or_vec(vece, m, m, k);
757         tcg_gen_orc_vec(vece, d, n, m);
758     }
759 }
760 
761 static void gen_bsl2n(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
762                       uint32_t a, uint32_t oprsz, uint32_t maxsz)
763 {
764     static const GVecGen4 op = {
765         .fni8 = gen_bsl2n_i64,
766         .fniv = gen_bsl2n_vec,
767         .fno = gen_helper_sve2_bsl2n,
768         .vece = MO_64,
769         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
770     };
771     tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
772 }
773 
774 TRANS_FEAT(BSL2N, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl2n, a)
775 
776 static void gen_nbsl_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
777 {
778     tcg_gen_and_i64(n, n, k);
779     tcg_gen_andc_i64(m, m, k);
780     tcg_gen_nor_i64(d, n, m);
781 }
782 
783 static void gen_nbsl_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
784                           TCGv_vec m, TCGv_vec k)
785 {
786     tcg_gen_bitsel_vec(vece, d, k, n, m);
787     tcg_gen_not_vec(vece, d, d);
788 }
789 
790 static void gen_nbsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
791                      uint32_t a, uint32_t oprsz, uint32_t maxsz)
792 {
793     static const GVecGen4 op = {
794         .fni8 = gen_nbsl_i64,
795         .fniv = gen_nbsl_vec,
796         .fno = gen_helper_sve2_nbsl,
797         .vece = MO_64,
798         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
799     };
800     tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
801 }
802 
803 TRANS_FEAT(NBSL, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_nbsl, a)
804 
805 /*
806  *** SVE Integer Arithmetic - Unpredicated Group
807  */
808 
809 TRANS_FEAT(ADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_add, a)
810 TRANS_FEAT(SUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_sub, a)
811 TRANS_FEAT(SQADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_ssadd, a)
812 TRANS_FEAT(SQSUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_sssub, a)
813 TRANS_FEAT(UQADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_usadd, a)
814 TRANS_FEAT(UQSUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_ussub, a)
815 
816 /*
817  *** SVE Integer Arithmetic - Binary Predicated Group
818  */
819 
820 /* Select active elememnts from Zn and inactive elements from Zm,
821  * storing the result in Zd.
822  */
823 static bool do_sel_z(DisasContext *s, int rd, int rn, int rm, int pg, int esz)
824 {
825     static gen_helper_gvec_4 * const fns[4] = {
826         gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h,
827         gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d
828     };
829     return gen_gvec_ool_zzzp(s, fns[esz], rd, rn, rm, pg, 0);
830 }
831 
832 #define DO_ZPZZ(NAME, FEAT, name) \
833     static gen_helper_gvec_4 * const name##_zpzz_fns[4] = {               \
834         gen_helper_##name##_zpzz_b, gen_helper_##name##_zpzz_h,           \
835         gen_helper_##name##_zpzz_s, gen_helper_##name##_zpzz_d,           \
836     };                                                                    \
837     TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpzz,                         \
838                name##_zpzz_fns[a->esz], a, 0)
839 
840 DO_ZPZZ(AND_zpzz, aa64_sve, sve_and)
841 DO_ZPZZ(EOR_zpzz, aa64_sve, sve_eor)
842 DO_ZPZZ(ORR_zpzz, aa64_sve, sve_orr)
843 DO_ZPZZ(BIC_zpzz, aa64_sve, sve_bic)
844 
845 DO_ZPZZ(ADD_zpzz, aa64_sve, sve_add)
846 DO_ZPZZ(SUB_zpzz, aa64_sve, sve_sub)
847 
848 DO_ZPZZ(SMAX_zpzz, aa64_sve, sve_smax)
849 DO_ZPZZ(UMAX_zpzz, aa64_sve, sve_umax)
850 DO_ZPZZ(SMIN_zpzz, aa64_sve, sve_smin)
851 DO_ZPZZ(UMIN_zpzz, aa64_sve, sve_umin)
852 DO_ZPZZ(SABD_zpzz, aa64_sve, sve_sabd)
853 DO_ZPZZ(UABD_zpzz, aa64_sve, sve_uabd)
854 
855 DO_ZPZZ(MUL_zpzz, aa64_sve, sve_mul)
856 DO_ZPZZ(SMULH_zpzz, aa64_sve, sve_smulh)
857 DO_ZPZZ(UMULH_zpzz, aa64_sve, sve_umulh)
858 
859 DO_ZPZZ(ASR_zpzz, aa64_sve, sve_asr)
860 DO_ZPZZ(LSR_zpzz, aa64_sve, sve_lsr)
861 DO_ZPZZ(LSL_zpzz, aa64_sve, sve_lsl)
862 
863 static gen_helper_gvec_4 * const sdiv_fns[4] = {
864     NULL, NULL, gen_helper_sve_sdiv_zpzz_s, gen_helper_sve_sdiv_zpzz_d
865 };
866 TRANS_FEAT(SDIV_zpzz, aa64_sve, gen_gvec_ool_arg_zpzz, sdiv_fns[a->esz], a, 0)
867 
868 static gen_helper_gvec_4 * const udiv_fns[4] = {
869     NULL, NULL, gen_helper_sve_udiv_zpzz_s, gen_helper_sve_udiv_zpzz_d
870 };
871 TRANS_FEAT(UDIV_zpzz, aa64_sve, gen_gvec_ool_arg_zpzz, udiv_fns[a->esz], a, 0)
872 
873 TRANS_FEAT(SEL_zpzz, aa64_sve, do_sel_z, a->rd, a->rn, a->rm, a->pg, a->esz)
874 
875 /*
876  *** SVE Integer Arithmetic - Unary Predicated Group
877  */
878 
879 #define DO_ZPZ(NAME, FEAT, name) \
880     static gen_helper_gvec_3 * const name##_fns[4] = {              \
881         gen_helper_##name##_b, gen_helper_##name##_h,               \
882         gen_helper_##name##_s, gen_helper_##name##_d,               \
883     };                                                              \
884     TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpz, name##_fns[a->esz], a, 0)
885 
886 DO_ZPZ(CLS, aa64_sve, sve_cls)
887 DO_ZPZ(CLZ, aa64_sve, sve_clz)
888 DO_ZPZ(CNT_zpz, aa64_sve, sve_cnt_zpz)
889 DO_ZPZ(CNOT, aa64_sve, sve_cnot)
890 DO_ZPZ(NOT_zpz, aa64_sve, sve_not_zpz)
891 DO_ZPZ(ABS, aa64_sve, sve_abs)
892 DO_ZPZ(NEG, aa64_sve, sve_neg)
893 DO_ZPZ(RBIT, aa64_sve, sve_rbit)
894 
895 static gen_helper_gvec_3 * const fabs_fns[4] = {
896     NULL,                  gen_helper_sve_fabs_h,
897     gen_helper_sve_fabs_s, gen_helper_sve_fabs_d,
898 };
899 TRANS_FEAT(FABS, aa64_sve, gen_gvec_ool_arg_zpz, fabs_fns[a->esz], a, 0)
900 
901 static gen_helper_gvec_3 * const fneg_fns[4] = {
902     NULL,                  gen_helper_sve_fneg_h,
903     gen_helper_sve_fneg_s, gen_helper_sve_fneg_d,
904 };
905 TRANS_FEAT(FNEG, aa64_sve, gen_gvec_ool_arg_zpz, fneg_fns[a->esz], a, 0)
906 
907 static gen_helper_gvec_3 * const sxtb_fns[4] = {
908     NULL,                  gen_helper_sve_sxtb_h,
909     gen_helper_sve_sxtb_s, gen_helper_sve_sxtb_d,
910 };
911 TRANS_FEAT(SXTB, aa64_sve, gen_gvec_ool_arg_zpz, sxtb_fns[a->esz], a, 0)
912 
913 static gen_helper_gvec_3 * const uxtb_fns[4] = {
914     NULL,                  gen_helper_sve_uxtb_h,
915     gen_helper_sve_uxtb_s, gen_helper_sve_uxtb_d,
916 };
917 TRANS_FEAT(UXTB, aa64_sve, gen_gvec_ool_arg_zpz, uxtb_fns[a->esz], a, 0)
918 
919 static gen_helper_gvec_3 * const sxth_fns[4] = {
920     NULL, NULL, gen_helper_sve_sxth_s, gen_helper_sve_sxth_d
921 };
922 TRANS_FEAT(SXTH, aa64_sve, gen_gvec_ool_arg_zpz, sxth_fns[a->esz], a, 0)
923 
924 static gen_helper_gvec_3 * const uxth_fns[4] = {
925     NULL, NULL, gen_helper_sve_uxth_s, gen_helper_sve_uxth_d
926 };
927 TRANS_FEAT(UXTH, aa64_sve, gen_gvec_ool_arg_zpz, uxth_fns[a->esz], a, 0)
928 
929 TRANS_FEAT(SXTW, aa64_sve, gen_gvec_ool_arg_zpz,
930            a->esz == 3 ? gen_helper_sve_sxtw_d : NULL, a, 0)
931 TRANS_FEAT(UXTW, aa64_sve, gen_gvec_ool_arg_zpz,
932            a->esz == 3 ? gen_helper_sve_uxtw_d : NULL, a, 0)
933 
934 /*
935  *** SVE Integer Reduction Group
936  */
937 
938 typedef void gen_helper_gvec_reduc(TCGv_i64, TCGv_ptr, TCGv_ptr, TCGv_i32);
939 static bool do_vpz_ool(DisasContext *s, arg_rpr_esz *a,
940                        gen_helper_gvec_reduc *fn)
941 {
942     unsigned vsz = vec_full_reg_size(s);
943     TCGv_ptr t_zn, t_pg;
944     TCGv_i32 desc;
945     TCGv_i64 temp;
946 
947     if (fn == NULL) {
948         return false;
949     }
950     if (!sve_access_check(s)) {
951         return true;
952     }
953 
954     desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
955     temp = tcg_temp_new_i64();
956     t_zn = tcg_temp_new_ptr();
957     t_pg = tcg_temp_new_ptr();
958 
959     tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, a->rn));
960     tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->pg));
961     fn(temp, t_zn, t_pg, desc);
962 
963     write_fp_dreg(s, a->rd, temp);
964     return true;
965 }
966 
967 #define DO_VPZ(NAME, name) \
968     static gen_helper_gvec_reduc * const name##_fns[4] = {               \
969         gen_helper_sve_##name##_b, gen_helper_sve_##name##_h,            \
970         gen_helper_sve_##name##_s, gen_helper_sve_##name##_d,            \
971     };                                                                   \
972     TRANS_FEAT(NAME, aa64_sve, do_vpz_ool, a, name##_fns[a->esz])
973 
974 DO_VPZ(ORV, orv)
975 DO_VPZ(ANDV, andv)
976 DO_VPZ(EORV, eorv)
977 
978 DO_VPZ(UADDV, uaddv)
979 DO_VPZ(SMAXV, smaxv)
980 DO_VPZ(UMAXV, umaxv)
981 DO_VPZ(SMINV, sminv)
982 DO_VPZ(UMINV, uminv)
983 
984 static gen_helper_gvec_reduc * const saddv_fns[4] = {
985     gen_helper_sve_saddv_b, gen_helper_sve_saddv_h,
986     gen_helper_sve_saddv_s, NULL
987 };
988 TRANS_FEAT(SADDV, aa64_sve, do_vpz_ool, a, saddv_fns[a->esz])
989 
990 #undef DO_VPZ
991 
992 /*
993  *** SVE Shift by Immediate - Predicated Group
994  */
995 
996 /*
997  * Copy Zn into Zd, storing zeros into inactive elements.
998  * If invert, store zeros into the active elements.
999  */
1000 static bool do_movz_zpz(DisasContext *s, int rd, int rn, int pg,
1001                         int esz, bool invert)
1002 {
1003     static gen_helper_gvec_3 * const fns[4] = {
1004         gen_helper_sve_movz_b, gen_helper_sve_movz_h,
1005         gen_helper_sve_movz_s, gen_helper_sve_movz_d,
1006     };
1007     return gen_gvec_ool_zzp(s, fns[esz], rd, rn, pg, invert);
1008 }
1009 
1010 static bool do_shift_zpzi(DisasContext *s, arg_rpri_esz *a, bool asr,
1011                           gen_helper_gvec_3 * const fns[4])
1012 {
1013     int max;
1014 
1015     if (a->esz < 0) {
1016         /* Invalid tsz encoding -- see tszimm_esz. */
1017         return false;
1018     }
1019 
1020     /*
1021      * Shift by element size is architecturally valid.
1022      * For arithmetic right-shift, it's the same as by one less.
1023      * For logical shifts and ASRD, it is a zeroing operation.
1024      */
1025     max = 8 << a->esz;
1026     if (a->imm >= max) {
1027         if (asr) {
1028             a->imm = max - 1;
1029         } else {
1030             return do_movz_zpz(s, a->rd, a->rd, a->pg, a->esz, true);
1031         }
1032     }
1033     return gen_gvec_ool_arg_zpzi(s, fns[a->esz], a);
1034 }
1035 
1036 static gen_helper_gvec_3 * const asr_zpzi_fns[4] = {
1037     gen_helper_sve_asr_zpzi_b, gen_helper_sve_asr_zpzi_h,
1038     gen_helper_sve_asr_zpzi_s, gen_helper_sve_asr_zpzi_d,
1039 };
1040 TRANS_FEAT(ASR_zpzi, aa64_sve, do_shift_zpzi, a, true, asr_zpzi_fns)
1041 
1042 static gen_helper_gvec_3 * const lsr_zpzi_fns[4] = {
1043     gen_helper_sve_lsr_zpzi_b, gen_helper_sve_lsr_zpzi_h,
1044     gen_helper_sve_lsr_zpzi_s, gen_helper_sve_lsr_zpzi_d,
1045 };
1046 TRANS_FEAT(LSR_zpzi, aa64_sve, do_shift_zpzi, a, false, lsr_zpzi_fns)
1047 
1048 static gen_helper_gvec_3 * const lsl_zpzi_fns[4] = {
1049     gen_helper_sve_lsl_zpzi_b, gen_helper_sve_lsl_zpzi_h,
1050     gen_helper_sve_lsl_zpzi_s, gen_helper_sve_lsl_zpzi_d,
1051 };
1052 TRANS_FEAT(LSL_zpzi, aa64_sve, do_shift_zpzi, a, false, lsl_zpzi_fns)
1053 
1054 static gen_helper_gvec_3 * const asrd_fns[4] = {
1055     gen_helper_sve_asrd_b, gen_helper_sve_asrd_h,
1056     gen_helper_sve_asrd_s, gen_helper_sve_asrd_d,
1057 };
1058 TRANS_FEAT(ASRD, aa64_sve, do_shift_zpzi, a, false, asrd_fns)
1059 
1060 static gen_helper_gvec_3 * const sqshl_zpzi_fns[4] = {
1061     gen_helper_sve2_sqshl_zpzi_b, gen_helper_sve2_sqshl_zpzi_h,
1062     gen_helper_sve2_sqshl_zpzi_s, gen_helper_sve2_sqshl_zpzi_d,
1063 };
1064 TRANS_FEAT(SQSHL_zpzi, aa64_sve2, gen_gvec_ool_arg_zpzi,
1065            a->esz < 0 ? NULL : sqshl_zpzi_fns[a->esz], a)
1066 
1067 static gen_helper_gvec_3 * const uqshl_zpzi_fns[4] = {
1068     gen_helper_sve2_uqshl_zpzi_b, gen_helper_sve2_uqshl_zpzi_h,
1069     gen_helper_sve2_uqshl_zpzi_s, gen_helper_sve2_uqshl_zpzi_d,
1070 };
1071 TRANS_FEAT(UQSHL_zpzi, aa64_sve2, gen_gvec_ool_arg_zpzi,
1072            a->esz < 0 ? NULL : uqshl_zpzi_fns[a->esz], a)
1073 
1074 static gen_helper_gvec_3 * const srshr_fns[4] = {
1075     gen_helper_sve2_srshr_b, gen_helper_sve2_srshr_h,
1076     gen_helper_sve2_srshr_s, gen_helper_sve2_srshr_d,
1077 };
1078 TRANS_FEAT(SRSHR, aa64_sve2, gen_gvec_ool_arg_zpzi,
1079            a->esz < 0 ? NULL : srshr_fns[a->esz], a)
1080 
1081 static gen_helper_gvec_3 * const urshr_fns[4] = {
1082     gen_helper_sve2_urshr_b, gen_helper_sve2_urshr_h,
1083     gen_helper_sve2_urshr_s, gen_helper_sve2_urshr_d,
1084 };
1085 TRANS_FEAT(URSHR, aa64_sve2, gen_gvec_ool_arg_zpzi,
1086            a->esz < 0 ? NULL : urshr_fns[a->esz], a)
1087 
1088 static gen_helper_gvec_3 * const sqshlu_fns[4] = {
1089     gen_helper_sve2_sqshlu_b, gen_helper_sve2_sqshlu_h,
1090     gen_helper_sve2_sqshlu_s, gen_helper_sve2_sqshlu_d,
1091 };
1092 TRANS_FEAT(SQSHLU, aa64_sve2, gen_gvec_ool_arg_zpzi,
1093            a->esz < 0 ? NULL : sqshlu_fns[a->esz], a)
1094 
1095 /*
1096  *** SVE Bitwise Shift - Predicated Group
1097  */
1098 
1099 #define DO_ZPZW(NAME, name) \
1100     static gen_helper_gvec_4 * const name##_zpzw_fns[4] = {               \
1101         gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h,   \
1102         gen_helper_sve_##name##_zpzw_s, NULL                              \
1103     };                                                                    \
1104     TRANS_FEAT(NAME##_zpzw, aa64_sve, gen_gvec_ool_arg_zpzz,              \
1105                a->esz < 0 ? NULL : name##_zpzw_fns[a->esz], a, 0)
1106 
1107 DO_ZPZW(ASR, asr)
1108 DO_ZPZW(LSR, lsr)
1109 DO_ZPZW(LSL, lsl)
1110 
1111 #undef DO_ZPZW
1112 
1113 /*
1114  *** SVE Bitwise Shift - Unpredicated Group
1115  */
1116 
1117 static bool do_shift_imm(DisasContext *s, arg_rri_esz *a, bool asr,
1118                          void (*gvec_fn)(unsigned, uint32_t, uint32_t,
1119                                          int64_t, uint32_t, uint32_t))
1120 {
1121     if (a->esz < 0) {
1122         /* Invalid tsz encoding -- see tszimm_esz. */
1123         return false;
1124     }
1125     if (sve_access_check(s)) {
1126         unsigned vsz = vec_full_reg_size(s);
1127         /* Shift by element size is architecturally valid.  For
1128            arithmetic right-shift, it's the same as by one less.
1129            Otherwise it is a zeroing operation.  */
1130         if (a->imm >= 8 << a->esz) {
1131             if (asr) {
1132                 a->imm = (8 << a->esz) - 1;
1133             } else {
1134                 do_dupi_z(s, a->rd, 0);
1135                 return true;
1136             }
1137         }
1138         gvec_fn(a->esz, vec_full_reg_offset(s, a->rd),
1139                 vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz);
1140     }
1141     return true;
1142 }
1143 
1144 TRANS_FEAT(ASR_zzi, aa64_sve, do_shift_imm, a, true, tcg_gen_gvec_sari)
1145 TRANS_FEAT(LSR_zzi, aa64_sve, do_shift_imm, a, false, tcg_gen_gvec_shri)
1146 TRANS_FEAT(LSL_zzi, aa64_sve, do_shift_imm, a, false, tcg_gen_gvec_shli)
1147 
1148 #define DO_ZZW(NAME, name) \
1149     static gen_helper_gvec_3 * const name##_zzw_fns[4] = {                \
1150         gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h,     \
1151         gen_helper_sve_##name##_zzw_s, NULL                               \
1152     };                                                                    \
1153     TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_arg_zzz,                      \
1154                name##_zzw_fns[a->esz], a, 0)
1155 
1156 DO_ZZW(ASR_zzw, asr)
1157 DO_ZZW(LSR_zzw, lsr)
1158 DO_ZZW(LSL_zzw, lsl)
1159 
1160 #undef DO_ZZW
1161 
1162 /*
1163  *** SVE Integer Multiply-Add Group
1164  */
1165 
1166 static bool do_zpzzz_ool(DisasContext *s, arg_rprrr_esz *a,
1167                          gen_helper_gvec_5 *fn)
1168 {
1169     if (sve_access_check(s)) {
1170         unsigned vsz = vec_full_reg_size(s);
1171         tcg_gen_gvec_5_ool(vec_full_reg_offset(s, a->rd),
1172                            vec_full_reg_offset(s, a->ra),
1173                            vec_full_reg_offset(s, a->rn),
1174                            vec_full_reg_offset(s, a->rm),
1175                            pred_full_reg_offset(s, a->pg),
1176                            vsz, vsz, 0, fn);
1177     }
1178     return true;
1179 }
1180 
1181 static gen_helper_gvec_5 * const mla_fns[4] = {
1182     gen_helper_sve_mla_b, gen_helper_sve_mla_h,
1183     gen_helper_sve_mla_s, gen_helper_sve_mla_d,
1184 };
1185 TRANS_FEAT(MLA, aa64_sve, do_zpzzz_ool, a, mla_fns[a->esz])
1186 
1187 static gen_helper_gvec_5 * const mls_fns[4] = {
1188     gen_helper_sve_mls_b, gen_helper_sve_mls_h,
1189     gen_helper_sve_mls_s, gen_helper_sve_mls_d,
1190 };
1191 TRANS_FEAT(MLS, aa64_sve, do_zpzzz_ool, a, mls_fns[a->esz])
1192 
1193 /*
1194  *** SVE Index Generation Group
1195  */
1196 
1197 static bool do_index(DisasContext *s, int esz, int rd,
1198                      TCGv_i64 start, TCGv_i64 incr)
1199 {
1200     unsigned vsz;
1201     TCGv_i32 desc;
1202     TCGv_ptr t_zd;
1203 
1204     if (!sve_access_check(s)) {
1205         return true;
1206     }
1207 
1208     vsz = vec_full_reg_size(s);
1209     desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
1210     t_zd = tcg_temp_new_ptr();
1211 
1212     tcg_gen_addi_ptr(t_zd, cpu_env, vec_full_reg_offset(s, rd));
1213     if (esz == 3) {
1214         gen_helper_sve_index_d(t_zd, start, incr, desc);
1215     } else {
1216         typedef void index_fn(TCGv_ptr, TCGv_i32, TCGv_i32, TCGv_i32);
1217         static index_fn * const fns[3] = {
1218             gen_helper_sve_index_b,
1219             gen_helper_sve_index_h,
1220             gen_helper_sve_index_s,
1221         };
1222         TCGv_i32 s32 = tcg_temp_new_i32();
1223         TCGv_i32 i32 = tcg_temp_new_i32();
1224 
1225         tcg_gen_extrl_i64_i32(s32, start);
1226         tcg_gen_extrl_i64_i32(i32, incr);
1227         fns[esz](t_zd, s32, i32, desc);
1228     }
1229     return true;
1230 }
1231 
1232 TRANS_FEAT(INDEX_ii, aa64_sve, do_index, a->esz, a->rd,
1233            tcg_constant_i64(a->imm1), tcg_constant_i64(a->imm2))
1234 TRANS_FEAT(INDEX_ir, aa64_sve, do_index, a->esz, a->rd,
1235            tcg_constant_i64(a->imm), cpu_reg(s, a->rm))
1236 TRANS_FEAT(INDEX_ri, aa64_sve, do_index, a->esz, a->rd,
1237            cpu_reg(s, a->rn), tcg_constant_i64(a->imm))
1238 TRANS_FEAT(INDEX_rr, aa64_sve, do_index, a->esz, a->rd,
1239            cpu_reg(s, a->rn), cpu_reg(s, a->rm))
1240 
1241 /*
1242  *** SVE Stack Allocation Group
1243  */
1244 
1245 static bool trans_ADDVL(DisasContext *s, arg_ADDVL *a)
1246 {
1247     if (!dc_isar_feature(aa64_sve, s)) {
1248         return false;
1249     }
1250     if (sve_access_check(s)) {
1251         TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1252         TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1253         tcg_gen_addi_i64(rd, rn, a->imm * vec_full_reg_size(s));
1254     }
1255     return true;
1256 }
1257 
1258 static bool trans_ADDSVL(DisasContext *s, arg_ADDSVL *a)
1259 {
1260     if (!dc_isar_feature(aa64_sme, s)) {
1261         return false;
1262     }
1263     if (sme_enabled_check(s)) {
1264         TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1265         TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1266         tcg_gen_addi_i64(rd, rn, a->imm * streaming_vec_reg_size(s));
1267     }
1268     return true;
1269 }
1270 
1271 static bool trans_ADDPL(DisasContext *s, arg_ADDPL *a)
1272 {
1273     if (!dc_isar_feature(aa64_sve, s)) {
1274         return false;
1275     }
1276     if (sve_access_check(s)) {
1277         TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1278         TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1279         tcg_gen_addi_i64(rd, rn, a->imm * pred_full_reg_size(s));
1280     }
1281     return true;
1282 }
1283 
1284 static bool trans_ADDSPL(DisasContext *s, arg_ADDSPL *a)
1285 {
1286     if (!dc_isar_feature(aa64_sme, s)) {
1287         return false;
1288     }
1289     if (sme_enabled_check(s)) {
1290         TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1291         TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1292         tcg_gen_addi_i64(rd, rn, a->imm * streaming_pred_reg_size(s));
1293     }
1294     return true;
1295 }
1296 
1297 static bool trans_RDVL(DisasContext *s, arg_RDVL *a)
1298 {
1299     if (!dc_isar_feature(aa64_sve, s)) {
1300         return false;
1301     }
1302     if (sve_access_check(s)) {
1303         TCGv_i64 reg = cpu_reg(s, a->rd);
1304         tcg_gen_movi_i64(reg, a->imm * vec_full_reg_size(s));
1305     }
1306     return true;
1307 }
1308 
1309 static bool trans_RDSVL(DisasContext *s, arg_RDSVL *a)
1310 {
1311     if (!dc_isar_feature(aa64_sme, s)) {
1312         return false;
1313     }
1314     if (sme_enabled_check(s)) {
1315         TCGv_i64 reg = cpu_reg(s, a->rd);
1316         tcg_gen_movi_i64(reg, a->imm * streaming_vec_reg_size(s));
1317     }
1318     return true;
1319 }
1320 
1321 /*
1322  *** SVE Compute Vector Address Group
1323  */
1324 
1325 static bool do_adr(DisasContext *s, arg_rrri *a, gen_helper_gvec_3 *fn)
1326 {
1327     return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, a->imm);
1328 }
1329 
1330 TRANS_FEAT_NONSTREAMING(ADR_p32, aa64_sve, do_adr, a, gen_helper_sve_adr_p32)
1331 TRANS_FEAT_NONSTREAMING(ADR_p64, aa64_sve, do_adr, a, gen_helper_sve_adr_p64)
1332 TRANS_FEAT_NONSTREAMING(ADR_s32, aa64_sve, do_adr, a, gen_helper_sve_adr_s32)
1333 TRANS_FEAT_NONSTREAMING(ADR_u32, aa64_sve, do_adr, a, gen_helper_sve_adr_u32)
1334 
1335 /*
1336  *** SVE Integer Misc - Unpredicated Group
1337  */
1338 
1339 static gen_helper_gvec_2 * const fexpa_fns[4] = {
1340     NULL,                   gen_helper_sve_fexpa_h,
1341     gen_helper_sve_fexpa_s, gen_helper_sve_fexpa_d,
1342 };
1343 TRANS_FEAT_NONSTREAMING(FEXPA, aa64_sve, gen_gvec_ool_zz,
1344                         fexpa_fns[a->esz], a->rd, a->rn, 0)
1345 
1346 static gen_helper_gvec_3 * const ftssel_fns[4] = {
1347     NULL,                    gen_helper_sve_ftssel_h,
1348     gen_helper_sve_ftssel_s, gen_helper_sve_ftssel_d,
1349 };
1350 TRANS_FEAT_NONSTREAMING(FTSSEL, aa64_sve, gen_gvec_ool_arg_zzz,
1351                         ftssel_fns[a->esz], a, 0)
1352 
1353 /*
1354  *** SVE Predicate Logical Operations Group
1355  */
1356 
1357 static bool do_pppp_flags(DisasContext *s, arg_rprr_s *a,
1358                           const GVecGen4 *gvec_op)
1359 {
1360     if (!sve_access_check(s)) {
1361         return true;
1362     }
1363 
1364     unsigned psz = pred_gvec_reg_size(s);
1365     int dofs = pred_full_reg_offset(s, a->rd);
1366     int nofs = pred_full_reg_offset(s, a->rn);
1367     int mofs = pred_full_reg_offset(s, a->rm);
1368     int gofs = pred_full_reg_offset(s, a->pg);
1369 
1370     if (!a->s) {
1371         tcg_gen_gvec_4(dofs, nofs, mofs, gofs, psz, psz, gvec_op);
1372         return true;
1373     }
1374 
1375     if (psz == 8) {
1376         /* Do the operation and the flags generation in temps.  */
1377         TCGv_i64 pd = tcg_temp_new_i64();
1378         TCGv_i64 pn = tcg_temp_new_i64();
1379         TCGv_i64 pm = tcg_temp_new_i64();
1380         TCGv_i64 pg = tcg_temp_new_i64();
1381 
1382         tcg_gen_ld_i64(pn, cpu_env, nofs);
1383         tcg_gen_ld_i64(pm, cpu_env, mofs);
1384         tcg_gen_ld_i64(pg, cpu_env, gofs);
1385 
1386         gvec_op->fni8(pd, pn, pm, pg);
1387         tcg_gen_st_i64(pd, cpu_env, dofs);
1388 
1389         do_predtest1(pd, pg);
1390     } else {
1391         /* The operation and flags generation is large.  The computation
1392          * of the flags depends on the original contents of the guarding
1393          * predicate.  If the destination overwrites the guarding predicate,
1394          * then the easiest way to get this right is to save a copy.
1395           */
1396         int tofs = gofs;
1397         if (a->rd == a->pg) {
1398             tofs = offsetof(CPUARMState, vfp.preg_tmp);
1399             tcg_gen_gvec_mov(0, tofs, gofs, psz, psz);
1400         }
1401 
1402         tcg_gen_gvec_4(dofs, nofs, mofs, gofs, psz, psz, gvec_op);
1403         do_predtest(s, dofs, tofs, psz / 8);
1404     }
1405     return true;
1406 }
1407 
1408 static void gen_and_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1409 {
1410     tcg_gen_and_i64(pd, pn, pm);
1411     tcg_gen_and_i64(pd, pd, pg);
1412 }
1413 
1414 static void gen_and_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1415                            TCGv_vec pm, TCGv_vec pg)
1416 {
1417     tcg_gen_and_vec(vece, pd, pn, pm);
1418     tcg_gen_and_vec(vece, pd, pd, pg);
1419 }
1420 
1421 static bool trans_AND_pppp(DisasContext *s, arg_rprr_s *a)
1422 {
1423     static const GVecGen4 op = {
1424         .fni8 = gen_and_pg_i64,
1425         .fniv = gen_and_pg_vec,
1426         .fno = gen_helper_sve_and_pppp,
1427         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1428     };
1429 
1430     if (!dc_isar_feature(aa64_sve, s)) {
1431         return false;
1432     }
1433     if (!a->s) {
1434         if (a->rn == a->rm) {
1435             if (a->pg == a->rn) {
1436                 return do_mov_p(s, a->rd, a->rn);
1437             }
1438             return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->pg);
1439         } else if (a->pg == a->rn || a->pg == a->rm) {
1440             return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->rm);
1441         }
1442     }
1443     return do_pppp_flags(s, a, &op);
1444 }
1445 
1446 static void gen_bic_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1447 {
1448     tcg_gen_andc_i64(pd, pn, pm);
1449     tcg_gen_and_i64(pd, pd, pg);
1450 }
1451 
1452 static void gen_bic_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1453                            TCGv_vec pm, TCGv_vec pg)
1454 {
1455     tcg_gen_andc_vec(vece, pd, pn, pm);
1456     tcg_gen_and_vec(vece, pd, pd, pg);
1457 }
1458 
1459 static bool trans_BIC_pppp(DisasContext *s, arg_rprr_s *a)
1460 {
1461     static const GVecGen4 op = {
1462         .fni8 = gen_bic_pg_i64,
1463         .fniv = gen_bic_pg_vec,
1464         .fno = gen_helper_sve_bic_pppp,
1465         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1466     };
1467 
1468     if (!dc_isar_feature(aa64_sve, s)) {
1469         return false;
1470     }
1471     if (!a->s && a->pg == a->rn) {
1472         return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->rn, a->rm);
1473     }
1474     return do_pppp_flags(s, a, &op);
1475 }
1476 
1477 static void gen_eor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1478 {
1479     tcg_gen_xor_i64(pd, pn, pm);
1480     tcg_gen_and_i64(pd, pd, pg);
1481 }
1482 
1483 static void gen_eor_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1484                            TCGv_vec pm, TCGv_vec pg)
1485 {
1486     tcg_gen_xor_vec(vece, pd, pn, pm);
1487     tcg_gen_and_vec(vece, pd, pd, pg);
1488 }
1489 
1490 static bool trans_EOR_pppp(DisasContext *s, arg_rprr_s *a)
1491 {
1492     static const GVecGen4 op = {
1493         .fni8 = gen_eor_pg_i64,
1494         .fniv = gen_eor_pg_vec,
1495         .fno = gen_helper_sve_eor_pppp,
1496         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1497     };
1498 
1499     if (!dc_isar_feature(aa64_sve, s)) {
1500         return false;
1501     }
1502     /* Alias NOT (predicate) is EOR Pd.B, Pg/Z, Pn.B, Pg.B */
1503     if (!a->s && a->pg == a->rm) {
1504         return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->pg, a->rn);
1505     }
1506     return do_pppp_flags(s, a, &op);
1507 }
1508 
1509 static bool trans_SEL_pppp(DisasContext *s, arg_rprr_s *a)
1510 {
1511     if (a->s || !dc_isar_feature(aa64_sve, s)) {
1512         return false;
1513     }
1514     if (sve_access_check(s)) {
1515         unsigned psz = pred_gvec_reg_size(s);
1516         tcg_gen_gvec_bitsel(MO_8, pred_full_reg_offset(s, a->rd),
1517                             pred_full_reg_offset(s, a->pg),
1518                             pred_full_reg_offset(s, a->rn),
1519                             pred_full_reg_offset(s, a->rm), psz, psz);
1520     }
1521     return true;
1522 }
1523 
1524 static void gen_orr_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1525 {
1526     tcg_gen_or_i64(pd, pn, pm);
1527     tcg_gen_and_i64(pd, pd, pg);
1528 }
1529 
1530 static void gen_orr_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1531                            TCGv_vec pm, TCGv_vec pg)
1532 {
1533     tcg_gen_or_vec(vece, pd, pn, pm);
1534     tcg_gen_and_vec(vece, pd, pd, pg);
1535 }
1536 
1537 static bool trans_ORR_pppp(DisasContext *s, arg_rprr_s *a)
1538 {
1539     static const GVecGen4 op = {
1540         .fni8 = gen_orr_pg_i64,
1541         .fniv = gen_orr_pg_vec,
1542         .fno = gen_helper_sve_orr_pppp,
1543         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1544     };
1545 
1546     if (!dc_isar_feature(aa64_sve, s)) {
1547         return false;
1548     }
1549     if (!a->s && a->pg == a->rn && a->rn == a->rm) {
1550         return do_mov_p(s, a->rd, a->rn);
1551     }
1552     return do_pppp_flags(s, a, &op);
1553 }
1554 
1555 static void gen_orn_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1556 {
1557     tcg_gen_orc_i64(pd, pn, pm);
1558     tcg_gen_and_i64(pd, pd, pg);
1559 }
1560 
1561 static void gen_orn_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1562                            TCGv_vec pm, TCGv_vec pg)
1563 {
1564     tcg_gen_orc_vec(vece, pd, pn, pm);
1565     tcg_gen_and_vec(vece, pd, pd, pg);
1566 }
1567 
1568 static bool trans_ORN_pppp(DisasContext *s, arg_rprr_s *a)
1569 {
1570     static const GVecGen4 op = {
1571         .fni8 = gen_orn_pg_i64,
1572         .fniv = gen_orn_pg_vec,
1573         .fno = gen_helper_sve_orn_pppp,
1574         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1575     };
1576 
1577     if (!dc_isar_feature(aa64_sve, s)) {
1578         return false;
1579     }
1580     return do_pppp_flags(s, a, &op);
1581 }
1582 
1583 static void gen_nor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1584 {
1585     tcg_gen_or_i64(pd, pn, pm);
1586     tcg_gen_andc_i64(pd, pg, pd);
1587 }
1588 
1589 static void gen_nor_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1590                            TCGv_vec pm, TCGv_vec pg)
1591 {
1592     tcg_gen_or_vec(vece, pd, pn, pm);
1593     tcg_gen_andc_vec(vece, pd, pg, pd);
1594 }
1595 
1596 static bool trans_NOR_pppp(DisasContext *s, arg_rprr_s *a)
1597 {
1598     static const GVecGen4 op = {
1599         .fni8 = gen_nor_pg_i64,
1600         .fniv = gen_nor_pg_vec,
1601         .fno = gen_helper_sve_nor_pppp,
1602         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1603     };
1604 
1605     if (!dc_isar_feature(aa64_sve, s)) {
1606         return false;
1607     }
1608     return do_pppp_flags(s, a, &op);
1609 }
1610 
1611 static void gen_nand_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1612 {
1613     tcg_gen_and_i64(pd, pn, pm);
1614     tcg_gen_andc_i64(pd, pg, pd);
1615 }
1616 
1617 static void gen_nand_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1618                            TCGv_vec pm, TCGv_vec pg)
1619 {
1620     tcg_gen_and_vec(vece, pd, pn, pm);
1621     tcg_gen_andc_vec(vece, pd, pg, pd);
1622 }
1623 
1624 static bool trans_NAND_pppp(DisasContext *s, arg_rprr_s *a)
1625 {
1626     static const GVecGen4 op = {
1627         .fni8 = gen_nand_pg_i64,
1628         .fniv = gen_nand_pg_vec,
1629         .fno = gen_helper_sve_nand_pppp,
1630         .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1631     };
1632 
1633     if (!dc_isar_feature(aa64_sve, s)) {
1634         return false;
1635     }
1636     return do_pppp_flags(s, a, &op);
1637 }
1638 
1639 /*
1640  *** SVE Predicate Misc Group
1641  */
1642 
1643 static bool trans_PTEST(DisasContext *s, arg_PTEST *a)
1644 {
1645     if (!dc_isar_feature(aa64_sve, s)) {
1646         return false;
1647     }
1648     if (sve_access_check(s)) {
1649         int nofs = pred_full_reg_offset(s, a->rn);
1650         int gofs = pred_full_reg_offset(s, a->pg);
1651         int words = DIV_ROUND_UP(pred_full_reg_size(s), 8);
1652 
1653         if (words == 1) {
1654             TCGv_i64 pn = tcg_temp_new_i64();
1655             TCGv_i64 pg = tcg_temp_new_i64();
1656 
1657             tcg_gen_ld_i64(pn, cpu_env, nofs);
1658             tcg_gen_ld_i64(pg, cpu_env, gofs);
1659             do_predtest1(pn, pg);
1660         } else {
1661             do_predtest(s, nofs, gofs, words);
1662         }
1663     }
1664     return true;
1665 }
1666 
1667 /* See the ARM pseudocode DecodePredCount.  */
1668 static unsigned decode_pred_count(unsigned fullsz, int pattern, int esz)
1669 {
1670     unsigned elements = fullsz >> esz;
1671     unsigned bound;
1672 
1673     switch (pattern) {
1674     case 0x0: /* POW2 */
1675         return pow2floor(elements);
1676     case 0x1: /* VL1 */
1677     case 0x2: /* VL2 */
1678     case 0x3: /* VL3 */
1679     case 0x4: /* VL4 */
1680     case 0x5: /* VL5 */
1681     case 0x6: /* VL6 */
1682     case 0x7: /* VL7 */
1683     case 0x8: /* VL8 */
1684         bound = pattern;
1685         break;
1686     case 0x9: /* VL16 */
1687     case 0xa: /* VL32 */
1688     case 0xb: /* VL64 */
1689     case 0xc: /* VL128 */
1690     case 0xd: /* VL256 */
1691         bound = 16 << (pattern - 9);
1692         break;
1693     case 0x1d: /* MUL4 */
1694         return elements - elements % 4;
1695     case 0x1e: /* MUL3 */
1696         return elements - elements % 3;
1697     case 0x1f: /* ALL */
1698         return elements;
1699     default:   /* #uimm5 */
1700         return 0;
1701     }
1702     return elements >= bound ? bound : 0;
1703 }
1704 
1705 /* This handles all of the predicate initialization instructions,
1706  * PTRUE, PFALSE, SETFFR.  For PFALSE, we will have set PAT == 32
1707  * so that decode_pred_count returns 0.  For SETFFR, we will have
1708  * set RD == 16 == FFR.
1709  */
1710 static bool do_predset(DisasContext *s, int esz, int rd, int pat, bool setflag)
1711 {
1712     if (!sve_access_check(s)) {
1713         return true;
1714     }
1715 
1716     unsigned fullsz = vec_full_reg_size(s);
1717     unsigned ofs = pred_full_reg_offset(s, rd);
1718     unsigned numelem, setsz, i;
1719     uint64_t word, lastword;
1720     TCGv_i64 t;
1721 
1722     numelem = decode_pred_count(fullsz, pat, esz);
1723 
1724     /* Determine what we must store into each bit, and how many.  */
1725     if (numelem == 0) {
1726         lastword = word = 0;
1727         setsz = fullsz;
1728     } else {
1729         setsz = numelem << esz;
1730         lastword = word = pred_esz_masks[esz];
1731         if (setsz % 64) {
1732             lastword &= MAKE_64BIT_MASK(0, setsz % 64);
1733         }
1734     }
1735 
1736     t = tcg_temp_new_i64();
1737     if (fullsz <= 64) {
1738         tcg_gen_movi_i64(t, lastword);
1739         tcg_gen_st_i64(t, cpu_env, ofs);
1740         goto done;
1741     }
1742 
1743     if (word == lastword) {
1744         unsigned maxsz = size_for_gvec(fullsz / 8);
1745         unsigned oprsz = size_for_gvec(setsz / 8);
1746 
1747         if (oprsz * 8 == setsz) {
1748             tcg_gen_gvec_dup_imm(MO_64, ofs, oprsz, maxsz, word);
1749             goto done;
1750         }
1751     }
1752 
1753     setsz /= 8;
1754     fullsz /= 8;
1755 
1756     tcg_gen_movi_i64(t, word);
1757     for (i = 0; i < QEMU_ALIGN_DOWN(setsz, 8); i += 8) {
1758         tcg_gen_st_i64(t, cpu_env, ofs + i);
1759     }
1760     if (lastword != word) {
1761         tcg_gen_movi_i64(t, lastword);
1762         tcg_gen_st_i64(t, cpu_env, ofs + i);
1763         i += 8;
1764     }
1765     if (i < fullsz) {
1766         tcg_gen_movi_i64(t, 0);
1767         for (; i < fullsz; i += 8) {
1768             tcg_gen_st_i64(t, cpu_env, ofs + i);
1769         }
1770     }
1771 
1772  done:
1773     /* PTRUES */
1774     if (setflag) {
1775         tcg_gen_movi_i32(cpu_NF, -(word != 0));
1776         tcg_gen_movi_i32(cpu_CF, word == 0);
1777         tcg_gen_movi_i32(cpu_VF, 0);
1778         tcg_gen_mov_i32(cpu_ZF, cpu_NF);
1779     }
1780     return true;
1781 }
1782 
1783 TRANS_FEAT(PTRUE, aa64_sve, do_predset, a->esz, a->rd, a->pat, a->s)
1784 
1785 /* Note pat == 31 is #all, to set all elements.  */
1786 TRANS_FEAT_NONSTREAMING(SETFFR, aa64_sve,
1787                         do_predset, 0, FFR_PRED_NUM, 31, false)
1788 
1789 /* Note pat == 32 is #unimp, to set no elements.  */
1790 TRANS_FEAT(PFALSE, aa64_sve, do_predset, 0, a->rd, 32, false)
1791 
1792 static bool trans_RDFFR_p(DisasContext *s, arg_RDFFR_p *a)
1793 {
1794     /* The path through do_pppp_flags is complicated enough to want to avoid
1795      * duplication.  Frob the arguments into the form of a predicated AND.
1796      */
1797     arg_rprr_s alt_a = {
1798         .rd = a->rd, .pg = a->pg, .s = a->s,
1799         .rn = FFR_PRED_NUM, .rm = FFR_PRED_NUM,
1800     };
1801 
1802     s->is_nonstreaming = true;
1803     return trans_AND_pppp(s, &alt_a);
1804 }
1805 
1806 TRANS_FEAT_NONSTREAMING(RDFFR, aa64_sve, do_mov_p, a->rd, FFR_PRED_NUM)
1807 TRANS_FEAT_NONSTREAMING(WRFFR, aa64_sve, do_mov_p, FFR_PRED_NUM, a->rn)
1808 
1809 static bool do_pfirst_pnext(DisasContext *s, arg_rr_esz *a,
1810                             void (*gen_fn)(TCGv_i32, TCGv_ptr,
1811                                            TCGv_ptr, TCGv_i32))
1812 {
1813     if (!sve_access_check(s)) {
1814         return true;
1815     }
1816 
1817     TCGv_ptr t_pd = tcg_temp_new_ptr();
1818     TCGv_ptr t_pg = tcg_temp_new_ptr();
1819     TCGv_i32 t;
1820     unsigned desc = 0;
1821 
1822     desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pred_full_reg_size(s));
1823     desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
1824 
1825     tcg_gen_addi_ptr(t_pd, cpu_env, pred_full_reg_offset(s, a->rd));
1826     tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->rn));
1827     t = tcg_temp_new_i32();
1828 
1829     gen_fn(t, t_pd, t_pg, tcg_constant_i32(desc));
1830 
1831     do_pred_flags(t);
1832     return true;
1833 }
1834 
1835 TRANS_FEAT(PFIRST, aa64_sve, do_pfirst_pnext, a, gen_helper_sve_pfirst)
1836 TRANS_FEAT(PNEXT, aa64_sve, do_pfirst_pnext, a, gen_helper_sve_pnext)
1837 
1838 /*
1839  *** SVE Element Count Group
1840  */
1841 
1842 /* Perform an inline saturating addition of a 32-bit value within
1843  * a 64-bit register.  The second operand is known to be positive,
1844  * which halves the comparisions we must perform to bound the result.
1845  */
1846 static void do_sat_addsub_32(TCGv_i64 reg, TCGv_i64 val, bool u, bool d)
1847 {
1848     int64_t ibound;
1849 
1850     /* Use normal 64-bit arithmetic to detect 32-bit overflow.  */
1851     if (u) {
1852         tcg_gen_ext32u_i64(reg, reg);
1853     } else {
1854         tcg_gen_ext32s_i64(reg, reg);
1855     }
1856     if (d) {
1857         tcg_gen_sub_i64(reg, reg, val);
1858         ibound = (u ? 0 : INT32_MIN);
1859         tcg_gen_smax_i64(reg, reg, tcg_constant_i64(ibound));
1860     } else {
1861         tcg_gen_add_i64(reg, reg, val);
1862         ibound = (u ? UINT32_MAX : INT32_MAX);
1863         tcg_gen_smin_i64(reg, reg, tcg_constant_i64(ibound));
1864     }
1865 }
1866 
1867 /* Similarly with 64-bit values.  */
1868 static void do_sat_addsub_64(TCGv_i64 reg, TCGv_i64 val, bool u, bool d)
1869 {
1870     TCGv_i64 t0 = tcg_temp_new_i64();
1871     TCGv_i64 t2;
1872 
1873     if (u) {
1874         if (d) {
1875             tcg_gen_sub_i64(t0, reg, val);
1876             t2 = tcg_constant_i64(0);
1877             tcg_gen_movcond_i64(TCG_COND_LTU, reg, reg, val, t2, t0);
1878         } else {
1879             tcg_gen_add_i64(t0, reg, val);
1880             t2 = tcg_constant_i64(-1);
1881             tcg_gen_movcond_i64(TCG_COND_LTU, reg, t0, reg, t2, t0);
1882         }
1883     } else {
1884         TCGv_i64 t1 = tcg_temp_new_i64();
1885         if (d) {
1886             /* Detect signed overflow for subtraction.  */
1887             tcg_gen_xor_i64(t0, reg, val);
1888             tcg_gen_sub_i64(t1, reg, val);
1889             tcg_gen_xor_i64(reg, reg, t1);
1890             tcg_gen_and_i64(t0, t0, reg);
1891 
1892             /* Bound the result.  */
1893             tcg_gen_movi_i64(reg, INT64_MIN);
1894             t2 = tcg_constant_i64(0);
1895             tcg_gen_movcond_i64(TCG_COND_LT, reg, t0, t2, reg, t1);
1896         } else {
1897             /* Detect signed overflow for addition.  */
1898             tcg_gen_xor_i64(t0, reg, val);
1899             tcg_gen_add_i64(reg, reg, val);
1900             tcg_gen_xor_i64(t1, reg, val);
1901             tcg_gen_andc_i64(t0, t1, t0);
1902 
1903             /* Bound the result.  */
1904             tcg_gen_movi_i64(t1, INT64_MAX);
1905             t2 = tcg_constant_i64(0);
1906             tcg_gen_movcond_i64(TCG_COND_LT, reg, t0, t2, t1, reg);
1907         }
1908     }
1909 }
1910 
1911 /* Similarly with a vector and a scalar operand.  */
1912 static void do_sat_addsub_vec(DisasContext *s, int esz, int rd, int rn,
1913                               TCGv_i64 val, bool u, bool d)
1914 {
1915     unsigned vsz = vec_full_reg_size(s);
1916     TCGv_ptr dptr, nptr;
1917     TCGv_i32 t32, desc;
1918     TCGv_i64 t64;
1919 
1920     dptr = tcg_temp_new_ptr();
1921     nptr = tcg_temp_new_ptr();
1922     tcg_gen_addi_ptr(dptr, cpu_env, vec_full_reg_offset(s, rd));
1923     tcg_gen_addi_ptr(nptr, cpu_env, vec_full_reg_offset(s, rn));
1924     desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
1925 
1926     switch (esz) {
1927     case MO_8:
1928         t32 = tcg_temp_new_i32();
1929         tcg_gen_extrl_i64_i32(t32, val);
1930         if (d) {
1931             tcg_gen_neg_i32(t32, t32);
1932         }
1933         if (u) {
1934             gen_helper_sve_uqaddi_b(dptr, nptr, t32, desc);
1935         } else {
1936             gen_helper_sve_sqaddi_b(dptr, nptr, t32, desc);
1937         }
1938         break;
1939 
1940     case MO_16:
1941         t32 = tcg_temp_new_i32();
1942         tcg_gen_extrl_i64_i32(t32, val);
1943         if (d) {
1944             tcg_gen_neg_i32(t32, t32);
1945         }
1946         if (u) {
1947             gen_helper_sve_uqaddi_h(dptr, nptr, t32, desc);
1948         } else {
1949             gen_helper_sve_sqaddi_h(dptr, nptr, t32, desc);
1950         }
1951         break;
1952 
1953     case MO_32:
1954         t64 = tcg_temp_new_i64();
1955         if (d) {
1956             tcg_gen_neg_i64(t64, val);
1957         } else {
1958             tcg_gen_mov_i64(t64, val);
1959         }
1960         if (u) {
1961             gen_helper_sve_uqaddi_s(dptr, nptr, t64, desc);
1962         } else {
1963             gen_helper_sve_sqaddi_s(dptr, nptr, t64, desc);
1964         }
1965         break;
1966 
1967     case MO_64:
1968         if (u) {
1969             if (d) {
1970                 gen_helper_sve_uqsubi_d(dptr, nptr, val, desc);
1971             } else {
1972                 gen_helper_sve_uqaddi_d(dptr, nptr, val, desc);
1973             }
1974         } else if (d) {
1975             t64 = tcg_temp_new_i64();
1976             tcg_gen_neg_i64(t64, val);
1977             gen_helper_sve_sqaddi_d(dptr, nptr, t64, desc);
1978         } else {
1979             gen_helper_sve_sqaddi_d(dptr, nptr, val, desc);
1980         }
1981         break;
1982 
1983     default:
1984         g_assert_not_reached();
1985     }
1986 }
1987 
1988 static bool trans_CNT_r(DisasContext *s, arg_CNT_r *a)
1989 {
1990     if (!dc_isar_feature(aa64_sve, s)) {
1991         return false;
1992     }
1993     if (sve_access_check(s)) {
1994         unsigned fullsz = vec_full_reg_size(s);
1995         unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
1996         tcg_gen_movi_i64(cpu_reg(s, a->rd), numelem * a->imm);
1997     }
1998     return true;
1999 }
2000 
2001 static bool trans_INCDEC_r(DisasContext *s, arg_incdec_cnt *a)
2002 {
2003     if (!dc_isar_feature(aa64_sve, s)) {
2004         return false;
2005     }
2006     if (sve_access_check(s)) {
2007         unsigned fullsz = vec_full_reg_size(s);
2008         unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2009         int inc = numelem * a->imm * (a->d ? -1 : 1);
2010         TCGv_i64 reg = cpu_reg(s, a->rd);
2011 
2012         tcg_gen_addi_i64(reg, reg, inc);
2013     }
2014     return true;
2015 }
2016 
2017 static bool trans_SINCDEC_r_32(DisasContext *s, arg_incdec_cnt *a)
2018 {
2019     if (!dc_isar_feature(aa64_sve, s)) {
2020         return false;
2021     }
2022     if (!sve_access_check(s)) {
2023         return true;
2024     }
2025 
2026     unsigned fullsz = vec_full_reg_size(s);
2027     unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2028     int inc = numelem * a->imm;
2029     TCGv_i64 reg = cpu_reg(s, a->rd);
2030 
2031     /* Use normal 64-bit arithmetic to detect 32-bit overflow.  */
2032     if (inc == 0) {
2033         if (a->u) {
2034             tcg_gen_ext32u_i64(reg, reg);
2035         } else {
2036             tcg_gen_ext32s_i64(reg, reg);
2037         }
2038     } else {
2039         do_sat_addsub_32(reg, tcg_constant_i64(inc), a->u, a->d);
2040     }
2041     return true;
2042 }
2043 
2044 static bool trans_SINCDEC_r_64(DisasContext *s, arg_incdec_cnt *a)
2045 {
2046     if (!dc_isar_feature(aa64_sve, s)) {
2047         return false;
2048     }
2049     if (!sve_access_check(s)) {
2050         return true;
2051     }
2052 
2053     unsigned fullsz = vec_full_reg_size(s);
2054     unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2055     int inc = numelem * a->imm;
2056     TCGv_i64 reg = cpu_reg(s, a->rd);
2057 
2058     if (inc != 0) {
2059         do_sat_addsub_64(reg, tcg_constant_i64(inc), a->u, a->d);
2060     }
2061     return true;
2062 }
2063 
2064 static bool trans_INCDEC_v(DisasContext *s, arg_incdec2_cnt *a)
2065 {
2066     if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
2067         return false;
2068     }
2069 
2070     unsigned fullsz = vec_full_reg_size(s);
2071     unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2072     int inc = numelem * a->imm;
2073 
2074     if (inc != 0) {
2075         if (sve_access_check(s)) {
2076             tcg_gen_gvec_adds(a->esz, vec_full_reg_offset(s, a->rd),
2077                               vec_full_reg_offset(s, a->rn),
2078                               tcg_constant_i64(a->d ? -inc : inc),
2079                               fullsz, fullsz);
2080         }
2081     } else {
2082         do_mov_z(s, a->rd, a->rn);
2083     }
2084     return true;
2085 }
2086 
2087 static bool trans_SINCDEC_v(DisasContext *s, arg_incdec2_cnt *a)
2088 {
2089     if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
2090         return false;
2091     }
2092 
2093     unsigned fullsz = vec_full_reg_size(s);
2094     unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2095     int inc = numelem * a->imm;
2096 
2097     if (inc != 0) {
2098         if (sve_access_check(s)) {
2099             do_sat_addsub_vec(s, a->esz, a->rd, a->rn,
2100                               tcg_constant_i64(inc), a->u, a->d);
2101         }
2102     } else {
2103         do_mov_z(s, a->rd, a->rn);
2104     }
2105     return true;
2106 }
2107 
2108 /*
2109  *** SVE Bitwise Immediate Group
2110  */
2111 
2112 static bool do_zz_dbm(DisasContext *s, arg_rr_dbm *a, GVecGen2iFn *gvec_fn)
2113 {
2114     uint64_t imm;
2115     if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1),
2116                                 extract32(a->dbm, 0, 6),
2117                                 extract32(a->dbm, 6, 6))) {
2118         return false;
2119     }
2120     return gen_gvec_fn_zzi(s, gvec_fn, MO_64, a->rd, a->rn, imm);
2121 }
2122 
2123 TRANS_FEAT(AND_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_andi)
2124 TRANS_FEAT(ORR_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_ori)
2125 TRANS_FEAT(EOR_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_xori)
2126 
2127 static bool trans_DUPM(DisasContext *s, arg_DUPM *a)
2128 {
2129     uint64_t imm;
2130 
2131     if (!dc_isar_feature(aa64_sve, s)) {
2132         return false;
2133     }
2134     if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1),
2135                                 extract32(a->dbm, 0, 6),
2136                                 extract32(a->dbm, 6, 6))) {
2137         return false;
2138     }
2139     if (sve_access_check(s)) {
2140         do_dupi_z(s, a->rd, imm);
2141     }
2142     return true;
2143 }
2144 
2145 /*
2146  *** SVE Integer Wide Immediate - Predicated Group
2147  */
2148 
2149 /* Implement all merging copies.  This is used for CPY (immediate),
2150  * FCPY, CPY (scalar), CPY (SIMD&FP scalar).
2151  */
2152 static void do_cpy_m(DisasContext *s, int esz, int rd, int rn, int pg,
2153                      TCGv_i64 val)
2154 {
2155     typedef void gen_cpy(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32);
2156     static gen_cpy * const fns[4] = {
2157         gen_helper_sve_cpy_m_b, gen_helper_sve_cpy_m_h,
2158         gen_helper_sve_cpy_m_s, gen_helper_sve_cpy_m_d,
2159     };
2160     unsigned vsz = vec_full_reg_size(s);
2161     TCGv_i32 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
2162     TCGv_ptr t_zd = tcg_temp_new_ptr();
2163     TCGv_ptr t_zn = tcg_temp_new_ptr();
2164     TCGv_ptr t_pg = tcg_temp_new_ptr();
2165 
2166     tcg_gen_addi_ptr(t_zd, cpu_env, vec_full_reg_offset(s, rd));
2167     tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, rn));
2168     tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
2169 
2170     fns[esz](t_zd, t_zn, t_pg, val, desc);
2171 }
2172 
2173 static bool trans_FCPY(DisasContext *s, arg_FCPY *a)
2174 {
2175     if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
2176         return false;
2177     }
2178     if (sve_access_check(s)) {
2179         /* Decode the VFP immediate.  */
2180         uint64_t imm = vfp_expand_imm(a->esz, a->imm);
2181         do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(imm));
2182     }
2183     return true;
2184 }
2185 
2186 static bool trans_CPY_m_i(DisasContext *s, arg_rpri_esz *a)
2187 {
2188     if (!dc_isar_feature(aa64_sve, s)) {
2189         return false;
2190     }
2191     if (sve_access_check(s)) {
2192         do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(a->imm));
2193     }
2194     return true;
2195 }
2196 
2197 static bool trans_CPY_z_i(DisasContext *s, arg_CPY_z_i *a)
2198 {
2199     static gen_helper_gvec_2i * const fns[4] = {
2200         gen_helper_sve_cpy_z_b, gen_helper_sve_cpy_z_h,
2201         gen_helper_sve_cpy_z_s, gen_helper_sve_cpy_z_d,
2202     };
2203 
2204     if (!dc_isar_feature(aa64_sve, s)) {
2205         return false;
2206     }
2207     if (sve_access_check(s)) {
2208         unsigned vsz = vec_full_reg_size(s);
2209         tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd),
2210                             pred_full_reg_offset(s, a->pg),
2211                             tcg_constant_i64(a->imm),
2212                             vsz, vsz, 0, fns[a->esz]);
2213     }
2214     return true;
2215 }
2216 
2217 /*
2218  *** SVE Permute Extract Group
2219  */
2220 
2221 static bool do_EXT(DisasContext *s, int rd, int rn, int rm, int imm)
2222 {
2223     if (!sve_access_check(s)) {
2224         return true;
2225     }
2226 
2227     unsigned vsz = vec_full_reg_size(s);
2228     unsigned n_ofs = imm >= vsz ? 0 : imm;
2229     unsigned n_siz = vsz - n_ofs;
2230     unsigned d = vec_full_reg_offset(s, rd);
2231     unsigned n = vec_full_reg_offset(s, rn);
2232     unsigned m = vec_full_reg_offset(s, rm);
2233 
2234     /* Use host vector move insns if we have appropriate sizes
2235      * and no unfortunate overlap.
2236      */
2237     if (m != d
2238         && n_ofs == size_for_gvec(n_ofs)
2239         && n_siz == size_for_gvec(n_siz)
2240         && (d != n || n_siz <= n_ofs)) {
2241         tcg_gen_gvec_mov(0, d, n + n_ofs, n_siz, n_siz);
2242         if (n_ofs != 0) {
2243             tcg_gen_gvec_mov(0, d + n_siz, m, n_ofs, n_ofs);
2244         }
2245     } else {
2246         tcg_gen_gvec_3_ool(d, n, m, vsz, vsz, n_ofs, gen_helper_sve_ext);
2247     }
2248     return true;
2249 }
2250 
2251 TRANS_FEAT(EXT, aa64_sve, do_EXT, a->rd, a->rn, a->rm, a->imm)
2252 TRANS_FEAT(EXT_sve2, aa64_sve2, do_EXT, a->rd, a->rn, (a->rn + 1) % 32, a->imm)
2253 
2254 /*
2255  *** SVE Permute - Unpredicated Group
2256  */
2257 
2258 static bool trans_DUP_s(DisasContext *s, arg_DUP_s *a)
2259 {
2260     if (!dc_isar_feature(aa64_sve, s)) {
2261         return false;
2262     }
2263     if (sve_access_check(s)) {
2264         unsigned vsz = vec_full_reg_size(s);
2265         tcg_gen_gvec_dup_i64(a->esz, vec_full_reg_offset(s, a->rd),
2266                              vsz, vsz, cpu_reg_sp(s, a->rn));
2267     }
2268     return true;
2269 }
2270 
2271 static bool trans_DUP_x(DisasContext *s, arg_DUP_x *a)
2272 {
2273     if (!dc_isar_feature(aa64_sve, s)) {
2274         return false;
2275     }
2276     if ((a->imm & 0x1f) == 0) {
2277         return false;
2278     }
2279     if (sve_access_check(s)) {
2280         unsigned vsz = vec_full_reg_size(s);
2281         unsigned dofs = vec_full_reg_offset(s, a->rd);
2282         unsigned esz, index;
2283 
2284         esz = ctz32(a->imm);
2285         index = a->imm >> (esz + 1);
2286 
2287         if ((index << esz) < vsz) {
2288             unsigned nofs = vec_reg_offset(s, a->rn, index, esz);
2289             tcg_gen_gvec_dup_mem(esz, dofs, nofs, vsz, vsz);
2290         } else {
2291             /*
2292              * While dup_mem handles 128-bit elements, dup_imm does not.
2293              * Thankfully element size doesn't matter for splatting zero.
2294              */
2295             tcg_gen_gvec_dup_imm(MO_64, dofs, vsz, vsz, 0);
2296         }
2297     }
2298     return true;
2299 }
2300 
2301 static void do_insr_i64(DisasContext *s, arg_rrr_esz *a, TCGv_i64 val)
2302 {
2303     typedef void gen_insr(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32);
2304     static gen_insr * const fns[4] = {
2305         gen_helper_sve_insr_b, gen_helper_sve_insr_h,
2306         gen_helper_sve_insr_s, gen_helper_sve_insr_d,
2307     };
2308     unsigned vsz = vec_full_reg_size(s);
2309     TCGv_i32 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
2310     TCGv_ptr t_zd = tcg_temp_new_ptr();
2311     TCGv_ptr t_zn = tcg_temp_new_ptr();
2312 
2313     tcg_gen_addi_ptr(t_zd, cpu_env, vec_full_reg_offset(s, a->rd));
2314     tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, a->rn));
2315 
2316     fns[a->esz](t_zd, t_zn, val, desc);
2317 }
2318 
2319 static bool trans_INSR_f(DisasContext *s, arg_rrr_esz *a)
2320 {
2321     if (!dc_isar_feature(aa64_sve, s)) {
2322         return false;
2323     }
2324     if (sve_access_check(s)) {
2325         TCGv_i64 t = tcg_temp_new_i64();
2326         tcg_gen_ld_i64(t, cpu_env, vec_reg_offset(s, a->rm, 0, MO_64));
2327         do_insr_i64(s, a, t);
2328     }
2329     return true;
2330 }
2331 
2332 static bool trans_INSR_r(DisasContext *s, arg_rrr_esz *a)
2333 {
2334     if (!dc_isar_feature(aa64_sve, s)) {
2335         return false;
2336     }
2337     if (sve_access_check(s)) {
2338         do_insr_i64(s, a, cpu_reg(s, a->rm));
2339     }
2340     return true;
2341 }
2342 
2343 static gen_helper_gvec_2 * const rev_fns[4] = {
2344     gen_helper_sve_rev_b, gen_helper_sve_rev_h,
2345     gen_helper_sve_rev_s, gen_helper_sve_rev_d
2346 };
2347 TRANS_FEAT(REV_v, aa64_sve, gen_gvec_ool_zz, rev_fns[a->esz], a->rd, a->rn, 0)
2348 
2349 static gen_helper_gvec_3 * const sve_tbl_fns[4] = {
2350     gen_helper_sve_tbl_b, gen_helper_sve_tbl_h,
2351     gen_helper_sve_tbl_s, gen_helper_sve_tbl_d
2352 };
2353 TRANS_FEAT(TBL, aa64_sve, gen_gvec_ool_arg_zzz, sve_tbl_fns[a->esz], a, 0)
2354 
2355 static gen_helper_gvec_4 * const sve2_tbl_fns[4] = {
2356     gen_helper_sve2_tbl_b, gen_helper_sve2_tbl_h,
2357     gen_helper_sve2_tbl_s, gen_helper_sve2_tbl_d
2358 };
2359 TRANS_FEAT(TBL_sve2, aa64_sve2, gen_gvec_ool_zzzz, sve2_tbl_fns[a->esz],
2360            a->rd, a->rn, (a->rn + 1) % 32, a->rm, 0)
2361 
2362 static gen_helper_gvec_3 * const tbx_fns[4] = {
2363     gen_helper_sve2_tbx_b, gen_helper_sve2_tbx_h,
2364     gen_helper_sve2_tbx_s, gen_helper_sve2_tbx_d
2365 };
2366 TRANS_FEAT(TBX, aa64_sve2, gen_gvec_ool_arg_zzz, tbx_fns[a->esz], a, 0)
2367 
2368 static bool trans_UNPK(DisasContext *s, arg_UNPK *a)
2369 {
2370     static gen_helper_gvec_2 * const fns[4][2] = {
2371         { NULL, NULL },
2372         { gen_helper_sve_sunpk_h, gen_helper_sve_uunpk_h },
2373         { gen_helper_sve_sunpk_s, gen_helper_sve_uunpk_s },
2374         { gen_helper_sve_sunpk_d, gen_helper_sve_uunpk_d },
2375     };
2376 
2377     if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
2378         return false;
2379     }
2380     if (sve_access_check(s)) {
2381         unsigned vsz = vec_full_reg_size(s);
2382         tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->rd),
2383                            vec_full_reg_offset(s, a->rn)
2384                            + (a->h ? vsz / 2 : 0),
2385                            vsz, vsz, 0, fns[a->esz][a->u]);
2386     }
2387     return true;
2388 }
2389 
2390 /*
2391  *** SVE Permute - Predicates Group
2392  */
2393 
2394 static bool do_perm_pred3(DisasContext *s, arg_rrr_esz *a, bool high_odd,
2395                           gen_helper_gvec_3 *fn)
2396 {
2397     if (!sve_access_check(s)) {
2398         return true;
2399     }
2400 
2401     unsigned vsz = pred_full_reg_size(s);
2402 
2403     TCGv_ptr t_d = tcg_temp_new_ptr();
2404     TCGv_ptr t_n = tcg_temp_new_ptr();
2405     TCGv_ptr t_m = tcg_temp_new_ptr();
2406     uint32_t desc = 0;
2407 
2408     desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz);
2409     desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
2410     desc = FIELD_DP32(desc, PREDDESC, DATA, high_odd);
2411 
2412     tcg_gen_addi_ptr(t_d, cpu_env, pred_full_reg_offset(s, a->rd));
2413     tcg_gen_addi_ptr(t_n, cpu_env, pred_full_reg_offset(s, a->rn));
2414     tcg_gen_addi_ptr(t_m, cpu_env, pred_full_reg_offset(s, a->rm));
2415 
2416     fn(t_d, t_n, t_m, tcg_constant_i32(desc));
2417     return true;
2418 }
2419 
2420 static bool do_perm_pred2(DisasContext *s, arg_rr_esz *a, bool high_odd,
2421                           gen_helper_gvec_2 *fn)
2422 {
2423     if (!sve_access_check(s)) {
2424         return true;
2425     }
2426 
2427     unsigned vsz = pred_full_reg_size(s);
2428     TCGv_ptr t_d = tcg_temp_new_ptr();
2429     TCGv_ptr t_n = tcg_temp_new_ptr();
2430     uint32_t desc = 0;
2431 
2432     tcg_gen_addi_ptr(t_d, cpu_env, pred_full_reg_offset(s, a->rd));
2433     tcg_gen_addi_ptr(t_n, cpu_env, pred_full_reg_offset(s, a->rn));
2434 
2435     desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz);
2436     desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
2437     desc = FIELD_DP32(desc, PREDDESC, DATA, high_odd);
2438 
2439     fn(t_d, t_n, tcg_constant_i32(desc));
2440     return true;
2441 }
2442 
2443 TRANS_FEAT(ZIP1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_zip_p)
2444 TRANS_FEAT(ZIP2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_zip_p)
2445 TRANS_FEAT(UZP1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_uzp_p)
2446 TRANS_FEAT(UZP2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_uzp_p)
2447 TRANS_FEAT(TRN1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_trn_p)
2448 TRANS_FEAT(TRN2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_trn_p)
2449 
2450 TRANS_FEAT(REV_p, aa64_sve, do_perm_pred2, a, 0, gen_helper_sve_rev_p)
2451 TRANS_FEAT(PUNPKLO, aa64_sve, do_perm_pred2, a, 0, gen_helper_sve_punpk_p)
2452 TRANS_FEAT(PUNPKHI, aa64_sve, do_perm_pred2, a, 1, gen_helper_sve_punpk_p)
2453 
2454 /*
2455  *** SVE Permute - Interleaving Group
2456  */
2457 
2458 static gen_helper_gvec_3 * const zip_fns[4] = {
2459     gen_helper_sve_zip_b, gen_helper_sve_zip_h,
2460     gen_helper_sve_zip_s, gen_helper_sve_zip_d,
2461 };
2462 TRANS_FEAT(ZIP1_z, aa64_sve, gen_gvec_ool_arg_zzz,
2463            zip_fns[a->esz], a, 0)
2464 TRANS_FEAT(ZIP2_z, aa64_sve, gen_gvec_ool_arg_zzz,
2465            zip_fns[a->esz], a, vec_full_reg_size(s) / 2)
2466 
2467 TRANS_FEAT(ZIP1_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz,
2468            gen_helper_sve2_zip_q, a, 0)
2469 TRANS_FEAT(ZIP2_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz,
2470            gen_helper_sve2_zip_q, a,
2471            QEMU_ALIGN_DOWN(vec_full_reg_size(s), 32) / 2)
2472 
2473 static gen_helper_gvec_3 * const uzp_fns[4] = {
2474     gen_helper_sve_uzp_b, gen_helper_sve_uzp_h,
2475     gen_helper_sve_uzp_s, gen_helper_sve_uzp_d,
2476 };
2477 
2478 TRANS_FEAT(UZP1_z, aa64_sve, gen_gvec_ool_arg_zzz,
2479            uzp_fns[a->esz], a, 0)
2480 TRANS_FEAT(UZP2_z, aa64_sve, gen_gvec_ool_arg_zzz,
2481            uzp_fns[a->esz], a, 1 << a->esz)
2482 
2483 TRANS_FEAT(UZP1_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz,
2484            gen_helper_sve2_uzp_q, a, 0)
2485 TRANS_FEAT(UZP2_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz,
2486            gen_helper_sve2_uzp_q, a, 16)
2487 
2488 static gen_helper_gvec_3 * const trn_fns[4] = {
2489     gen_helper_sve_trn_b, gen_helper_sve_trn_h,
2490     gen_helper_sve_trn_s, gen_helper_sve_trn_d,
2491 };
2492 
2493 TRANS_FEAT(TRN1_z, aa64_sve, gen_gvec_ool_arg_zzz,
2494            trn_fns[a->esz], a, 0)
2495 TRANS_FEAT(TRN2_z, aa64_sve, gen_gvec_ool_arg_zzz,
2496            trn_fns[a->esz], a, 1 << a->esz)
2497 
2498 TRANS_FEAT(TRN1_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz,
2499            gen_helper_sve2_trn_q, a, 0)
2500 TRANS_FEAT(TRN2_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz,
2501            gen_helper_sve2_trn_q, a, 16)
2502 
2503 /*
2504  *** SVE Permute Vector - Predicated Group
2505  */
2506 
2507 static gen_helper_gvec_3 * const compact_fns[4] = {
2508     NULL, NULL, gen_helper_sve_compact_s, gen_helper_sve_compact_d
2509 };
2510 TRANS_FEAT_NONSTREAMING(COMPACT, aa64_sve, gen_gvec_ool_arg_zpz,
2511                         compact_fns[a->esz], a, 0)
2512 
2513 /* Call the helper that computes the ARM LastActiveElement pseudocode
2514  * function, scaled by the element size.  This includes the not found
2515  * indication; e.g. not found for esz=3 is -8.
2516  */
2517 static void find_last_active(DisasContext *s, TCGv_i32 ret, int esz, int pg)
2518 {
2519     /* Predicate sizes may be smaller and cannot use simd_desc.  We cannot
2520      * round up, as we do elsewhere, because we need the exact size.
2521      */
2522     TCGv_ptr t_p = tcg_temp_new_ptr();
2523     unsigned desc = 0;
2524 
2525     desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pred_full_reg_size(s));
2526     desc = FIELD_DP32(desc, PREDDESC, ESZ, esz);
2527 
2528     tcg_gen_addi_ptr(t_p, cpu_env, pred_full_reg_offset(s, pg));
2529 
2530     gen_helper_sve_last_active_element(ret, t_p, tcg_constant_i32(desc));
2531 }
2532 
2533 /* Increment LAST to the offset of the next element in the vector,
2534  * wrapping around to 0.
2535  */
2536 static void incr_last_active(DisasContext *s, TCGv_i32 last, int esz)
2537 {
2538     unsigned vsz = vec_full_reg_size(s);
2539 
2540     tcg_gen_addi_i32(last, last, 1 << esz);
2541     if (is_power_of_2(vsz)) {
2542         tcg_gen_andi_i32(last, last, vsz - 1);
2543     } else {
2544         TCGv_i32 max = tcg_constant_i32(vsz);
2545         TCGv_i32 zero = tcg_constant_i32(0);
2546         tcg_gen_movcond_i32(TCG_COND_GEU, last, last, max, zero, last);
2547     }
2548 }
2549 
2550 /* If LAST < 0, set LAST to the offset of the last element in the vector.  */
2551 static void wrap_last_active(DisasContext *s, TCGv_i32 last, int esz)
2552 {
2553     unsigned vsz = vec_full_reg_size(s);
2554 
2555     if (is_power_of_2(vsz)) {
2556         tcg_gen_andi_i32(last, last, vsz - 1);
2557     } else {
2558         TCGv_i32 max = tcg_constant_i32(vsz - (1 << esz));
2559         TCGv_i32 zero = tcg_constant_i32(0);
2560         tcg_gen_movcond_i32(TCG_COND_LT, last, last, zero, max, last);
2561     }
2562 }
2563 
2564 /* Load an unsigned element of ESZ from BASE+OFS.  */
2565 static TCGv_i64 load_esz(TCGv_ptr base, int ofs, int esz)
2566 {
2567     TCGv_i64 r = tcg_temp_new_i64();
2568 
2569     switch (esz) {
2570     case 0:
2571         tcg_gen_ld8u_i64(r, base, ofs);
2572         break;
2573     case 1:
2574         tcg_gen_ld16u_i64(r, base, ofs);
2575         break;
2576     case 2:
2577         tcg_gen_ld32u_i64(r, base, ofs);
2578         break;
2579     case 3:
2580         tcg_gen_ld_i64(r, base, ofs);
2581         break;
2582     default:
2583         g_assert_not_reached();
2584     }
2585     return r;
2586 }
2587 
2588 /* Load an unsigned element of ESZ from RM[LAST].  */
2589 static TCGv_i64 load_last_active(DisasContext *s, TCGv_i32 last,
2590                                  int rm, int esz)
2591 {
2592     TCGv_ptr p = tcg_temp_new_ptr();
2593 
2594     /* Convert offset into vector into offset into ENV.
2595      * The final adjustment for the vector register base
2596      * is added via constant offset to the load.
2597      */
2598 #if HOST_BIG_ENDIAN
2599     /* Adjust for element ordering.  See vec_reg_offset.  */
2600     if (esz < 3) {
2601         tcg_gen_xori_i32(last, last, 8 - (1 << esz));
2602     }
2603 #endif
2604     tcg_gen_ext_i32_ptr(p, last);
2605     tcg_gen_add_ptr(p, p, cpu_env);
2606 
2607     return load_esz(p, vec_full_reg_offset(s, rm), esz);
2608 }
2609 
2610 /* Compute CLAST for a Zreg.  */
2611 static bool do_clast_vector(DisasContext *s, arg_rprr_esz *a, bool before)
2612 {
2613     TCGv_i32 last;
2614     TCGLabel *over;
2615     TCGv_i64 ele;
2616     unsigned vsz, esz = a->esz;
2617 
2618     if (!sve_access_check(s)) {
2619         return true;
2620     }
2621 
2622     last = tcg_temp_new_i32();
2623     over = gen_new_label();
2624 
2625     find_last_active(s, last, esz, a->pg);
2626 
2627     /* There is of course no movcond for a 2048-bit vector,
2628      * so we must branch over the actual store.
2629      */
2630     tcg_gen_brcondi_i32(TCG_COND_LT, last, 0, over);
2631 
2632     if (!before) {
2633         incr_last_active(s, last, esz);
2634     }
2635 
2636     ele = load_last_active(s, last, a->rm, esz);
2637 
2638     vsz = vec_full_reg_size(s);
2639     tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), vsz, vsz, ele);
2640 
2641     /* If this insn used MOVPRFX, we may need a second move.  */
2642     if (a->rd != a->rn) {
2643         TCGLabel *done = gen_new_label();
2644         tcg_gen_br(done);
2645 
2646         gen_set_label(over);
2647         do_mov_z(s, a->rd, a->rn);
2648 
2649         gen_set_label(done);
2650     } else {
2651         gen_set_label(over);
2652     }
2653     return true;
2654 }
2655 
2656 TRANS_FEAT(CLASTA_z, aa64_sve, do_clast_vector, a, false)
2657 TRANS_FEAT(CLASTB_z, aa64_sve, do_clast_vector, a, true)
2658 
2659 /* Compute CLAST for a scalar.  */
2660 static void do_clast_scalar(DisasContext *s, int esz, int pg, int rm,
2661                             bool before, TCGv_i64 reg_val)
2662 {
2663     TCGv_i32 last = tcg_temp_new_i32();
2664     TCGv_i64 ele, cmp;
2665 
2666     find_last_active(s, last, esz, pg);
2667 
2668     /* Extend the original value of last prior to incrementing.  */
2669     cmp = tcg_temp_new_i64();
2670     tcg_gen_ext_i32_i64(cmp, last);
2671 
2672     if (!before) {
2673         incr_last_active(s, last, esz);
2674     }
2675 
2676     /* The conceit here is that while last < 0 indicates not found, after
2677      * adjusting for cpu_env->vfp.zregs[rm], it is still a valid address
2678      * from which we can load garbage.  We then discard the garbage with
2679      * a conditional move.
2680      */
2681     ele = load_last_active(s, last, rm, esz);
2682 
2683     tcg_gen_movcond_i64(TCG_COND_GE, reg_val, cmp, tcg_constant_i64(0),
2684                         ele, reg_val);
2685 }
2686 
2687 /* Compute CLAST for a Vreg.  */
2688 static bool do_clast_fp(DisasContext *s, arg_rpr_esz *a, bool before)
2689 {
2690     if (sve_access_check(s)) {
2691         int esz = a->esz;
2692         int ofs = vec_reg_offset(s, a->rd, 0, esz);
2693         TCGv_i64 reg = load_esz(cpu_env, ofs, esz);
2694 
2695         do_clast_scalar(s, esz, a->pg, a->rn, before, reg);
2696         write_fp_dreg(s, a->rd, reg);
2697     }
2698     return true;
2699 }
2700 
2701 TRANS_FEAT(CLASTA_v, aa64_sve, do_clast_fp, a, false)
2702 TRANS_FEAT(CLASTB_v, aa64_sve, do_clast_fp, a, true)
2703 
2704 /* Compute CLAST for a Xreg.  */
2705 static bool do_clast_general(DisasContext *s, arg_rpr_esz *a, bool before)
2706 {
2707     TCGv_i64 reg;
2708 
2709     if (!sve_access_check(s)) {
2710         return true;
2711     }
2712 
2713     reg = cpu_reg(s, a->rd);
2714     switch (a->esz) {
2715     case 0:
2716         tcg_gen_ext8u_i64(reg, reg);
2717         break;
2718     case 1:
2719         tcg_gen_ext16u_i64(reg, reg);
2720         break;
2721     case 2:
2722         tcg_gen_ext32u_i64(reg, reg);
2723         break;
2724     case 3:
2725         break;
2726     default:
2727         g_assert_not_reached();
2728     }
2729 
2730     do_clast_scalar(s, a->esz, a->pg, a->rn, before, reg);
2731     return true;
2732 }
2733 
2734 TRANS_FEAT(CLASTA_r, aa64_sve, do_clast_general, a, false)
2735 TRANS_FEAT(CLASTB_r, aa64_sve, do_clast_general, a, true)
2736 
2737 /* Compute LAST for a scalar.  */
2738 static TCGv_i64 do_last_scalar(DisasContext *s, int esz,
2739                                int pg, int rm, bool before)
2740 {
2741     TCGv_i32 last = tcg_temp_new_i32();
2742 
2743     find_last_active(s, last, esz, pg);
2744     if (before) {
2745         wrap_last_active(s, last, esz);
2746     } else {
2747         incr_last_active(s, last, esz);
2748     }
2749 
2750     return load_last_active(s, last, rm, esz);
2751 }
2752 
2753 /* Compute LAST for a Vreg.  */
2754 static bool do_last_fp(DisasContext *s, arg_rpr_esz *a, bool before)
2755 {
2756     if (sve_access_check(s)) {
2757         TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before);
2758         write_fp_dreg(s, a->rd, val);
2759     }
2760     return true;
2761 }
2762 
2763 TRANS_FEAT(LASTA_v, aa64_sve, do_last_fp, a, false)
2764 TRANS_FEAT(LASTB_v, aa64_sve, do_last_fp, a, true)
2765 
2766 /* Compute LAST for a Xreg.  */
2767 static bool do_last_general(DisasContext *s, arg_rpr_esz *a, bool before)
2768 {
2769     if (sve_access_check(s)) {
2770         TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before);
2771         tcg_gen_mov_i64(cpu_reg(s, a->rd), val);
2772     }
2773     return true;
2774 }
2775 
2776 TRANS_FEAT(LASTA_r, aa64_sve, do_last_general, a, false)
2777 TRANS_FEAT(LASTB_r, aa64_sve, do_last_general, a, true)
2778 
2779 static bool trans_CPY_m_r(DisasContext *s, arg_rpr_esz *a)
2780 {
2781     if (!dc_isar_feature(aa64_sve, s)) {
2782         return false;
2783     }
2784     if (sve_access_check(s)) {
2785         do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, cpu_reg_sp(s, a->rn));
2786     }
2787     return true;
2788 }
2789 
2790 static bool trans_CPY_m_v(DisasContext *s, arg_rpr_esz *a)
2791 {
2792     if (!dc_isar_feature(aa64_sve, s)) {
2793         return false;
2794     }
2795     if (sve_access_check(s)) {
2796         int ofs = vec_reg_offset(s, a->rn, 0, a->esz);
2797         TCGv_i64 t = load_esz(cpu_env, ofs, a->esz);
2798         do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, t);
2799     }
2800     return true;
2801 }
2802 
2803 static gen_helper_gvec_3 * const revb_fns[4] = {
2804     NULL,                  gen_helper_sve_revb_h,
2805     gen_helper_sve_revb_s, gen_helper_sve_revb_d,
2806 };
2807 TRANS_FEAT(REVB, aa64_sve, gen_gvec_ool_arg_zpz, revb_fns[a->esz], a, 0)
2808 
2809 static gen_helper_gvec_3 * const revh_fns[4] = {
2810     NULL, NULL, gen_helper_sve_revh_s, gen_helper_sve_revh_d,
2811 };
2812 TRANS_FEAT(REVH, aa64_sve, gen_gvec_ool_arg_zpz, revh_fns[a->esz], a, 0)
2813 
2814 TRANS_FEAT(REVW, aa64_sve, gen_gvec_ool_arg_zpz,
2815            a->esz == 3 ? gen_helper_sve_revw_d : NULL, a, 0)
2816 
2817 TRANS_FEAT(REVD, aa64_sme, gen_gvec_ool_arg_zpz, gen_helper_sme_revd_q, a, 0)
2818 
2819 TRANS_FEAT(SPLICE, aa64_sve, gen_gvec_ool_arg_zpzz,
2820            gen_helper_sve_splice, a, a->esz)
2821 
2822 TRANS_FEAT(SPLICE_sve2, aa64_sve2, gen_gvec_ool_zzzp, gen_helper_sve_splice,
2823            a->rd, a->rn, (a->rn + 1) % 32, a->pg, a->esz)
2824 
2825 /*
2826  *** SVE Integer Compare - Vectors Group
2827  */
2828 
2829 static bool do_ppzz_flags(DisasContext *s, arg_rprr_esz *a,
2830                           gen_helper_gvec_flags_4 *gen_fn)
2831 {
2832     TCGv_ptr pd, zn, zm, pg;
2833     unsigned vsz;
2834     TCGv_i32 t;
2835 
2836     if (gen_fn == NULL) {
2837         return false;
2838     }
2839     if (!sve_access_check(s)) {
2840         return true;
2841     }
2842 
2843     vsz = vec_full_reg_size(s);
2844     t = tcg_temp_new_i32();
2845     pd = tcg_temp_new_ptr();
2846     zn = tcg_temp_new_ptr();
2847     zm = tcg_temp_new_ptr();
2848     pg = tcg_temp_new_ptr();
2849 
2850     tcg_gen_addi_ptr(pd, cpu_env, pred_full_reg_offset(s, a->rd));
2851     tcg_gen_addi_ptr(zn, cpu_env, vec_full_reg_offset(s, a->rn));
2852     tcg_gen_addi_ptr(zm, cpu_env, vec_full_reg_offset(s, a->rm));
2853     tcg_gen_addi_ptr(pg, cpu_env, pred_full_reg_offset(s, a->pg));
2854 
2855     gen_fn(t, pd, zn, zm, pg, tcg_constant_i32(simd_desc(vsz, vsz, 0)));
2856 
2857     do_pred_flags(t);
2858     return true;
2859 }
2860 
2861 #define DO_PPZZ(NAME, name) \
2862     static gen_helper_gvec_flags_4 * const name##_ppzz_fns[4] = {       \
2863         gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
2864         gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
2865     };                                                                  \
2866     TRANS_FEAT(NAME##_ppzz, aa64_sve, do_ppzz_flags,                    \
2867                a, name##_ppzz_fns[a->esz])
2868 
2869 DO_PPZZ(CMPEQ, cmpeq)
2870 DO_PPZZ(CMPNE, cmpne)
2871 DO_PPZZ(CMPGT, cmpgt)
2872 DO_PPZZ(CMPGE, cmpge)
2873 DO_PPZZ(CMPHI, cmphi)
2874 DO_PPZZ(CMPHS, cmphs)
2875 
2876 #undef DO_PPZZ
2877 
2878 #define DO_PPZW(NAME, name) \
2879     static gen_helper_gvec_flags_4 * const name##_ppzw_fns[4] = {       \
2880         gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
2881         gen_helper_sve_##name##_ppzw_s, NULL                            \
2882     };                                                                  \
2883     TRANS_FEAT(NAME##_ppzw, aa64_sve, do_ppzz_flags,                    \
2884                a, name##_ppzw_fns[a->esz])
2885 
2886 DO_PPZW(CMPEQ, cmpeq)
2887 DO_PPZW(CMPNE, cmpne)
2888 DO_PPZW(CMPGT, cmpgt)
2889 DO_PPZW(CMPGE, cmpge)
2890 DO_PPZW(CMPHI, cmphi)
2891 DO_PPZW(CMPHS, cmphs)
2892 DO_PPZW(CMPLT, cmplt)
2893 DO_PPZW(CMPLE, cmple)
2894 DO_PPZW(CMPLO, cmplo)
2895 DO_PPZW(CMPLS, cmpls)
2896 
2897 #undef DO_PPZW
2898 
2899 /*
2900  *** SVE Integer Compare - Immediate Groups
2901  */
2902 
2903 static bool do_ppzi_flags(DisasContext *s, arg_rpri_esz *a,
2904                           gen_helper_gvec_flags_3 *gen_fn)
2905 {
2906     TCGv_ptr pd, zn, pg;
2907     unsigned vsz;
2908     TCGv_i32 t;
2909 
2910     if (gen_fn == NULL) {
2911         return false;
2912     }
2913     if (!sve_access_check(s)) {
2914         return true;
2915     }
2916 
2917     vsz = vec_full_reg_size(s);
2918     t = tcg_temp_new_i32();
2919     pd = tcg_temp_new_ptr();
2920     zn = tcg_temp_new_ptr();
2921     pg = tcg_temp_new_ptr();
2922 
2923     tcg_gen_addi_ptr(pd, cpu_env, pred_full_reg_offset(s, a->rd));
2924     tcg_gen_addi_ptr(zn, cpu_env, vec_full_reg_offset(s, a->rn));
2925     tcg_gen_addi_ptr(pg, cpu_env, pred_full_reg_offset(s, a->pg));
2926 
2927     gen_fn(t, pd, zn, pg, tcg_constant_i32(simd_desc(vsz, vsz, a->imm)));
2928 
2929     do_pred_flags(t);
2930     return true;
2931 }
2932 
2933 #define DO_PPZI(NAME, name) \
2934     static gen_helper_gvec_flags_3 * const name##_ppzi_fns[4] = {         \
2935         gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h,   \
2936         gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d,   \
2937     };                                                                    \
2938     TRANS_FEAT(NAME##_ppzi, aa64_sve, do_ppzi_flags, a,                   \
2939                name##_ppzi_fns[a->esz])
2940 
2941 DO_PPZI(CMPEQ, cmpeq)
2942 DO_PPZI(CMPNE, cmpne)
2943 DO_PPZI(CMPGT, cmpgt)
2944 DO_PPZI(CMPGE, cmpge)
2945 DO_PPZI(CMPHI, cmphi)
2946 DO_PPZI(CMPHS, cmphs)
2947 DO_PPZI(CMPLT, cmplt)
2948 DO_PPZI(CMPLE, cmple)
2949 DO_PPZI(CMPLO, cmplo)
2950 DO_PPZI(CMPLS, cmpls)
2951 
2952 #undef DO_PPZI
2953 
2954 /*
2955  *** SVE Partition Break Group
2956  */
2957 
2958 static bool do_brk3(DisasContext *s, arg_rprr_s *a,
2959                     gen_helper_gvec_4 *fn, gen_helper_gvec_flags_4 *fn_s)
2960 {
2961     if (!sve_access_check(s)) {
2962         return true;
2963     }
2964 
2965     unsigned vsz = pred_full_reg_size(s);
2966 
2967     /* Predicate sizes may be smaller and cannot use simd_desc.  */
2968     TCGv_ptr d = tcg_temp_new_ptr();
2969     TCGv_ptr n = tcg_temp_new_ptr();
2970     TCGv_ptr m = tcg_temp_new_ptr();
2971     TCGv_ptr g = tcg_temp_new_ptr();
2972     TCGv_i32 desc = tcg_constant_i32(FIELD_DP32(0, PREDDESC, OPRSZ, vsz));
2973 
2974     tcg_gen_addi_ptr(d, cpu_env, pred_full_reg_offset(s, a->rd));
2975     tcg_gen_addi_ptr(n, cpu_env, pred_full_reg_offset(s, a->rn));
2976     tcg_gen_addi_ptr(m, cpu_env, pred_full_reg_offset(s, a->rm));
2977     tcg_gen_addi_ptr(g, cpu_env, pred_full_reg_offset(s, a->pg));
2978 
2979     if (a->s) {
2980         TCGv_i32 t = tcg_temp_new_i32();
2981         fn_s(t, d, n, m, g, desc);
2982         do_pred_flags(t);
2983     } else {
2984         fn(d, n, m, g, desc);
2985     }
2986     return true;
2987 }
2988 
2989 static bool do_brk2(DisasContext *s, arg_rpr_s *a,
2990                     gen_helper_gvec_3 *fn, gen_helper_gvec_flags_3 *fn_s)
2991 {
2992     if (!sve_access_check(s)) {
2993         return true;
2994     }
2995 
2996     unsigned vsz = pred_full_reg_size(s);
2997 
2998     /* Predicate sizes may be smaller and cannot use simd_desc.  */
2999     TCGv_ptr d = tcg_temp_new_ptr();
3000     TCGv_ptr n = tcg_temp_new_ptr();
3001     TCGv_ptr g = tcg_temp_new_ptr();
3002     TCGv_i32 desc = tcg_constant_i32(FIELD_DP32(0, PREDDESC, OPRSZ, vsz));
3003 
3004     tcg_gen_addi_ptr(d, cpu_env, pred_full_reg_offset(s, a->rd));
3005     tcg_gen_addi_ptr(n, cpu_env, pred_full_reg_offset(s, a->rn));
3006     tcg_gen_addi_ptr(g, cpu_env, pred_full_reg_offset(s, a->pg));
3007 
3008     if (a->s) {
3009         TCGv_i32 t = tcg_temp_new_i32();
3010         fn_s(t, d, n, g, desc);
3011         do_pred_flags(t);
3012     } else {
3013         fn(d, n, g, desc);
3014     }
3015     return true;
3016 }
3017 
3018 TRANS_FEAT(BRKPA, aa64_sve, do_brk3, a,
3019            gen_helper_sve_brkpa, gen_helper_sve_brkpas)
3020 TRANS_FEAT(BRKPB, aa64_sve, do_brk3, a,
3021            gen_helper_sve_brkpb, gen_helper_sve_brkpbs)
3022 
3023 TRANS_FEAT(BRKA_m, aa64_sve, do_brk2, a,
3024            gen_helper_sve_brka_m, gen_helper_sve_brkas_m)
3025 TRANS_FEAT(BRKB_m, aa64_sve, do_brk2, a,
3026            gen_helper_sve_brkb_m, gen_helper_sve_brkbs_m)
3027 
3028 TRANS_FEAT(BRKA_z, aa64_sve, do_brk2, a,
3029            gen_helper_sve_brka_z, gen_helper_sve_brkas_z)
3030 TRANS_FEAT(BRKB_z, aa64_sve, do_brk2, a,
3031            gen_helper_sve_brkb_z, gen_helper_sve_brkbs_z)
3032 
3033 TRANS_FEAT(BRKN, aa64_sve, do_brk2, a,
3034            gen_helper_sve_brkn, gen_helper_sve_brkns)
3035 
3036 /*
3037  *** SVE Predicate Count Group
3038  */
3039 
3040 static void do_cntp(DisasContext *s, TCGv_i64 val, int esz, int pn, int pg)
3041 {
3042     unsigned psz = pred_full_reg_size(s);
3043 
3044     if (psz <= 8) {
3045         uint64_t psz_mask;
3046 
3047         tcg_gen_ld_i64(val, cpu_env, pred_full_reg_offset(s, pn));
3048         if (pn != pg) {
3049             TCGv_i64 g = tcg_temp_new_i64();
3050             tcg_gen_ld_i64(g, cpu_env, pred_full_reg_offset(s, pg));
3051             tcg_gen_and_i64(val, val, g);
3052         }
3053 
3054         /* Reduce the pred_esz_masks value simply to reduce the
3055          * size of the code generated here.
3056          */
3057         psz_mask = MAKE_64BIT_MASK(0, psz * 8);
3058         tcg_gen_andi_i64(val, val, pred_esz_masks[esz] & psz_mask);
3059 
3060         tcg_gen_ctpop_i64(val, val);
3061     } else {
3062         TCGv_ptr t_pn = tcg_temp_new_ptr();
3063         TCGv_ptr t_pg = tcg_temp_new_ptr();
3064         unsigned desc = 0;
3065 
3066         desc = FIELD_DP32(desc, PREDDESC, OPRSZ, psz);
3067         desc = FIELD_DP32(desc, PREDDESC, ESZ, esz);
3068 
3069         tcg_gen_addi_ptr(t_pn, cpu_env, pred_full_reg_offset(s, pn));
3070         tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
3071 
3072         gen_helper_sve_cntp(val, t_pn, t_pg, tcg_constant_i32(desc));
3073     }
3074 }
3075 
3076 static bool trans_CNTP(DisasContext *s, arg_CNTP *a)
3077 {
3078     if (!dc_isar_feature(aa64_sve, s)) {
3079         return false;
3080     }
3081     if (sve_access_check(s)) {
3082         do_cntp(s, cpu_reg(s, a->rd), a->esz, a->rn, a->pg);
3083     }
3084     return true;
3085 }
3086 
3087 static bool trans_INCDECP_r(DisasContext *s, arg_incdec_pred *a)
3088 {
3089     if (!dc_isar_feature(aa64_sve, s)) {
3090         return false;
3091     }
3092     if (sve_access_check(s)) {
3093         TCGv_i64 reg = cpu_reg(s, a->rd);
3094         TCGv_i64 val = tcg_temp_new_i64();
3095 
3096         do_cntp(s, val, a->esz, a->pg, a->pg);
3097         if (a->d) {
3098             tcg_gen_sub_i64(reg, reg, val);
3099         } else {
3100             tcg_gen_add_i64(reg, reg, val);
3101         }
3102     }
3103     return true;
3104 }
3105 
3106 static bool trans_INCDECP_z(DisasContext *s, arg_incdec2_pred *a)
3107 {
3108     if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
3109         return false;
3110     }
3111     if (sve_access_check(s)) {
3112         unsigned vsz = vec_full_reg_size(s);
3113         TCGv_i64 val = tcg_temp_new_i64();
3114         GVecGen2sFn *gvec_fn = a->d ? tcg_gen_gvec_subs : tcg_gen_gvec_adds;
3115 
3116         do_cntp(s, val, a->esz, a->pg, a->pg);
3117         gvec_fn(a->esz, vec_full_reg_offset(s, a->rd),
3118                 vec_full_reg_offset(s, a->rn), val, vsz, vsz);
3119     }
3120     return true;
3121 }
3122 
3123 static bool trans_SINCDECP_r_32(DisasContext *s, arg_incdec_pred *a)
3124 {
3125     if (!dc_isar_feature(aa64_sve, s)) {
3126         return false;
3127     }
3128     if (sve_access_check(s)) {
3129         TCGv_i64 reg = cpu_reg(s, a->rd);
3130         TCGv_i64 val = tcg_temp_new_i64();
3131 
3132         do_cntp(s, val, a->esz, a->pg, a->pg);
3133         do_sat_addsub_32(reg, val, a->u, a->d);
3134     }
3135     return true;
3136 }
3137 
3138 static bool trans_SINCDECP_r_64(DisasContext *s, arg_incdec_pred *a)
3139 {
3140     if (!dc_isar_feature(aa64_sve, s)) {
3141         return false;
3142     }
3143     if (sve_access_check(s)) {
3144         TCGv_i64 reg = cpu_reg(s, a->rd);
3145         TCGv_i64 val = tcg_temp_new_i64();
3146 
3147         do_cntp(s, val, a->esz, a->pg, a->pg);
3148         do_sat_addsub_64(reg, val, a->u, a->d);
3149     }
3150     return true;
3151 }
3152 
3153 static bool trans_SINCDECP_z(DisasContext *s, arg_incdec2_pred *a)
3154 {
3155     if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
3156         return false;
3157     }
3158     if (sve_access_check(s)) {
3159         TCGv_i64 val = tcg_temp_new_i64();
3160         do_cntp(s, val, a->esz, a->pg, a->pg);
3161         do_sat_addsub_vec(s, a->esz, a->rd, a->rn, val, a->u, a->d);
3162     }
3163     return true;
3164 }
3165 
3166 /*
3167  *** SVE Integer Compare Scalars Group
3168  */
3169 
3170 static bool trans_CTERM(DisasContext *s, arg_CTERM *a)
3171 {
3172     if (!dc_isar_feature(aa64_sve, s)) {
3173         return false;
3174     }
3175     if (!sve_access_check(s)) {
3176         return true;
3177     }
3178 
3179     TCGCond cond = (a->ne ? TCG_COND_NE : TCG_COND_EQ);
3180     TCGv_i64 rn = read_cpu_reg(s, a->rn, a->sf);
3181     TCGv_i64 rm = read_cpu_reg(s, a->rm, a->sf);
3182     TCGv_i64 cmp = tcg_temp_new_i64();
3183 
3184     tcg_gen_setcond_i64(cond, cmp, rn, rm);
3185     tcg_gen_extrl_i64_i32(cpu_NF, cmp);
3186 
3187     /* VF = !NF & !CF.  */
3188     tcg_gen_xori_i32(cpu_VF, cpu_NF, 1);
3189     tcg_gen_andc_i32(cpu_VF, cpu_VF, cpu_CF);
3190 
3191     /* Both NF and VF actually look at bit 31.  */
3192     tcg_gen_neg_i32(cpu_NF, cpu_NF);
3193     tcg_gen_neg_i32(cpu_VF, cpu_VF);
3194     return true;
3195 }
3196 
3197 static bool trans_WHILE(DisasContext *s, arg_WHILE *a)
3198 {
3199     TCGv_i64 op0, op1, t0, t1, tmax;
3200     TCGv_i32 t2;
3201     TCGv_ptr ptr;
3202     unsigned vsz = vec_full_reg_size(s);
3203     unsigned desc = 0;
3204     TCGCond cond;
3205     uint64_t maxval;
3206     /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
3207     bool eq = a->eq == a->lt;
3208 
3209     /* The greater-than conditions are all SVE2. */
3210     if (a->lt
3211         ? !dc_isar_feature(aa64_sve, s)
3212         : !dc_isar_feature(aa64_sve2, s)) {
3213         return false;
3214     }
3215     if (!sve_access_check(s)) {
3216         return true;
3217     }
3218 
3219     op0 = read_cpu_reg(s, a->rn, 1);
3220     op1 = read_cpu_reg(s, a->rm, 1);
3221 
3222     if (!a->sf) {
3223         if (a->u) {
3224             tcg_gen_ext32u_i64(op0, op0);
3225             tcg_gen_ext32u_i64(op1, op1);
3226         } else {
3227             tcg_gen_ext32s_i64(op0, op0);
3228             tcg_gen_ext32s_i64(op1, op1);
3229         }
3230     }
3231 
3232     /* For the helper, compress the different conditions into a computation
3233      * of how many iterations for which the condition is true.
3234      */
3235     t0 = tcg_temp_new_i64();
3236     t1 = tcg_temp_new_i64();
3237 
3238     if (a->lt) {
3239         tcg_gen_sub_i64(t0, op1, op0);
3240         if (a->u) {
3241             maxval = a->sf ? UINT64_MAX : UINT32_MAX;
3242             cond = eq ? TCG_COND_LEU : TCG_COND_LTU;
3243         } else {
3244             maxval = a->sf ? INT64_MAX : INT32_MAX;
3245             cond = eq ? TCG_COND_LE : TCG_COND_LT;
3246         }
3247     } else {
3248         tcg_gen_sub_i64(t0, op0, op1);
3249         if (a->u) {
3250             maxval = 0;
3251             cond = eq ? TCG_COND_GEU : TCG_COND_GTU;
3252         } else {
3253             maxval = a->sf ? INT64_MIN : INT32_MIN;
3254             cond = eq ? TCG_COND_GE : TCG_COND_GT;
3255         }
3256     }
3257 
3258     tmax = tcg_constant_i64(vsz >> a->esz);
3259     if (eq) {
3260         /* Equality means one more iteration.  */
3261         tcg_gen_addi_i64(t0, t0, 1);
3262 
3263         /*
3264          * For the less-than while, if op1 is maxval (and the only time
3265          * the addition above could overflow), then we produce an all-true
3266          * predicate by setting the count to the vector length.  This is
3267          * because the pseudocode is described as an increment + compare
3268          * loop, and the maximum integer would always compare true.
3269          * Similarly, the greater-than while has the same issue with the
3270          * minimum integer due to the decrement + compare loop.
3271          */
3272         tcg_gen_movi_i64(t1, maxval);
3273         tcg_gen_movcond_i64(TCG_COND_EQ, t0, op1, t1, tmax, t0);
3274     }
3275 
3276     /* Bound to the maximum.  */
3277     tcg_gen_umin_i64(t0, t0, tmax);
3278 
3279     /* Set the count to zero if the condition is false.  */
3280     tcg_gen_movi_i64(t1, 0);
3281     tcg_gen_movcond_i64(cond, t0, op0, op1, t0, t1);
3282 
3283     /* Since we're bounded, pass as a 32-bit type.  */
3284     t2 = tcg_temp_new_i32();
3285     tcg_gen_extrl_i64_i32(t2, t0);
3286 
3287     /* Scale elements to bits.  */
3288     tcg_gen_shli_i32(t2, t2, a->esz);
3289 
3290     desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz / 8);
3291     desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
3292 
3293     ptr = tcg_temp_new_ptr();
3294     tcg_gen_addi_ptr(ptr, cpu_env, pred_full_reg_offset(s, a->rd));
3295 
3296     if (a->lt) {
3297         gen_helper_sve_whilel(t2, ptr, t2, tcg_constant_i32(desc));
3298     } else {
3299         gen_helper_sve_whileg(t2, ptr, t2, tcg_constant_i32(desc));
3300     }
3301     do_pred_flags(t2);
3302     return true;
3303 }
3304 
3305 static bool trans_WHILE_ptr(DisasContext *s, arg_WHILE_ptr *a)
3306 {
3307     TCGv_i64 op0, op1, diff, t1, tmax;
3308     TCGv_i32 t2;
3309     TCGv_ptr ptr;
3310     unsigned vsz = vec_full_reg_size(s);
3311     unsigned desc = 0;
3312 
3313     if (!dc_isar_feature(aa64_sve2, s)) {
3314         return false;
3315     }
3316     if (!sve_access_check(s)) {
3317         return true;
3318     }
3319 
3320     op0 = read_cpu_reg(s, a->rn, 1);
3321     op1 = read_cpu_reg(s, a->rm, 1);
3322 
3323     tmax = tcg_constant_i64(vsz);
3324     diff = tcg_temp_new_i64();
3325 
3326     if (a->rw) {
3327         /* WHILERW */
3328         /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
3329         t1 = tcg_temp_new_i64();
3330         tcg_gen_sub_i64(diff, op0, op1);
3331         tcg_gen_sub_i64(t1, op1, op0);
3332         tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, diff, t1);
3333         /* Round down to a multiple of ESIZE.  */
3334         tcg_gen_andi_i64(diff, diff, -1 << a->esz);
3335         /* If op1 == op0, diff == 0, and the condition is always true. */
3336         tcg_gen_movcond_i64(TCG_COND_EQ, diff, op0, op1, tmax, diff);
3337     } else {
3338         /* WHILEWR */
3339         tcg_gen_sub_i64(diff, op1, op0);
3340         /* Round down to a multiple of ESIZE.  */
3341         tcg_gen_andi_i64(diff, diff, -1 << a->esz);
3342         /* If op0 >= op1, diff <= 0, the condition is always true. */
3343         tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, tmax, diff);
3344     }
3345 
3346     /* Bound to the maximum.  */
3347     tcg_gen_umin_i64(diff, diff, tmax);
3348 
3349     /* Since we're bounded, pass as a 32-bit type.  */
3350     t2 = tcg_temp_new_i32();
3351     tcg_gen_extrl_i64_i32(t2, diff);
3352 
3353     desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz / 8);
3354     desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
3355 
3356     ptr = tcg_temp_new_ptr();
3357     tcg_gen_addi_ptr(ptr, cpu_env, pred_full_reg_offset(s, a->rd));
3358 
3359     gen_helper_sve_whilel(t2, ptr, t2, tcg_constant_i32(desc));
3360     do_pred_flags(t2);
3361     return true;
3362 }
3363 
3364 /*
3365  *** SVE Integer Wide Immediate - Unpredicated Group
3366  */
3367 
3368 static bool trans_FDUP(DisasContext *s, arg_FDUP *a)
3369 {
3370     if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
3371         return false;
3372     }
3373     if (sve_access_check(s)) {
3374         unsigned vsz = vec_full_reg_size(s);
3375         int dofs = vec_full_reg_offset(s, a->rd);
3376         uint64_t imm;
3377 
3378         /* Decode the VFP immediate.  */
3379         imm = vfp_expand_imm(a->esz, a->imm);
3380         tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, imm);
3381     }
3382     return true;
3383 }
3384 
3385 static bool trans_DUP_i(DisasContext *s, arg_DUP_i *a)
3386 {
3387     if (!dc_isar_feature(aa64_sve, s)) {
3388         return false;
3389     }
3390     if (sve_access_check(s)) {
3391         unsigned vsz = vec_full_reg_size(s);
3392         int dofs = vec_full_reg_offset(s, a->rd);
3393         tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, a->imm);
3394     }
3395     return true;
3396 }
3397 
3398 TRANS_FEAT(ADD_zzi, aa64_sve, gen_gvec_fn_arg_zzi, tcg_gen_gvec_addi, a)
3399 
3400 static bool trans_SUB_zzi(DisasContext *s, arg_rri_esz *a)
3401 {
3402     a->imm = -a->imm;
3403     return trans_ADD_zzi(s, a);
3404 }
3405 
3406 static bool trans_SUBR_zzi(DisasContext *s, arg_rri_esz *a)
3407 {
3408     static const TCGOpcode vecop_list[] = { INDEX_op_sub_vec, 0 };
3409     static const GVecGen2s op[4] = {
3410         { .fni8 = tcg_gen_vec_sub8_i64,
3411           .fniv = tcg_gen_sub_vec,
3412           .fno = gen_helper_sve_subri_b,
3413           .opt_opc = vecop_list,
3414           .vece = MO_8,
3415           .scalar_first = true },
3416         { .fni8 = tcg_gen_vec_sub16_i64,
3417           .fniv = tcg_gen_sub_vec,
3418           .fno = gen_helper_sve_subri_h,
3419           .opt_opc = vecop_list,
3420           .vece = MO_16,
3421           .scalar_first = true },
3422         { .fni4 = tcg_gen_sub_i32,
3423           .fniv = tcg_gen_sub_vec,
3424           .fno = gen_helper_sve_subri_s,
3425           .opt_opc = vecop_list,
3426           .vece = MO_32,
3427           .scalar_first = true },
3428         { .fni8 = tcg_gen_sub_i64,
3429           .fniv = tcg_gen_sub_vec,
3430           .fno = gen_helper_sve_subri_d,
3431           .opt_opc = vecop_list,
3432           .prefer_i64 = TCG_TARGET_REG_BITS == 64,
3433           .vece = MO_64,
3434           .scalar_first = true }
3435     };
3436 
3437     if (!dc_isar_feature(aa64_sve, s)) {
3438         return false;
3439     }
3440     if (sve_access_check(s)) {
3441         unsigned vsz = vec_full_reg_size(s);
3442         tcg_gen_gvec_2s(vec_full_reg_offset(s, a->rd),
3443                         vec_full_reg_offset(s, a->rn),
3444                         vsz, vsz, tcg_constant_i64(a->imm), &op[a->esz]);
3445     }
3446     return true;
3447 }
3448 
3449 TRANS_FEAT(MUL_zzi, aa64_sve, gen_gvec_fn_arg_zzi, tcg_gen_gvec_muli, a)
3450 
3451 static bool do_zzi_sat(DisasContext *s, arg_rri_esz *a, bool u, bool d)
3452 {
3453     if (sve_access_check(s)) {
3454         do_sat_addsub_vec(s, a->esz, a->rd, a->rn,
3455                           tcg_constant_i64(a->imm), u, d);
3456     }
3457     return true;
3458 }
3459 
3460 TRANS_FEAT(SQADD_zzi, aa64_sve, do_zzi_sat, a, false, false)
3461 TRANS_FEAT(UQADD_zzi, aa64_sve, do_zzi_sat, a, true, false)
3462 TRANS_FEAT(SQSUB_zzi, aa64_sve, do_zzi_sat, a, false, true)
3463 TRANS_FEAT(UQSUB_zzi, aa64_sve, do_zzi_sat, a, true, true)
3464 
3465 static bool do_zzi_ool(DisasContext *s, arg_rri_esz *a, gen_helper_gvec_2i *fn)
3466 {
3467     if (sve_access_check(s)) {
3468         unsigned vsz = vec_full_reg_size(s);
3469         tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd),
3470                             vec_full_reg_offset(s, a->rn),
3471                             tcg_constant_i64(a->imm), vsz, vsz, 0, fn);
3472     }
3473     return true;
3474 }
3475 
3476 #define DO_ZZI(NAME, name) \
3477     static gen_helper_gvec_2i * const name##i_fns[4] = {                \
3478         gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h,         \
3479         gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d,         \
3480     };                                                                  \
3481     TRANS_FEAT(NAME##_zzi, aa64_sve, do_zzi_ool, a, name##i_fns[a->esz])
3482 
3483 DO_ZZI(SMAX, smax)
3484 DO_ZZI(UMAX, umax)
3485 DO_ZZI(SMIN, smin)
3486 DO_ZZI(UMIN, umin)
3487 
3488 #undef DO_ZZI
3489 
3490 static gen_helper_gvec_4 * const dot_fns[2][2] = {
3491     { gen_helper_gvec_sdot_b, gen_helper_gvec_sdot_h },
3492     { gen_helper_gvec_udot_b, gen_helper_gvec_udot_h }
3493 };
3494 TRANS_FEAT(DOT_zzzz, aa64_sve, gen_gvec_ool_zzzz,
3495            dot_fns[a->u][a->sz], a->rd, a->rn, a->rm, a->ra, 0)
3496 
3497 /*
3498  * SVE Multiply - Indexed
3499  */
3500 
3501 TRANS_FEAT(SDOT_zzxw_s, aa64_sve, gen_gvec_ool_arg_zzxz,
3502            gen_helper_gvec_sdot_idx_b, a)
3503 TRANS_FEAT(SDOT_zzxw_d, aa64_sve, gen_gvec_ool_arg_zzxz,
3504            gen_helper_gvec_sdot_idx_h, a)
3505 TRANS_FEAT(UDOT_zzxw_s, aa64_sve, gen_gvec_ool_arg_zzxz,
3506            gen_helper_gvec_udot_idx_b, a)
3507 TRANS_FEAT(UDOT_zzxw_d, aa64_sve, gen_gvec_ool_arg_zzxz,
3508            gen_helper_gvec_udot_idx_h, a)
3509 
3510 TRANS_FEAT(SUDOT_zzxw_s, aa64_sve_i8mm, gen_gvec_ool_arg_zzxz,
3511            gen_helper_gvec_sudot_idx_b, a)
3512 TRANS_FEAT(USDOT_zzxw_s, aa64_sve_i8mm, gen_gvec_ool_arg_zzxz,
3513            gen_helper_gvec_usdot_idx_b, a)
3514 
3515 #define DO_SVE2_RRX(NAME, FUNC) \
3516     TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC,          \
3517                a->rd, a->rn, a->rm, a->index)
3518 
3519 DO_SVE2_RRX(MUL_zzx_h, gen_helper_gvec_mul_idx_h)
3520 DO_SVE2_RRX(MUL_zzx_s, gen_helper_gvec_mul_idx_s)
3521 DO_SVE2_RRX(MUL_zzx_d, gen_helper_gvec_mul_idx_d)
3522 
3523 DO_SVE2_RRX(SQDMULH_zzx_h, gen_helper_sve2_sqdmulh_idx_h)
3524 DO_SVE2_RRX(SQDMULH_zzx_s, gen_helper_sve2_sqdmulh_idx_s)
3525 DO_SVE2_RRX(SQDMULH_zzx_d, gen_helper_sve2_sqdmulh_idx_d)
3526 
3527 DO_SVE2_RRX(SQRDMULH_zzx_h, gen_helper_sve2_sqrdmulh_idx_h)
3528 DO_SVE2_RRX(SQRDMULH_zzx_s, gen_helper_sve2_sqrdmulh_idx_s)
3529 DO_SVE2_RRX(SQRDMULH_zzx_d, gen_helper_sve2_sqrdmulh_idx_d)
3530 
3531 #undef DO_SVE2_RRX
3532 
3533 #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
3534     TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC,          \
3535                a->rd, a->rn, a->rm, (a->index << 1) | TOP)
3536 
3537 DO_SVE2_RRX_TB(SQDMULLB_zzx_s, gen_helper_sve2_sqdmull_idx_s, false)
3538 DO_SVE2_RRX_TB(SQDMULLB_zzx_d, gen_helper_sve2_sqdmull_idx_d, false)
3539 DO_SVE2_RRX_TB(SQDMULLT_zzx_s, gen_helper_sve2_sqdmull_idx_s, true)
3540 DO_SVE2_RRX_TB(SQDMULLT_zzx_d, gen_helper_sve2_sqdmull_idx_d, true)
3541 
3542 DO_SVE2_RRX_TB(SMULLB_zzx_s, gen_helper_sve2_smull_idx_s, false)
3543 DO_SVE2_RRX_TB(SMULLB_zzx_d, gen_helper_sve2_smull_idx_d, false)
3544 DO_SVE2_RRX_TB(SMULLT_zzx_s, gen_helper_sve2_smull_idx_s, true)
3545 DO_SVE2_RRX_TB(SMULLT_zzx_d, gen_helper_sve2_smull_idx_d, true)
3546 
3547 DO_SVE2_RRX_TB(UMULLB_zzx_s, gen_helper_sve2_umull_idx_s, false)
3548 DO_SVE2_RRX_TB(UMULLB_zzx_d, gen_helper_sve2_umull_idx_d, false)
3549 DO_SVE2_RRX_TB(UMULLT_zzx_s, gen_helper_sve2_umull_idx_s, true)
3550 DO_SVE2_RRX_TB(UMULLT_zzx_d, gen_helper_sve2_umull_idx_d, true)
3551 
3552 #undef DO_SVE2_RRX_TB
3553 
3554 #define DO_SVE2_RRXR(NAME, FUNC) \
3555     TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzxz, FUNC, a)
3556 
3557 DO_SVE2_RRXR(MLA_zzxz_h, gen_helper_gvec_mla_idx_h)
3558 DO_SVE2_RRXR(MLA_zzxz_s, gen_helper_gvec_mla_idx_s)
3559 DO_SVE2_RRXR(MLA_zzxz_d, gen_helper_gvec_mla_idx_d)
3560 
3561 DO_SVE2_RRXR(MLS_zzxz_h, gen_helper_gvec_mls_idx_h)
3562 DO_SVE2_RRXR(MLS_zzxz_s, gen_helper_gvec_mls_idx_s)
3563 DO_SVE2_RRXR(MLS_zzxz_d, gen_helper_gvec_mls_idx_d)
3564 
3565 DO_SVE2_RRXR(SQRDMLAH_zzxz_h, gen_helper_sve2_sqrdmlah_idx_h)
3566 DO_SVE2_RRXR(SQRDMLAH_zzxz_s, gen_helper_sve2_sqrdmlah_idx_s)
3567 DO_SVE2_RRXR(SQRDMLAH_zzxz_d, gen_helper_sve2_sqrdmlah_idx_d)
3568 
3569 DO_SVE2_RRXR(SQRDMLSH_zzxz_h, gen_helper_sve2_sqrdmlsh_idx_h)
3570 DO_SVE2_RRXR(SQRDMLSH_zzxz_s, gen_helper_sve2_sqrdmlsh_idx_s)
3571 DO_SVE2_RRXR(SQRDMLSH_zzxz_d, gen_helper_sve2_sqrdmlsh_idx_d)
3572 
3573 #undef DO_SVE2_RRXR
3574 
3575 #define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
3576     TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC,        \
3577                a->rd, a->rn, a->rm, a->ra, (a->index << 1) | TOP)
3578 
3579 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, false)
3580 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, false)
3581 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, true)
3582 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, true)
3583 
3584 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, false)
3585 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, false)
3586 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, true)
3587 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, true)
3588 
3589 DO_SVE2_RRXR_TB(SMLALB_zzxw_s, gen_helper_sve2_smlal_idx_s, false)
3590 DO_SVE2_RRXR_TB(SMLALB_zzxw_d, gen_helper_sve2_smlal_idx_d, false)
3591 DO_SVE2_RRXR_TB(SMLALT_zzxw_s, gen_helper_sve2_smlal_idx_s, true)
3592 DO_SVE2_RRXR_TB(SMLALT_zzxw_d, gen_helper_sve2_smlal_idx_d, true)
3593 
3594 DO_SVE2_RRXR_TB(UMLALB_zzxw_s, gen_helper_sve2_umlal_idx_s, false)
3595 DO_SVE2_RRXR_TB(UMLALB_zzxw_d, gen_helper_sve2_umlal_idx_d, false)
3596 DO_SVE2_RRXR_TB(UMLALT_zzxw_s, gen_helper_sve2_umlal_idx_s, true)
3597 DO_SVE2_RRXR_TB(UMLALT_zzxw_d, gen_helper_sve2_umlal_idx_d, true)
3598 
3599 DO_SVE2_RRXR_TB(SMLSLB_zzxw_s, gen_helper_sve2_smlsl_idx_s, false)
3600 DO_SVE2_RRXR_TB(SMLSLB_zzxw_d, gen_helper_sve2_smlsl_idx_d, false)
3601 DO_SVE2_RRXR_TB(SMLSLT_zzxw_s, gen_helper_sve2_smlsl_idx_s, true)
3602 DO_SVE2_RRXR_TB(SMLSLT_zzxw_d, gen_helper_sve2_smlsl_idx_d, true)
3603 
3604 DO_SVE2_RRXR_TB(UMLSLB_zzxw_s, gen_helper_sve2_umlsl_idx_s, false)
3605 DO_SVE2_RRXR_TB(UMLSLB_zzxw_d, gen_helper_sve2_umlsl_idx_d, false)
3606 DO_SVE2_RRXR_TB(UMLSLT_zzxw_s, gen_helper_sve2_umlsl_idx_s, true)
3607 DO_SVE2_RRXR_TB(UMLSLT_zzxw_d, gen_helper_sve2_umlsl_idx_d, true)
3608 
3609 #undef DO_SVE2_RRXR_TB
3610 
3611 #define DO_SVE2_RRXR_ROT(NAME, FUNC) \
3612     TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC,           \
3613                a->rd, a->rn, a->rm, a->ra, (a->index << 2) | a->rot)
3614 
3615 DO_SVE2_RRXR_ROT(CMLA_zzxz_h, gen_helper_sve2_cmla_idx_h)
3616 DO_SVE2_RRXR_ROT(CMLA_zzxz_s, gen_helper_sve2_cmla_idx_s)
3617 
3618 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h, gen_helper_sve2_sqrdcmlah_idx_h)
3619 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s, gen_helper_sve2_sqrdcmlah_idx_s)
3620 
3621 DO_SVE2_RRXR_ROT(CDOT_zzxw_s, gen_helper_sve2_cdot_idx_s)
3622 DO_SVE2_RRXR_ROT(CDOT_zzxw_d, gen_helper_sve2_cdot_idx_d)
3623 
3624 #undef DO_SVE2_RRXR_ROT
3625 
3626 /*
3627  *** SVE Floating Point Multiply-Add Indexed Group
3628  */
3629 
3630 static bool do_FMLA_zzxz(DisasContext *s, arg_rrxr_esz *a, bool sub)
3631 {
3632     static gen_helper_gvec_4_ptr * const fns[4] = {
3633         NULL,
3634         gen_helper_gvec_fmla_idx_h,
3635         gen_helper_gvec_fmla_idx_s,
3636         gen_helper_gvec_fmla_idx_d,
3637     };
3638     return gen_gvec_fpst_zzzz(s, fns[a->esz], a->rd, a->rn, a->rm, a->ra,
3639                               (a->index << 1) | sub,
3640                               a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
3641 }
3642 
3643 TRANS_FEAT(FMLA_zzxz, aa64_sve, do_FMLA_zzxz, a, false)
3644 TRANS_FEAT(FMLS_zzxz, aa64_sve, do_FMLA_zzxz, a, true)
3645 
3646 /*
3647  *** SVE Floating Point Multiply Indexed Group
3648  */
3649 
3650 static gen_helper_gvec_3_ptr * const fmul_idx_fns[4] = {
3651     NULL,                       gen_helper_gvec_fmul_idx_h,
3652     gen_helper_gvec_fmul_idx_s, gen_helper_gvec_fmul_idx_d,
3653 };
3654 TRANS_FEAT(FMUL_zzx, aa64_sve, gen_gvec_fpst_zzz,
3655            fmul_idx_fns[a->esz], a->rd, a->rn, a->rm, a->index,
3656            a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3657 
3658 /*
3659  *** SVE Floating Point Fast Reduction Group
3660  */
3661 
3662 typedef void gen_helper_fp_reduce(TCGv_i64, TCGv_ptr, TCGv_ptr,
3663                                   TCGv_ptr, TCGv_i32);
3664 
3665 static bool do_reduce(DisasContext *s, arg_rpr_esz *a,
3666                       gen_helper_fp_reduce *fn)
3667 {
3668     unsigned vsz, p2vsz;
3669     TCGv_i32 t_desc;
3670     TCGv_ptr t_zn, t_pg, status;
3671     TCGv_i64 temp;
3672 
3673     if (fn == NULL) {
3674         return false;
3675     }
3676     if (!sve_access_check(s)) {
3677         return true;
3678     }
3679 
3680     vsz = vec_full_reg_size(s);
3681     p2vsz = pow2ceil(vsz);
3682     t_desc = tcg_constant_i32(simd_desc(vsz, vsz, p2vsz));
3683     temp = tcg_temp_new_i64();
3684     t_zn = tcg_temp_new_ptr();
3685     t_pg = tcg_temp_new_ptr();
3686 
3687     tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, a->rn));
3688     tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->pg));
3689     status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
3690 
3691     fn(temp, t_zn, t_pg, status, t_desc);
3692 
3693     write_fp_dreg(s, a->rd, temp);
3694     return true;
3695 }
3696 
3697 #define DO_VPZ(NAME, name) \
3698     static gen_helper_fp_reduce * const name##_fns[4] = {                \
3699         NULL,                      gen_helper_sve_##name##_h,            \
3700         gen_helper_sve_##name##_s, gen_helper_sve_##name##_d,            \
3701     };                                                                   \
3702     TRANS_FEAT(NAME, aa64_sve, do_reduce, a, name##_fns[a->esz])
3703 
3704 DO_VPZ(FADDV, faddv)
3705 DO_VPZ(FMINNMV, fminnmv)
3706 DO_VPZ(FMAXNMV, fmaxnmv)
3707 DO_VPZ(FMINV, fminv)
3708 DO_VPZ(FMAXV, fmaxv)
3709 
3710 #undef DO_VPZ
3711 
3712 /*
3713  *** SVE Floating Point Unary Operations - Unpredicated Group
3714  */
3715 
3716 static gen_helper_gvec_2_ptr * const frecpe_fns[] = {
3717     NULL,                     gen_helper_gvec_frecpe_h,
3718     gen_helper_gvec_frecpe_s, gen_helper_gvec_frecpe_d,
3719 };
3720 TRANS_FEAT(FRECPE, aa64_sve, gen_gvec_fpst_arg_zz, frecpe_fns[a->esz], a, 0)
3721 
3722 static gen_helper_gvec_2_ptr * const frsqrte_fns[] = {
3723     NULL,                      gen_helper_gvec_frsqrte_h,
3724     gen_helper_gvec_frsqrte_s, gen_helper_gvec_frsqrte_d,
3725 };
3726 TRANS_FEAT(FRSQRTE, aa64_sve, gen_gvec_fpst_arg_zz, frsqrte_fns[a->esz], a, 0)
3727 
3728 /*
3729  *** SVE Floating Point Compare with Zero Group
3730  */
3731 
3732 static bool do_ppz_fp(DisasContext *s, arg_rpr_esz *a,
3733                       gen_helper_gvec_3_ptr *fn)
3734 {
3735     if (fn == NULL) {
3736         return false;
3737     }
3738     if (sve_access_check(s)) {
3739         unsigned vsz = vec_full_reg_size(s);
3740         TCGv_ptr status =
3741             fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
3742 
3743         tcg_gen_gvec_3_ptr(pred_full_reg_offset(s, a->rd),
3744                            vec_full_reg_offset(s, a->rn),
3745                            pred_full_reg_offset(s, a->pg),
3746                            status, vsz, vsz, 0, fn);
3747     }
3748     return true;
3749 }
3750 
3751 #define DO_PPZ(NAME, name) \
3752     static gen_helper_gvec_3_ptr * const name##_fns[] = {         \
3753         NULL,                      gen_helper_sve_##name##_h,     \
3754         gen_helper_sve_##name##_s, gen_helper_sve_##name##_d,     \
3755     };                                                            \
3756     TRANS_FEAT(NAME, aa64_sve, do_ppz_fp, a, name##_fns[a->esz])
3757 
3758 DO_PPZ(FCMGE_ppz0, fcmge0)
3759 DO_PPZ(FCMGT_ppz0, fcmgt0)
3760 DO_PPZ(FCMLE_ppz0, fcmle0)
3761 DO_PPZ(FCMLT_ppz0, fcmlt0)
3762 DO_PPZ(FCMEQ_ppz0, fcmeq0)
3763 DO_PPZ(FCMNE_ppz0, fcmne0)
3764 
3765 #undef DO_PPZ
3766 
3767 /*
3768  *** SVE floating-point trig multiply-add coefficient
3769  */
3770 
3771 static gen_helper_gvec_3_ptr * const ftmad_fns[4] = {
3772     NULL,                   gen_helper_sve_ftmad_h,
3773     gen_helper_sve_ftmad_s, gen_helper_sve_ftmad_d,
3774 };
3775 TRANS_FEAT_NONSTREAMING(FTMAD, aa64_sve, gen_gvec_fpst_zzz,
3776                         ftmad_fns[a->esz], a->rd, a->rn, a->rm, a->imm,
3777                         a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3778 
3779 /*
3780  *** SVE Floating Point Accumulating Reduction Group
3781  */
3782 
3783 static bool trans_FADDA(DisasContext *s, arg_rprr_esz *a)
3784 {
3785     typedef void fadda_fn(TCGv_i64, TCGv_i64, TCGv_ptr,
3786                           TCGv_ptr, TCGv_ptr, TCGv_i32);
3787     static fadda_fn * const fns[3] = {
3788         gen_helper_sve_fadda_h,
3789         gen_helper_sve_fadda_s,
3790         gen_helper_sve_fadda_d,
3791     };
3792     unsigned vsz = vec_full_reg_size(s);
3793     TCGv_ptr t_rm, t_pg, t_fpst;
3794     TCGv_i64 t_val;
3795     TCGv_i32 t_desc;
3796 
3797     if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
3798         return false;
3799     }
3800     s->is_nonstreaming = true;
3801     if (!sve_access_check(s)) {
3802         return true;
3803     }
3804 
3805     t_val = load_esz(cpu_env, vec_reg_offset(s, a->rn, 0, a->esz), a->esz);
3806     t_rm = tcg_temp_new_ptr();
3807     t_pg = tcg_temp_new_ptr();
3808     tcg_gen_addi_ptr(t_rm, cpu_env, vec_full_reg_offset(s, a->rm));
3809     tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->pg));
3810     t_fpst = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
3811     t_desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
3812 
3813     fns[a->esz - 1](t_val, t_val, t_rm, t_pg, t_fpst, t_desc);
3814 
3815     write_fp_dreg(s, a->rd, t_val);
3816     return true;
3817 }
3818 
3819 /*
3820  *** SVE Floating Point Arithmetic - Unpredicated Group
3821  */
3822 
3823 #define DO_FP3(NAME, name) \
3824     static gen_helper_gvec_3_ptr * const name##_fns[4] = {          \
3825         NULL, gen_helper_gvec_##name##_h,                           \
3826         gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d      \
3827     };                                                              \
3828     TRANS_FEAT(NAME, aa64_sve, gen_gvec_fpst_arg_zzz, name##_fns[a->esz], a, 0)
3829 
3830 DO_FP3(FADD_zzz, fadd)
3831 DO_FP3(FSUB_zzz, fsub)
3832 DO_FP3(FMUL_zzz, fmul)
3833 DO_FP3(FRECPS, recps)
3834 DO_FP3(FRSQRTS, rsqrts)
3835 
3836 #undef DO_FP3
3837 
3838 static gen_helper_gvec_3_ptr * const ftsmul_fns[4] = {
3839     NULL,                     gen_helper_gvec_ftsmul_h,
3840     gen_helper_gvec_ftsmul_s, gen_helper_gvec_ftsmul_d
3841 };
3842 TRANS_FEAT_NONSTREAMING(FTSMUL, aa64_sve, gen_gvec_fpst_arg_zzz,
3843                         ftsmul_fns[a->esz], a, 0)
3844 
3845 /*
3846  *** SVE Floating Point Arithmetic - Predicated Group
3847  */
3848 
3849 #define DO_ZPZZ_FP(NAME, FEAT, name) \
3850     static gen_helper_gvec_4_ptr * const name##_zpzz_fns[4] = { \
3851         NULL,                  gen_helper_##name##_h,           \
3852         gen_helper_##name##_s, gen_helper_##name##_d            \
3853     };                                                          \
3854     TRANS_FEAT(NAME, FEAT, gen_gvec_fpst_arg_zpzz, name##_zpzz_fns[a->esz], a)
3855 
3856 DO_ZPZZ_FP(FADD_zpzz, aa64_sve, sve_fadd)
3857 DO_ZPZZ_FP(FSUB_zpzz, aa64_sve, sve_fsub)
3858 DO_ZPZZ_FP(FMUL_zpzz, aa64_sve, sve_fmul)
3859 DO_ZPZZ_FP(FMIN_zpzz, aa64_sve, sve_fmin)
3860 DO_ZPZZ_FP(FMAX_zpzz, aa64_sve, sve_fmax)
3861 DO_ZPZZ_FP(FMINNM_zpzz, aa64_sve, sve_fminnum)
3862 DO_ZPZZ_FP(FMAXNM_zpzz, aa64_sve, sve_fmaxnum)
3863 DO_ZPZZ_FP(FABD, aa64_sve, sve_fabd)
3864 DO_ZPZZ_FP(FSCALE, aa64_sve, sve_fscalbn)
3865 DO_ZPZZ_FP(FDIV, aa64_sve, sve_fdiv)
3866 DO_ZPZZ_FP(FMULX, aa64_sve, sve_fmulx)
3867 
3868 typedef void gen_helper_sve_fp2scalar(TCGv_ptr, TCGv_ptr, TCGv_ptr,
3869                                       TCGv_i64, TCGv_ptr, TCGv_i32);
3870 
3871 static void do_fp_scalar(DisasContext *s, int zd, int zn, int pg, bool is_fp16,
3872                          TCGv_i64 scalar, gen_helper_sve_fp2scalar *fn)
3873 {
3874     unsigned vsz = vec_full_reg_size(s);
3875     TCGv_ptr t_zd, t_zn, t_pg, status;
3876     TCGv_i32 desc;
3877 
3878     t_zd = tcg_temp_new_ptr();
3879     t_zn = tcg_temp_new_ptr();
3880     t_pg = tcg_temp_new_ptr();
3881     tcg_gen_addi_ptr(t_zd, cpu_env, vec_full_reg_offset(s, zd));
3882     tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, zn));
3883     tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
3884 
3885     status = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR);
3886     desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
3887     fn(t_zd, t_zn, t_pg, scalar, status, desc);
3888 }
3889 
3890 static bool do_fp_imm(DisasContext *s, arg_rpri_esz *a, uint64_t imm,
3891                       gen_helper_sve_fp2scalar *fn)
3892 {
3893     if (fn == NULL) {
3894         return false;
3895     }
3896     if (sve_access_check(s)) {
3897         do_fp_scalar(s, a->rd, a->rn, a->pg, a->esz == MO_16,
3898                      tcg_constant_i64(imm), fn);
3899     }
3900     return true;
3901 }
3902 
3903 #define DO_FP_IMM(NAME, name, const0, const1)                           \
3904     static gen_helper_sve_fp2scalar * const name##_fns[4] = {           \
3905         NULL, gen_helper_sve_##name##_h,                                \
3906         gen_helper_sve_##name##_s,                                      \
3907         gen_helper_sve_##name##_d                                       \
3908     };                                                                  \
3909     static uint64_t const name##_const[4][2] = {                        \
3910         { -1, -1 },                                                     \
3911         { float16_##const0, float16_##const1 },                         \
3912         { float32_##const0, float32_##const1 },                         \
3913         { float64_##const0, float64_##const1 },                         \
3914     };                                                                  \
3915     TRANS_FEAT(NAME##_zpzi, aa64_sve, do_fp_imm, a,                     \
3916                name##_const[a->esz][a->imm], name##_fns[a->esz])
3917 
3918 DO_FP_IMM(FADD, fadds, half, one)
3919 DO_FP_IMM(FSUB, fsubs, half, one)
3920 DO_FP_IMM(FMUL, fmuls, half, two)
3921 DO_FP_IMM(FSUBR, fsubrs, half, one)
3922 DO_FP_IMM(FMAXNM, fmaxnms, zero, one)
3923 DO_FP_IMM(FMINNM, fminnms, zero, one)
3924 DO_FP_IMM(FMAX, fmaxs, zero, one)
3925 DO_FP_IMM(FMIN, fmins, zero, one)
3926 
3927 #undef DO_FP_IMM
3928 
3929 static bool do_fp_cmp(DisasContext *s, arg_rprr_esz *a,
3930                       gen_helper_gvec_4_ptr *fn)
3931 {
3932     if (fn == NULL) {
3933         return false;
3934     }
3935     if (sve_access_check(s)) {
3936         unsigned vsz = vec_full_reg_size(s);
3937         TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
3938         tcg_gen_gvec_4_ptr(pred_full_reg_offset(s, a->rd),
3939                            vec_full_reg_offset(s, a->rn),
3940                            vec_full_reg_offset(s, a->rm),
3941                            pred_full_reg_offset(s, a->pg),
3942                            status, vsz, vsz, 0, fn);
3943     }
3944     return true;
3945 }
3946 
3947 #define DO_FPCMP(NAME, name) \
3948     static gen_helper_gvec_4_ptr * const name##_fns[4] = {            \
3949         NULL, gen_helper_sve_##name##_h,                              \
3950         gen_helper_sve_##name##_s, gen_helper_sve_##name##_d          \
3951     };                                                                \
3952     TRANS_FEAT(NAME##_ppzz, aa64_sve, do_fp_cmp, a, name##_fns[a->esz])
3953 
3954 DO_FPCMP(FCMGE, fcmge)
3955 DO_FPCMP(FCMGT, fcmgt)
3956 DO_FPCMP(FCMEQ, fcmeq)
3957 DO_FPCMP(FCMNE, fcmne)
3958 DO_FPCMP(FCMUO, fcmuo)
3959 DO_FPCMP(FACGE, facge)
3960 DO_FPCMP(FACGT, facgt)
3961 
3962 #undef DO_FPCMP
3963 
3964 static gen_helper_gvec_4_ptr * const fcadd_fns[] = {
3965     NULL,                   gen_helper_sve_fcadd_h,
3966     gen_helper_sve_fcadd_s, gen_helper_sve_fcadd_d,
3967 };
3968 TRANS_FEAT(FCADD, aa64_sve, gen_gvec_fpst_zzzp, fcadd_fns[a->esz],
3969            a->rd, a->rn, a->rm, a->pg, a->rot,
3970            a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3971 
3972 #define DO_FMLA(NAME, name) \
3973     static gen_helper_gvec_5_ptr * const name##_fns[4] = {              \
3974         NULL, gen_helper_sve_##name##_h,                                \
3975         gen_helper_sve_##name##_s, gen_helper_sve_##name##_d            \
3976     };                                                                  \
3977     TRANS_FEAT(NAME, aa64_sve, gen_gvec_fpst_zzzzp, name##_fns[a->esz], \
3978                a->rd, a->rn, a->rm, a->ra, a->pg, 0,                    \
3979                a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3980 
3981 DO_FMLA(FMLA_zpzzz, fmla_zpzzz)
3982 DO_FMLA(FMLS_zpzzz, fmls_zpzzz)
3983 DO_FMLA(FNMLA_zpzzz, fnmla_zpzzz)
3984 DO_FMLA(FNMLS_zpzzz, fnmls_zpzzz)
3985 
3986 #undef DO_FMLA
3987 
3988 static gen_helper_gvec_5_ptr * const fcmla_fns[4] = {
3989     NULL,                         gen_helper_sve_fcmla_zpzzz_h,
3990     gen_helper_sve_fcmla_zpzzz_s, gen_helper_sve_fcmla_zpzzz_d,
3991 };
3992 TRANS_FEAT(FCMLA_zpzzz, aa64_sve, gen_gvec_fpst_zzzzp, fcmla_fns[a->esz],
3993            a->rd, a->rn, a->rm, a->ra, a->pg, a->rot,
3994            a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3995 
3996 static gen_helper_gvec_4_ptr * const fcmla_idx_fns[4] = {
3997     NULL, gen_helper_gvec_fcmlah_idx, gen_helper_gvec_fcmlas_idx, NULL
3998 };
3999 TRANS_FEAT(FCMLA_zzxz, aa64_sve, gen_gvec_fpst_zzzz, fcmla_idx_fns[a->esz],
4000            a->rd, a->rn, a->rm, a->ra, a->index * 4 + a->rot,
4001            a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
4002 
4003 /*
4004  *** SVE Floating Point Unary Operations Predicated Group
4005  */
4006 
4007 TRANS_FEAT(FCVT_sh, aa64_sve, gen_gvec_fpst_arg_zpz,
4008            gen_helper_sve_fcvt_sh, a, 0, FPST_FPCR)
4009 TRANS_FEAT(FCVT_hs, aa64_sve, gen_gvec_fpst_arg_zpz,
4010            gen_helper_sve_fcvt_hs, a, 0, FPST_FPCR)
4011 
4012 TRANS_FEAT(BFCVT, aa64_sve_bf16, gen_gvec_fpst_arg_zpz,
4013            gen_helper_sve_bfcvt, a, 0, FPST_FPCR)
4014 
4015 TRANS_FEAT(FCVT_dh, aa64_sve, gen_gvec_fpst_arg_zpz,
4016            gen_helper_sve_fcvt_dh, a, 0, FPST_FPCR)
4017 TRANS_FEAT(FCVT_hd, aa64_sve, gen_gvec_fpst_arg_zpz,
4018            gen_helper_sve_fcvt_hd, a, 0, FPST_FPCR)
4019 TRANS_FEAT(FCVT_ds, aa64_sve, gen_gvec_fpst_arg_zpz,
4020            gen_helper_sve_fcvt_ds, a, 0, FPST_FPCR)
4021 TRANS_FEAT(FCVT_sd, aa64_sve, gen_gvec_fpst_arg_zpz,
4022            gen_helper_sve_fcvt_sd, a, 0, FPST_FPCR)
4023 
4024 TRANS_FEAT(FCVTZS_hh, aa64_sve, gen_gvec_fpst_arg_zpz,
4025            gen_helper_sve_fcvtzs_hh, a, 0, FPST_FPCR_F16)
4026 TRANS_FEAT(FCVTZU_hh, aa64_sve, gen_gvec_fpst_arg_zpz,
4027            gen_helper_sve_fcvtzu_hh, a, 0, FPST_FPCR_F16)
4028 TRANS_FEAT(FCVTZS_hs, aa64_sve, gen_gvec_fpst_arg_zpz,
4029            gen_helper_sve_fcvtzs_hs, a, 0, FPST_FPCR_F16)
4030 TRANS_FEAT(FCVTZU_hs, aa64_sve, gen_gvec_fpst_arg_zpz,
4031            gen_helper_sve_fcvtzu_hs, a, 0, FPST_FPCR_F16)
4032 TRANS_FEAT(FCVTZS_hd, aa64_sve, gen_gvec_fpst_arg_zpz,
4033            gen_helper_sve_fcvtzs_hd, a, 0, FPST_FPCR_F16)
4034 TRANS_FEAT(FCVTZU_hd, aa64_sve, gen_gvec_fpst_arg_zpz,
4035            gen_helper_sve_fcvtzu_hd, a, 0, FPST_FPCR_F16)
4036 
4037 TRANS_FEAT(FCVTZS_ss, aa64_sve, gen_gvec_fpst_arg_zpz,
4038            gen_helper_sve_fcvtzs_ss, a, 0, FPST_FPCR)
4039 TRANS_FEAT(FCVTZU_ss, aa64_sve, gen_gvec_fpst_arg_zpz,
4040            gen_helper_sve_fcvtzu_ss, a, 0, FPST_FPCR)
4041 TRANS_FEAT(FCVTZS_sd, aa64_sve, gen_gvec_fpst_arg_zpz,
4042            gen_helper_sve_fcvtzs_sd, a, 0, FPST_FPCR)
4043 TRANS_FEAT(FCVTZU_sd, aa64_sve, gen_gvec_fpst_arg_zpz,
4044            gen_helper_sve_fcvtzu_sd, a, 0, FPST_FPCR)
4045 TRANS_FEAT(FCVTZS_ds, aa64_sve, gen_gvec_fpst_arg_zpz,
4046            gen_helper_sve_fcvtzs_ds, a, 0, FPST_FPCR)
4047 TRANS_FEAT(FCVTZU_ds, aa64_sve, gen_gvec_fpst_arg_zpz,
4048            gen_helper_sve_fcvtzu_ds, a, 0, FPST_FPCR)
4049 
4050 TRANS_FEAT(FCVTZS_dd, aa64_sve, gen_gvec_fpst_arg_zpz,
4051            gen_helper_sve_fcvtzs_dd, a, 0, FPST_FPCR)
4052 TRANS_FEAT(FCVTZU_dd, aa64_sve, gen_gvec_fpst_arg_zpz,
4053            gen_helper_sve_fcvtzu_dd, a, 0, FPST_FPCR)
4054 
4055 static gen_helper_gvec_3_ptr * const frint_fns[] = {
4056     NULL,
4057     gen_helper_sve_frint_h,
4058     gen_helper_sve_frint_s,
4059     gen_helper_sve_frint_d
4060 };
4061 TRANS_FEAT(FRINTI, aa64_sve, gen_gvec_fpst_arg_zpz, frint_fns[a->esz],
4062            a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
4063 
4064 static gen_helper_gvec_3_ptr * const frintx_fns[] = {
4065     NULL,
4066     gen_helper_sve_frintx_h,
4067     gen_helper_sve_frintx_s,
4068     gen_helper_sve_frintx_d
4069 };
4070 TRANS_FEAT(FRINTX, aa64_sve, gen_gvec_fpst_arg_zpz, frintx_fns[a->esz],
4071            a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
4072 
4073 static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a,
4074                           ARMFPRounding mode, gen_helper_gvec_3_ptr *fn)
4075 {
4076     unsigned vsz;
4077     TCGv_i32 tmode;
4078     TCGv_ptr status;
4079 
4080     if (fn == NULL) {
4081         return false;
4082     }
4083     if (!sve_access_check(s)) {
4084         return true;
4085     }
4086 
4087     vsz = vec_full_reg_size(s);
4088     status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
4089     tmode = gen_set_rmode(mode, status);
4090 
4091     tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
4092                        vec_full_reg_offset(s, a->rn),
4093                        pred_full_reg_offset(s, a->pg),
4094                        status, vsz, vsz, 0, fn);
4095 
4096     gen_restore_rmode(tmode, status);
4097     return true;
4098 }
4099 
4100 TRANS_FEAT(FRINTN, aa64_sve, do_frint_mode, a,
4101            FPROUNDING_TIEEVEN, frint_fns[a->esz])
4102 TRANS_FEAT(FRINTP, aa64_sve, do_frint_mode, a,
4103            FPROUNDING_POSINF, frint_fns[a->esz])
4104 TRANS_FEAT(FRINTM, aa64_sve, do_frint_mode, a,
4105            FPROUNDING_NEGINF, frint_fns[a->esz])
4106 TRANS_FEAT(FRINTZ, aa64_sve, do_frint_mode, a,
4107            FPROUNDING_ZERO, frint_fns[a->esz])
4108 TRANS_FEAT(FRINTA, aa64_sve, do_frint_mode, a,
4109            FPROUNDING_TIEAWAY, frint_fns[a->esz])
4110 
4111 static gen_helper_gvec_3_ptr * const frecpx_fns[] = {
4112     NULL,                    gen_helper_sve_frecpx_h,
4113     gen_helper_sve_frecpx_s, gen_helper_sve_frecpx_d,
4114 };
4115 TRANS_FEAT(FRECPX, aa64_sve, gen_gvec_fpst_arg_zpz, frecpx_fns[a->esz],
4116            a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
4117 
4118 static gen_helper_gvec_3_ptr * const fsqrt_fns[] = {
4119     NULL,                   gen_helper_sve_fsqrt_h,
4120     gen_helper_sve_fsqrt_s, gen_helper_sve_fsqrt_d,
4121 };
4122 TRANS_FEAT(FSQRT, aa64_sve, gen_gvec_fpst_arg_zpz, fsqrt_fns[a->esz],
4123            a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
4124 
4125 TRANS_FEAT(SCVTF_hh, aa64_sve, gen_gvec_fpst_arg_zpz,
4126            gen_helper_sve_scvt_hh, a, 0, FPST_FPCR_F16)
4127 TRANS_FEAT(SCVTF_sh, aa64_sve, gen_gvec_fpst_arg_zpz,
4128            gen_helper_sve_scvt_sh, a, 0, FPST_FPCR_F16)
4129 TRANS_FEAT(SCVTF_dh, aa64_sve, gen_gvec_fpst_arg_zpz,
4130            gen_helper_sve_scvt_dh, a, 0, FPST_FPCR_F16)
4131 
4132 TRANS_FEAT(SCVTF_ss, aa64_sve, gen_gvec_fpst_arg_zpz,
4133            gen_helper_sve_scvt_ss, a, 0, FPST_FPCR)
4134 TRANS_FEAT(SCVTF_ds, aa64_sve, gen_gvec_fpst_arg_zpz,
4135            gen_helper_sve_scvt_ds, a, 0, FPST_FPCR)
4136 
4137 TRANS_FEAT(SCVTF_sd, aa64_sve, gen_gvec_fpst_arg_zpz,
4138            gen_helper_sve_scvt_sd, a, 0, FPST_FPCR)
4139 TRANS_FEAT(SCVTF_dd, aa64_sve, gen_gvec_fpst_arg_zpz,
4140            gen_helper_sve_scvt_dd, a, 0, FPST_FPCR)
4141 
4142 TRANS_FEAT(UCVTF_hh, aa64_sve, gen_gvec_fpst_arg_zpz,
4143            gen_helper_sve_ucvt_hh, a, 0, FPST_FPCR_F16)
4144 TRANS_FEAT(UCVTF_sh, aa64_sve, gen_gvec_fpst_arg_zpz,
4145            gen_helper_sve_ucvt_sh, a, 0, FPST_FPCR_F16)
4146 TRANS_FEAT(UCVTF_dh, aa64_sve, gen_gvec_fpst_arg_zpz,
4147            gen_helper_sve_ucvt_dh, a, 0, FPST_FPCR_F16)
4148 
4149 TRANS_FEAT(UCVTF_ss, aa64_sve, gen_gvec_fpst_arg_zpz,
4150            gen_helper_sve_ucvt_ss, a, 0, FPST_FPCR)
4151 TRANS_FEAT(UCVTF_ds, aa64_sve, gen_gvec_fpst_arg_zpz,
4152            gen_helper_sve_ucvt_ds, a, 0, FPST_FPCR)
4153 TRANS_FEAT(UCVTF_sd, aa64_sve, gen_gvec_fpst_arg_zpz,
4154            gen_helper_sve_ucvt_sd, a, 0, FPST_FPCR)
4155 
4156 TRANS_FEAT(UCVTF_dd, aa64_sve, gen_gvec_fpst_arg_zpz,
4157            gen_helper_sve_ucvt_dd, a, 0, FPST_FPCR)
4158 
4159 /*
4160  *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
4161  */
4162 
4163 /* Subroutine loading a vector register at VOFS of LEN bytes.
4164  * The load should begin at the address Rn + IMM.
4165  */
4166 
4167 void gen_sve_ldr(DisasContext *s, TCGv_ptr base, int vofs,
4168                  int len, int rn, int imm)
4169 {
4170     int len_align = QEMU_ALIGN_DOWN(len, 16);
4171     int len_remain = len % 16;
4172     int nparts = len / 16 + ctpop8(len_remain);
4173     int midx = get_mem_index(s);
4174     TCGv_i64 dirty_addr, clean_addr, t0, t1;
4175     TCGv_i128 t16;
4176 
4177     dirty_addr = tcg_temp_new_i64();
4178     tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm);
4179     clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len, MO_8);
4180 
4181     /*
4182      * Note that unpredicated load/store of vector/predicate registers
4183      * are defined as a stream of bytes, which equates to little-endian
4184      * operations on larger quantities.
4185      * Attempt to keep code expansion to a minimum by limiting the
4186      * amount of unrolling done.
4187      */
4188     if (nparts <= 4) {
4189         int i;
4190 
4191         t0 = tcg_temp_new_i64();
4192         t1 = tcg_temp_new_i64();
4193         t16 = tcg_temp_new_i128();
4194 
4195         for (i = 0; i < len_align; i += 16) {
4196             tcg_gen_qemu_ld_i128(t16, clean_addr, midx,
4197                                  MO_LE | MO_128 | MO_ATOM_NONE);
4198             tcg_gen_extr_i128_i64(t0, t1, t16);
4199             tcg_gen_st_i64(t0, base, vofs + i);
4200             tcg_gen_st_i64(t1, base, vofs + i + 8);
4201             tcg_gen_addi_i64(clean_addr, clean_addr, 16);
4202         }
4203     } else {
4204         TCGLabel *loop = gen_new_label();
4205         TCGv_ptr tp, i = tcg_temp_new_ptr();
4206 
4207         tcg_gen_movi_ptr(i, 0);
4208         gen_set_label(loop);
4209 
4210         t16 = tcg_temp_new_i128();
4211         tcg_gen_qemu_ld_i128(t16, clean_addr, midx,
4212                              MO_LE | MO_128 | MO_ATOM_NONE);
4213         tcg_gen_addi_i64(clean_addr, clean_addr, 16);
4214 
4215         tp = tcg_temp_new_ptr();
4216         tcg_gen_add_ptr(tp, base, i);
4217         tcg_gen_addi_ptr(i, i, 16);
4218 
4219         t0 = tcg_temp_new_i64();
4220         t1 = tcg_temp_new_i64();
4221         tcg_gen_extr_i128_i64(t0, t1, t16);
4222 
4223         tcg_gen_st_i64(t0, tp, vofs);
4224         tcg_gen_st_i64(t1, tp, vofs + 8);
4225 
4226         tcg_gen_brcondi_ptr(TCG_COND_LTU, i, len_align, loop);
4227     }
4228 
4229     /*
4230      * Predicate register loads can be any multiple of 2.
4231      * Note that we still store the entire 64-bit unit into cpu_env.
4232      */
4233     if (len_remain >= 8) {
4234         t0 = tcg_temp_new_i64();
4235         tcg_gen_qemu_ld_i64(t0, clean_addr, midx, MO_LEUQ | MO_ATOM_NONE);
4236         tcg_gen_st_i64(t0, base, vofs + len_align);
4237         len_remain -= 8;
4238         len_align += 8;
4239         if (len_remain) {
4240             tcg_gen_addi_i64(clean_addr, clean_addr, 8);
4241         }
4242     }
4243     if (len_remain) {
4244         t0 = tcg_temp_new_i64();
4245         switch (len_remain) {
4246         case 2:
4247         case 4:
4248         case 8:
4249             tcg_gen_qemu_ld_i64(t0, clean_addr, midx,
4250                                 MO_LE | ctz32(len_remain) | MO_ATOM_NONE);
4251             break;
4252 
4253         case 6:
4254             t1 = tcg_temp_new_i64();
4255             tcg_gen_qemu_ld_i64(t0, clean_addr, midx, MO_LEUL | MO_ATOM_NONE);
4256             tcg_gen_addi_i64(clean_addr, clean_addr, 4);
4257             tcg_gen_qemu_ld_i64(t1, clean_addr, midx, MO_LEUW | MO_ATOM_NONE);
4258             tcg_gen_deposit_i64(t0, t0, t1, 32, 32);
4259             break;
4260 
4261         default:
4262             g_assert_not_reached();
4263         }
4264         tcg_gen_st_i64(t0, base, vofs + len_align);
4265     }
4266 }
4267 
4268 /* Similarly for stores.  */
4269 void gen_sve_str(DisasContext *s, TCGv_ptr base, int vofs,
4270                  int len, int rn, int imm)
4271 {
4272     int len_align = QEMU_ALIGN_DOWN(len, 16);
4273     int len_remain = len % 16;
4274     int nparts = len / 16 + ctpop8(len_remain);
4275     int midx = get_mem_index(s);
4276     TCGv_i64 dirty_addr, clean_addr, t0, t1;
4277     TCGv_i128 t16;
4278 
4279     dirty_addr = tcg_temp_new_i64();
4280     tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm);
4281     clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len, MO_8);
4282 
4283     /* Note that unpredicated load/store of vector/predicate registers
4284      * are defined as a stream of bytes, which equates to little-endian
4285      * operations on larger quantities.  There is no nice way to force
4286      * a little-endian store for aarch64_be-linux-user out of line.
4287      *
4288      * Attempt to keep code expansion to a minimum by limiting the
4289      * amount of unrolling done.
4290      */
4291     if (nparts <= 4) {
4292         int i;
4293 
4294         t0 = tcg_temp_new_i64();
4295         t1 = tcg_temp_new_i64();
4296         t16 = tcg_temp_new_i128();
4297         for (i = 0; i < len_align; i += 8) {
4298             tcg_gen_ld_i64(t0, base, vofs + i);
4299             tcg_gen_ld_i64(t1, base, vofs + i + 8);
4300             tcg_gen_concat_i64_i128(t16, t0, t1);
4301             tcg_gen_qemu_st_i128(t16, clean_addr, midx,
4302                                  MO_LE | MO_128 | MO_ATOM_NONE);
4303             tcg_gen_addi_i64(clean_addr, clean_addr, 16);
4304         }
4305     } else {
4306         TCGLabel *loop = gen_new_label();
4307         TCGv_ptr tp, i = tcg_temp_new_ptr();
4308 
4309         tcg_gen_movi_ptr(i, 0);
4310         gen_set_label(loop);
4311 
4312         t0 = tcg_temp_new_i64();
4313         t1 = tcg_temp_new_i64();
4314         tp = tcg_temp_new_ptr();
4315         tcg_gen_add_ptr(tp, base, i);
4316         tcg_gen_ld_i64(t0, tp, vofs);
4317         tcg_gen_ld_i64(t1, tp, vofs + 8);
4318         tcg_gen_addi_ptr(i, i, 16);
4319 
4320         t16 = tcg_temp_new_i128();
4321         tcg_gen_concat_i64_i128(t16, t0, t1);
4322 
4323         tcg_gen_qemu_st_i128(t16, clean_addr, midx, MO_LEUQ);
4324         tcg_gen_addi_i64(clean_addr, clean_addr, 16);
4325 
4326         tcg_gen_brcondi_ptr(TCG_COND_LTU, i, len_align, loop);
4327     }
4328 
4329     /* Predicate register stores can be any multiple of 2.  */
4330     if (len_remain >= 8) {
4331         t0 = tcg_temp_new_i64();
4332         tcg_gen_ld_i64(t0, base, vofs + len_align);
4333         tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUQ | MO_ATOM_NONE);
4334         len_remain -= 8;
4335         len_align += 8;
4336         if (len_remain) {
4337             tcg_gen_addi_i64(clean_addr, clean_addr, 8);
4338         }
4339     }
4340     if (len_remain) {
4341         t0 = tcg_temp_new_i64();
4342         tcg_gen_ld_i64(t0, base, vofs + len_align);
4343 
4344         switch (len_remain) {
4345         case 2:
4346         case 4:
4347         case 8:
4348             tcg_gen_qemu_st_i64(t0, clean_addr, midx,
4349                                 MO_LE | ctz32(len_remain) | MO_ATOM_NONE);
4350             break;
4351 
4352         case 6:
4353             tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUL | MO_ATOM_NONE);
4354             tcg_gen_addi_i64(clean_addr, clean_addr, 4);
4355             tcg_gen_shri_i64(t0, t0, 32);
4356             tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUW | MO_ATOM_NONE);
4357             break;
4358 
4359         default:
4360             g_assert_not_reached();
4361         }
4362     }
4363 }
4364 
4365 static bool trans_LDR_zri(DisasContext *s, arg_rri *a)
4366 {
4367     if (!dc_isar_feature(aa64_sve, s)) {
4368         return false;
4369     }
4370     if (sve_access_check(s)) {
4371         int size = vec_full_reg_size(s);
4372         int off = vec_full_reg_offset(s, a->rd);
4373         gen_sve_ldr(s, cpu_env, off, size, a->rn, a->imm * size);
4374     }
4375     return true;
4376 }
4377 
4378 static bool trans_LDR_pri(DisasContext *s, arg_rri *a)
4379 {
4380     if (!dc_isar_feature(aa64_sve, s)) {
4381         return false;
4382     }
4383     if (sve_access_check(s)) {
4384         int size = pred_full_reg_size(s);
4385         int off = pred_full_reg_offset(s, a->rd);
4386         gen_sve_ldr(s, cpu_env, off, size, a->rn, a->imm * size);
4387     }
4388     return true;
4389 }
4390 
4391 static bool trans_STR_zri(DisasContext *s, arg_rri *a)
4392 {
4393     if (!dc_isar_feature(aa64_sve, s)) {
4394         return false;
4395     }
4396     if (sve_access_check(s)) {
4397         int size = vec_full_reg_size(s);
4398         int off = vec_full_reg_offset(s, a->rd);
4399         gen_sve_str(s, cpu_env, off, size, a->rn, a->imm * size);
4400     }
4401     return true;
4402 }
4403 
4404 static bool trans_STR_pri(DisasContext *s, arg_rri *a)
4405 {
4406     if (!dc_isar_feature(aa64_sve, s)) {
4407         return false;
4408     }
4409     if (sve_access_check(s)) {
4410         int size = pred_full_reg_size(s);
4411         int off = pred_full_reg_offset(s, a->rd);
4412         gen_sve_str(s, cpu_env, off, size, a->rn, a->imm * size);
4413     }
4414     return true;
4415 }
4416 
4417 /*
4418  *** SVE Memory - Contiguous Load Group
4419  */
4420 
4421 /* The memory mode of the dtype.  */
4422 static const MemOp dtype_mop[16] = {
4423     MO_UB, MO_UB, MO_UB, MO_UB,
4424     MO_SL, MO_UW, MO_UW, MO_UW,
4425     MO_SW, MO_SW, MO_UL, MO_UL,
4426     MO_SB, MO_SB, MO_SB, MO_UQ
4427 };
4428 
4429 #define dtype_msz(x)  (dtype_mop[x] & MO_SIZE)
4430 
4431 /* The vector element size of dtype.  */
4432 static const uint8_t dtype_esz[16] = {
4433     0, 1, 2, 3,
4434     3, 1, 2, 3,
4435     3, 2, 2, 3,
4436     3, 2, 1, 3
4437 };
4438 
4439 static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
4440                        int dtype, uint32_t mte_n, bool is_write,
4441                        gen_helper_gvec_mem *fn)
4442 {
4443     unsigned vsz = vec_full_reg_size(s);
4444     TCGv_ptr t_pg;
4445     int desc = 0;
4446 
4447     /*
4448      * For e.g. LD4, there are not enough arguments to pass all 4
4449      * registers as pointers, so encode the regno into the data field.
4450      * For consistency, do this even for LD1.
4451      */
4452     if (s->mte_active[0]) {
4453         int msz = dtype_msz(dtype);
4454 
4455         desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
4456         desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
4457         desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
4458         desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
4459         desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (mte_n << msz) - 1);
4460         desc <<= SVE_MTEDESC_SHIFT;
4461     } else {
4462         addr = clean_data_tbi(s, addr);
4463     }
4464 
4465     desc = simd_desc(vsz, vsz, zt | desc);
4466     t_pg = tcg_temp_new_ptr();
4467 
4468     tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
4469     fn(cpu_env, t_pg, addr, tcg_constant_i32(desc));
4470 }
4471 
4472 /* Indexed by [mte][be][dtype][nreg] */
4473 static gen_helper_gvec_mem * const ldr_fns[2][2][16][4] = {
4474     { /* mte inactive, little-endian */
4475       { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r,
4476           gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r },
4477         { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL },
4478         { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL },
4479         { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL },
4480 
4481         { gen_helper_sve_ld1sds_le_r, NULL, NULL, NULL },
4482         { gen_helper_sve_ld1hh_le_r, gen_helper_sve_ld2hh_le_r,
4483           gen_helper_sve_ld3hh_le_r, gen_helper_sve_ld4hh_le_r },
4484         { gen_helper_sve_ld1hsu_le_r, NULL, NULL, NULL },
4485         { gen_helper_sve_ld1hdu_le_r, NULL, NULL, NULL },
4486 
4487         { gen_helper_sve_ld1hds_le_r, NULL, NULL, NULL },
4488         { gen_helper_sve_ld1hss_le_r, NULL, NULL, NULL },
4489         { gen_helper_sve_ld1ss_le_r, gen_helper_sve_ld2ss_le_r,
4490           gen_helper_sve_ld3ss_le_r, gen_helper_sve_ld4ss_le_r },
4491         { gen_helper_sve_ld1sdu_le_r, NULL, NULL, NULL },
4492 
4493         { gen_helper_sve_ld1bds_r, NULL, NULL, NULL },
4494         { gen_helper_sve_ld1bss_r, NULL, NULL, NULL },
4495         { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL },
4496         { gen_helper_sve_ld1dd_le_r, gen_helper_sve_ld2dd_le_r,
4497           gen_helper_sve_ld3dd_le_r, gen_helper_sve_ld4dd_le_r } },
4498 
4499       /* mte inactive, big-endian */
4500       { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r,
4501           gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r },
4502         { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL },
4503         { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL },
4504         { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL },
4505 
4506         { gen_helper_sve_ld1sds_be_r, NULL, NULL, NULL },
4507         { gen_helper_sve_ld1hh_be_r, gen_helper_sve_ld2hh_be_r,
4508           gen_helper_sve_ld3hh_be_r, gen_helper_sve_ld4hh_be_r },
4509         { gen_helper_sve_ld1hsu_be_r, NULL, NULL, NULL },
4510         { gen_helper_sve_ld1hdu_be_r, NULL, NULL, NULL },
4511 
4512         { gen_helper_sve_ld1hds_be_r, NULL, NULL, NULL },
4513         { gen_helper_sve_ld1hss_be_r, NULL, NULL, NULL },
4514         { gen_helper_sve_ld1ss_be_r, gen_helper_sve_ld2ss_be_r,
4515           gen_helper_sve_ld3ss_be_r, gen_helper_sve_ld4ss_be_r },
4516         { gen_helper_sve_ld1sdu_be_r, NULL, NULL, NULL },
4517 
4518         { gen_helper_sve_ld1bds_r, NULL, NULL, NULL },
4519         { gen_helper_sve_ld1bss_r, NULL, NULL, NULL },
4520         { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL },
4521         { gen_helper_sve_ld1dd_be_r, gen_helper_sve_ld2dd_be_r,
4522           gen_helper_sve_ld3dd_be_r, gen_helper_sve_ld4dd_be_r } } },
4523 
4524     { /* mte active, little-endian */
4525       { { gen_helper_sve_ld1bb_r_mte,
4526           gen_helper_sve_ld2bb_r_mte,
4527           gen_helper_sve_ld3bb_r_mte,
4528           gen_helper_sve_ld4bb_r_mte },
4529         { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL },
4530         { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL },
4531         { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL },
4532 
4533         { gen_helper_sve_ld1sds_le_r_mte, NULL, NULL, NULL },
4534         { gen_helper_sve_ld1hh_le_r_mte,
4535           gen_helper_sve_ld2hh_le_r_mte,
4536           gen_helper_sve_ld3hh_le_r_mte,
4537           gen_helper_sve_ld4hh_le_r_mte },
4538         { gen_helper_sve_ld1hsu_le_r_mte, NULL, NULL, NULL },
4539         { gen_helper_sve_ld1hdu_le_r_mte, NULL, NULL, NULL },
4540 
4541         { gen_helper_sve_ld1hds_le_r_mte, NULL, NULL, NULL },
4542         { gen_helper_sve_ld1hss_le_r_mte, NULL, NULL, NULL },
4543         { gen_helper_sve_ld1ss_le_r_mte,
4544           gen_helper_sve_ld2ss_le_r_mte,
4545           gen_helper_sve_ld3ss_le_r_mte,
4546           gen_helper_sve_ld4ss_le_r_mte },
4547         { gen_helper_sve_ld1sdu_le_r_mte, NULL, NULL, NULL },
4548 
4549         { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL },
4550         { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL },
4551         { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL },
4552         { gen_helper_sve_ld1dd_le_r_mte,
4553           gen_helper_sve_ld2dd_le_r_mte,
4554           gen_helper_sve_ld3dd_le_r_mte,
4555           gen_helper_sve_ld4dd_le_r_mte } },
4556 
4557       /* mte active, big-endian */
4558       { { gen_helper_sve_ld1bb_r_mte,
4559           gen_helper_sve_ld2bb_r_mte,
4560           gen_helper_sve_ld3bb_r_mte,
4561           gen_helper_sve_ld4bb_r_mte },
4562         { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL },
4563         { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL },
4564         { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL },
4565 
4566         { gen_helper_sve_ld1sds_be_r_mte, NULL, NULL, NULL },
4567         { gen_helper_sve_ld1hh_be_r_mte,
4568           gen_helper_sve_ld2hh_be_r_mte,
4569           gen_helper_sve_ld3hh_be_r_mte,
4570           gen_helper_sve_ld4hh_be_r_mte },
4571         { gen_helper_sve_ld1hsu_be_r_mte, NULL, NULL, NULL },
4572         { gen_helper_sve_ld1hdu_be_r_mte, NULL, NULL, NULL },
4573 
4574         { gen_helper_sve_ld1hds_be_r_mte, NULL, NULL, NULL },
4575         { gen_helper_sve_ld1hss_be_r_mte, NULL, NULL, NULL },
4576         { gen_helper_sve_ld1ss_be_r_mte,
4577           gen_helper_sve_ld2ss_be_r_mte,
4578           gen_helper_sve_ld3ss_be_r_mte,
4579           gen_helper_sve_ld4ss_be_r_mte },
4580         { gen_helper_sve_ld1sdu_be_r_mte, NULL, NULL, NULL },
4581 
4582         { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL },
4583         { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL },
4584         { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL },
4585         { gen_helper_sve_ld1dd_be_r_mte,
4586           gen_helper_sve_ld2dd_be_r_mte,
4587           gen_helper_sve_ld3dd_be_r_mte,
4588           gen_helper_sve_ld4dd_be_r_mte } } },
4589 };
4590 
4591 static void do_ld_zpa(DisasContext *s, int zt, int pg,
4592                       TCGv_i64 addr, int dtype, int nreg)
4593 {
4594     gen_helper_gvec_mem *fn
4595         = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][nreg];
4596 
4597     /*
4598      * While there are holes in the table, they are not
4599      * accessible via the instruction encoding.
4600      */
4601     assert(fn != NULL);
4602     do_mem_zpa(s, zt, pg, addr, dtype, nreg, false, fn);
4603 }
4604 
4605 static bool trans_LD_zprr(DisasContext *s, arg_rprr_load *a)
4606 {
4607     if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) {
4608         return false;
4609     }
4610     if (sve_access_check(s)) {
4611         TCGv_i64 addr = tcg_temp_new_i64();
4612         tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));
4613         tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
4614         do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg);
4615     }
4616     return true;
4617 }
4618 
4619 static bool trans_LD_zpri(DisasContext *s, arg_rpri_load *a)
4620 {
4621     if (!dc_isar_feature(aa64_sve, s)) {
4622         return false;
4623     }
4624     if (sve_access_check(s)) {
4625         int vsz = vec_full_reg_size(s);
4626         int elements = vsz >> dtype_esz[a->dtype];
4627         TCGv_i64 addr = tcg_temp_new_i64();
4628 
4629         tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn),
4630                          (a->imm * elements * (a->nreg + 1))
4631                          << dtype_msz(a->dtype));
4632         do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg);
4633     }
4634     return true;
4635 }
4636 
4637 static bool trans_LDFF1_zprr(DisasContext *s, arg_rprr_load *a)
4638 {
4639     static gen_helper_gvec_mem * const fns[2][2][16] = {
4640         { /* mte inactive, little-endian */
4641           { gen_helper_sve_ldff1bb_r,
4642             gen_helper_sve_ldff1bhu_r,
4643             gen_helper_sve_ldff1bsu_r,
4644             gen_helper_sve_ldff1bdu_r,
4645 
4646             gen_helper_sve_ldff1sds_le_r,
4647             gen_helper_sve_ldff1hh_le_r,
4648             gen_helper_sve_ldff1hsu_le_r,
4649             gen_helper_sve_ldff1hdu_le_r,
4650 
4651             gen_helper_sve_ldff1hds_le_r,
4652             gen_helper_sve_ldff1hss_le_r,
4653             gen_helper_sve_ldff1ss_le_r,
4654             gen_helper_sve_ldff1sdu_le_r,
4655 
4656             gen_helper_sve_ldff1bds_r,
4657             gen_helper_sve_ldff1bss_r,
4658             gen_helper_sve_ldff1bhs_r,
4659             gen_helper_sve_ldff1dd_le_r },
4660 
4661           /* mte inactive, big-endian */
4662           { gen_helper_sve_ldff1bb_r,
4663             gen_helper_sve_ldff1bhu_r,
4664             gen_helper_sve_ldff1bsu_r,
4665             gen_helper_sve_ldff1bdu_r,
4666 
4667             gen_helper_sve_ldff1sds_be_r,
4668             gen_helper_sve_ldff1hh_be_r,
4669             gen_helper_sve_ldff1hsu_be_r,
4670             gen_helper_sve_ldff1hdu_be_r,
4671 
4672             gen_helper_sve_ldff1hds_be_r,
4673             gen_helper_sve_ldff1hss_be_r,
4674             gen_helper_sve_ldff1ss_be_r,
4675             gen_helper_sve_ldff1sdu_be_r,
4676 
4677             gen_helper_sve_ldff1bds_r,
4678             gen_helper_sve_ldff1bss_r,
4679             gen_helper_sve_ldff1bhs_r,
4680             gen_helper_sve_ldff1dd_be_r } },
4681 
4682         { /* mte active, little-endian */
4683           { gen_helper_sve_ldff1bb_r_mte,
4684             gen_helper_sve_ldff1bhu_r_mte,
4685             gen_helper_sve_ldff1bsu_r_mte,
4686             gen_helper_sve_ldff1bdu_r_mte,
4687 
4688             gen_helper_sve_ldff1sds_le_r_mte,
4689             gen_helper_sve_ldff1hh_le_r_mte,
4690             gen_helper_sve_ldff1hsu_le_r_mte,
4691             gen_helper_sve_ldff1hdu_le_r_mte,
4692 
4693             gen_helper_sve_ldff1hds_le_r_mte,
4694             gen_helper_sve_ldff1hss_le_r_mte,
4695             gen_helper_sve_ldff1ss_le_r_mte,
4696             gen_helper_sve_ldff1sdu_le_r_mte,
4697 
4698             gen_helper_sve_ldff1bds_r_mte,
4699             gen_helper_sve_ldff1bss_r_mte,
4700             gen_helper_sve_ldff1bhs_r_mte,
4701             gen_helper_sve_ldff1dd_le_r_mte },
4702 
4703           /* mte active, big-endian */
4704           { gen_helper_sve_ldff1bb_r_mte,
4705             gen_helper_sve_ldff1bhu_r_mte,
4706             gen_helper_sve_ldff1bsu_r_mte,
4707             gen_helper_sve_ldff1bdu_r_mte,
4708 
4709             gen_helper_sve_ldff1sds_be_r_mte,
4710             gen_helper_sve_ldff1hh_be_r_mte,
4711             gen_helper_sve_ldff1hsu_be_r_mte,
4712             gen_helper_sve_ldff1hdu_be_r_mte,
4713 
4714             gen_helper_sve_ldff1hds_be_r_mte,
4715             gen_helper_sve_ldff1hss_be_r_mte,
4716             gen_helper_sve_ldff1ss_be_r_mte,
4717             gen_helper_sve_ldff1sdu_be_r_mte,
4718 
4719             gen_helper_sve_ldff1bds_r_mte,
4720             gen_helper_sve_ldff1bss_r_mte,
4721             gen_helper_sve_ldff1bhs_r_mte,
4722             gen_helper_sve_ldff1dd_be_r_mte } },
4723     };
4724 
4725     if (!dc_isar_feature(aa64_sve, s)) {
4726         return false;
4727     }
4728     s->is_nonstreaming = true;
4729     if (sve_access_check(s)) {
4730         TCGv_i64 addr = tcg_temp_new_i64();
4731         tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));
4732         tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
4733         do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false,
4734                    fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]);
4735     }
4736     return true;
4737 }
4738 
4739 static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a)
4740 {
4741     static gen_helper_gvec_mem * const fns[2][2][16] = {
4742         { /* mte inactive, little-endian */
4743           { gen_helper_sve_ldnf1bb_r,
4744             gen_helper_sve_ldnf1bhu_r,
4745             gen_helper_sve_ldnf1bsu_r,
4746             gen_helper_sve_ldnf1bdu_r,
4747 
4748             gen_helper_sve_ldnf1sds_le_r,
4749             gen_helper_sve_ldnf1hh_le_r,
4750             gen_helper_sve_ldnf1hsu_le_r,
4751             gen_helper_sve_ldnf1hdu_le_r,
4752 
4753             gen_helper_sve_ldnf1hds_le_r,
4754             gen_helper_sve_ldnf1hss_le_r,
4755             gen_helper_sve_ldnf1ss_le_r,
4756             gen_helper_sve_ldnf1sdu_le_r,
4757 
4758             gen_helper_sve_ldnf1bds_r,
4759             gen_helper_sve_ldnf1bss_r,
4760             gen_helper_sve_ldnf1bhs_r,
4761             gen_helper_sve_ldnf1dd_le_r },
4762 
4763           /* mte inactive, big-endian */
4764           { gen_helper_sve_ldnf1bb_r,
4765             gen_helper_sve_ldnf1bhu_r,
4766             gen_helper_sve_ldnf1bsu_r,
4767             gen_helper_sve_ldnf1bdu_r,
4768 
4769             gen_helper_sve_ldnf1sds_be_r,
4770             gen_helper_sve_ldnf1hh_be_r,
4771             gen_helper_sve_ldnf1hsu_be_r,
4772             gen_helper_sve_ldnf1hdu_be_r,
4773 
4774             gen_helper_sve_ldnf1hds_be_r,
4775             gen_helper_sve_ldnf1hss_be_r,
4776             gen_helper_sve_ldnf1ss_be_r,
4777             gen_helper_sve_ldnf1sdu_be_r,
4778 
4779             gen_helper_sve_ldnf1bds_r,
4780             gen_helper_sve_ldnf1bss_r,
4781             gen_helper_sve_ldnf1bhs_r,
4782             gen_helper_sve_ldnf1dd_be_r } },
4783 
4784         { /* mte inactive, little-endian */
4785           { gen_helper_sve_ldnf1bb_r_mte,
4786             gen_helper_sve_ldnf1bhu_r_mte,
4787             gen_helper_sve_ldnf1bsu_r_mte,
4788             gen_helper_sve_ldnf1bdu_r_mte,
4789 
4790             gen_helper_sve_ldnf1sds_le_r_mte,
4791             gen_helper_sve_ldnf1hh_le_r_mte,
4792             gen_helper_sve_ldnf1hsu_le_r_mte,
4793             gen_helper_sve_ldnf1hdu_le_r_mte,
4794 
4795             gen_helper_sve_ldnf1hds_le_r_mte,
4796             gen_helper_sve_ldnf1hss_le_r_mte,
4797             gen_helper_sve_ldnf1ss_le_r_mte,
4798             gen_helper_sve_ldnf1sdu_le_r_mte,
4799 
4800             gen_helper_sve_ldnf1bds_r_mte,
4801             gen_helper_sve_ldnf1bss_r_mte,
4802             gen_helper_sve_ldnf1bhs_r_mte,
4803             gen_helper_sve_ldnf1dd_le_r_mte },
4804 
4805           /* mte inactive, big-endian */
4806           { gen_helper_sve_ldnf1bb_r_mte,
4807             gen_helper_sve_ldnf1bhu_r_mte,
4808             gen_helper_sve_ldnf1bsu_r_mte,
4809             gen_helper_sve_ldnf1bdu_r_mte,
4810 
4811             gen_helper_sve_ldnf1sds_be_r_mte,
4812             gen_helper_sve_ldnf1hh_be_r_mte,
4813             gen_helper_sve_ldnf1hsu_be_r_mte,
4814             gen_helper_sve_ldnf1hdu_be_r_mte,
4815 
4816             gen_helper_sve_ldnf1hds_be_r_mte,
4817             gen_helper_sve_ldnf1hss_be_r_mte,
4818             gen_helper_sve_ldnf1ss_be_r_mte,
4819             gen_helper_sve_ldnf1sdu_be_r_mte,
4820 
4821             gen_helper_sve_ldnf1bds_r_mte,
4822             gen_helper_sve_ldnf1bss_r_mte,
4823             gen_helper_sve_ldnf1bhs_r_mte,
4824             gen_helper_sve_ldnf1dd_be_r_mte } },
4825     };
4826 
4827     if (!dc_isar_feature(aa64_sve, s)) {
4828         return false;
4829     }
4830     s->is_nonstreaming = true;
4831     if (sve_access_check(s)) {
4832         int vsz = vec_full_reg_size(s);
4833         int elements = vsz >> dtype_esz[a->dtype];
4834         int off = (a->imm * elements) << dtype_msz(a->dtype);
4835         TCGv_i64 addr = tcg_temp_new_i64();
4836 
4837         tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), off);
4838         do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false,
4839                    fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]);
4840     }
4841     return true;
4842 }
4843 
4844 static void do_ldrq(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype)
4845 {
4846     unsigned vsz = vec_full_reg_size(s);
4847     TCGv_ptr t_pg;
4848     int poff;
4849 
4850     /* Load the first quadword using the normal predicated load helpers.  */
4851     poff = pred_full_reg_offset(s, pg);
4852     if (vsz > 16) {
4853         /*
4854          * Zero-extend the first 16 bits of the predicate into a temporary.
4855          * This avoids triggering an assert making sure we don't have bits
4856          * set within a predicate beyond VQ, but we have lowered VQ to 1
4857          * for this load operation.
4858          */
4859         TCGv_i64 tmp = tcg_temp_new_i64();
4860 #if HOST_BIG_ENDIAN
4861         poff += 6;
4862 #endif
4863         tcg_gen_ld16u_i64(tmp, cpu_env, poff);
4864 
4865         poff = offsetof(CPUARMState, vfp.preg_tmp);
4866         tcg_gen_st_i64(tmp, cpu_env, poff);
4867     }
4868 
4869     t_pg = tcg_temp_new_ptr();
4870     tcg_gen_addi_ptr(t_pg, cpu_env, poff);
4871 
4872     gen_helper_gvec_mem *fn
4873         = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0];
4874     fn(cpu_env, t_pg, addr, tcg_constant_i32(simd_desc(16, 16, zt)));
4875 
4876     /* Replicate that first quadword.  */
4877     if (vsz > 16) {
4878         int doff = vec_full_reg_offset(s, zt);
4879         tcg_gen_gvec_dup_mem(4, doff + 16, doff, vsz - 16, vsz - 16);
4880     }
4881 }
4882 
4883 static bool trans_LD1RQ_zprr(DisasContext *s, arg_rprr_load *a)
4884 {
4885     if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) {
4886         return false;
4887     }
4888     if (sve_access_check(s)) {
4889         int msz = dtype_msz(a->dtype);
4890         TCGv_i64 addr = tcg_temp_new_i64();
4891         tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), msz);
4892         tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
4893         do_ldrq(s, a->rd, a->pg, addr, a->dtype);
4894     }
4895     return true;
4896 }
4897 
4898 static bool trans_LD1RQ_zpri(DisasContext *s, arg_rpri_load *a)
4899 {
4900     if (!dc_isar_feature(aa64_sve, s)) {
4901         return false;
4902     }
4903     if (sve_access_check(s)) {
4904         TCGv_i64 addr = tcg_temp_new_i64();
4905         tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 16);
4906         do_ldrq(s, a->rd, a->pg, addr, a->dtype);
4907     }
4908     return true;
4909 }
4910 
4911 static void do_ldro(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype)
4912 {
4913     unsigned vsz = vec_full_reg_size(s);
4914     unsigned vsz_r32;
4915     TCGv_ptr t_pg;
4916     int poff, doff;
4917 
4918     if (vsz < 32) {
4919         /*
4920          * Note that this UNDEFINED check comes after CheckSVEEnabled()
4921          * in the ARM pseudocode, which is the sve_access_check() done
4922          * in our caller.  We should not now return false from the caller.
4923          */
4924         unallocated_encoding(s);
4925         return;
4926     }
4927 
4928     /* Load the first octaword using the normal predicated load helpers.  */
4929 
4930     poff = pred_full_reg_offset(s, pg);
4931     if (vsz > 32) {
4932         /*
4933          * Zero-extend the first 32 bits of the predicate into a temporary.
4934          * This avoids triggering an assert making sure we don't have bits
4935          * set within a predicate beyond VQ, but we have lowered VQ to 2
4936          * for this load operation.
4937          */
4938         TCGv_i64 tmp = tcg_temp_new_i64();
4939 #if HOST_BIG_ENDIAN
4940         poff += 4;
4941 #endif
4942         tcg_gen_ld32u_i64(tmp, cpu_env, poff);
4943 
4944         poff = offsetof(CPUARMState, vfp.preg_tmp);
4945         tcg_gen_st_i64(tmp, cpu_env, poff);
4946     }
4947 
4948     t_pg = tcg_temp_new_ptr();
4949     tcg_gen_addi_ptr(t_pg, cpu_env, poff);
4950 
4951     gen_helper_gvec_mem *fn
4952         = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0];
4953     fn(cpu_env, t_pg, addr, tcg_constant_i32(simd_desc(32, 32, zt)));
4954 
4955     /*
4956      * Replicate that first octaword.
4957      * The replication happens in units of 32; if the full vector size
4958      * is not a multiple of 32, the final bits are zeroed.
4959      */
4960     doff = vec_full_reg_offset(s, zt);
4961     vsz_r32 = QEMU_ALIGN_DOWN(vsz, 32);
4962     if (vsz >= 64) {
4963         tcg_gen_gvec_dup_mem(5, doff + 32, doff, vsz_r32 - 32, vsz_r32 - 32);
4964     }
4965     vsz -= vsz_r32;
4966     if (vsz) {
4967         tcg_gen_gvec_dup_imm(MO_64, doff + vsz_r32, vsz, vsz, 0);
4968     }
4969 }
4970 
4971 static bool trans_LD1RO_zprr(DisasContext *s, arg_rprr_load *a)
4972 {
4973     if (!dc_isar_feature(aa64_sve_f64mm, s)) {
4974         return false;
4975     }
4976     if (a->rm == 31) {
4977         return false;
4978     }
4979     s->is_nonstreaming = true;
4980     if (sve_access_check(s)) {
4981         TCGv_i64 addr = tcg_temp_new_i64();
4982         tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));
4983         tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
4984         do_ldro(s, a->rd, a->pg, addr, a->dtype);
4985     }
4986     return true;
4987 }
4988 
4989 static bool trans_LD1RO_zpri(DisasContext *s, arg_rpri_load *a)
4990 {
4991     if (!dc_isar_feature(aa64_sve_f64mm, s)) {
4992         return false;
4993     }
4994     s->is_nonstreaming = true;
4995     if (sve_access_check(s)) {
4996         TCGv_i64 addr = tcg_temp_new_i64();
4997         tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 32);
4998         do_ldro(s, a->rd, a->pg, addr, a->dtype);
4999     }
5000     return true;
5001 }
5002 
5003 /* Load and broadcast element.  */
5004 static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a)
5005 {
5006     unsigned vsz = vec_full_reg_size(s);
5007     unsigned psz = pred_full_reg_size(s);
5008     unsigned esz = dtype_esz[a->dtype];
5009     unsigned msz = dtype_msz(a->dtype);
5010     TCGLabel *over;
5011     TCGv_i64 temp, clean_addr;
5012     MemOp memop;
5013 
5014     if (!dc_isar_feature(aa64_sve, s)) {
5015         return false;
5016     }
5017     if (!sve_access_check(s)) {
5018         return true;
5019     }
5020 
5021     over = gen_new_label();
5022 
5023     /* If the guarding predicate has no bits set, no load occurs.  */
5024     if (psz <= 8) {
5025         /* Reduce the pred_esz_masks value simply to reduce the
5026          * size of the code generated here.
5027          */
5028         uint64_t psz_mask = MAKE_64BIT_MASK(0, psz * 8);
5029         temp = tcg_temp_new_i64();
5030         tcg_gen_ld_i64(temp, cpu_env, pred_full_reg_offset(s, a->pg));
5031         tcg_gen_andi_i64(temp, temp, pred_esz_masks[esz] & psz_mask);
5032         tcg_gen_brcondi_i64(TCG_COND_EQ, temp, 0, over);
5033     } else {
5034         TCGv_i32 t32 = tcg_temp_new_i32();
5035         find_last_active(s, t32, esz, a->pg);
5036         tcg_gen_brcondi_i32(TCG_COND_LT, t32, 0, over);
5037     }
5038 
5039     /* Load the data.  */
5040     temp = tcg_temp_new_i64();
5041     tcg_gen_addi_i64(temp, cpu_reg_sp(s, a->rn), a->imm << msz);
5042 
5043     memop = finalize_memop(s, dtype_mop[a->dtype]);
5044     clean_addr = gen_mte_check1(s, temp, false, true, memop);
5045     tcg_gen_qemu_ld_i64(temp, clean_addr, get_mem_index(s), memop);
5046 
5047     /* Broadcast to *all* elements.  */
5048     tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd),
5049                          vsz, vsz, temp);
5050 
5051     /* Zero the inactive elements.  */
5052     gen_set_label(over);
5053     return do_movz_zpz(s, a->rd, a->rd, a->pg, esz, false);
5054 }
5055 
5056 static void do_st_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
5057                       int msz, int esz, int nreg)
5058 {
5059     static gen_helper_gvec_mem * const fn_single[2][2][4][4] = {
5060         { { { gen_helper_sve_st1bb_r,
5061               gen_helper_sve_st1bh_r,
5062               gen_helper_sve_st1bs_r,
5063               gen_helper_sve_st1bd_r },
5064             { NULL,
5065               gen_helper_sve_st1hh_le_r,
5066               gen_helper_sve_st1hs_le_r,
5067               gen_helper_sve_st1hd_le_r },
5068             { NULL, NULL,
5069               gen_helper_sve_st1ss_le_r,
5070               gen_helper_sve_st1sd_le_r },
5071             { NULL, NULL, NULL,
5072               gen_helper_sve_st1dd_le_r } },
5073           { { gen_helper_sve_st1bb_r,
5074               gen_helper_sve_st1bh_r,
5075               gen_helper_sve_st1bs_r,
5076               gen_helper_sve_st1bd_r },
5077             { NULL,
5078               gen_helper_sve_st1hh_be_r,
5079               gen_helper_sve_st1hs_be_r,
5080               gen_helper_sve_st1hd_be_r },
5081             { NULL, NULL,
5082               gen_helper_sve_st1ss_be_r,
5083               gen_helper_sve_st1sd_be_r },
5084             { NULL, NULL, NULL,
5085               gen_helper_sve_st1dd_be_r } } },
5086 
5087         { { { gen_helper_sve_st1bb_r_mte,
5088               gen_helper_sve_st1bh_r_mte,
5089               gen_helper_sve_st1bs_r_mte,
5090               gen_helper_sve_st1bd_r_mte },
5091             { NULL,
5092               gen_helper_sve_st1hh_le_r_mte,
5093               gen_helper_sve_st1hs_le_r_mte,
5094               gen_helper_sve_st1hd_le_r_mte },
5095             { NULL, NULL,
5096               gen_helper_sve_st1ss_le_r_mte,
5097               gen_helper_sve_st1sd_le_r_mte },
5098             { NULL, NULL, NULL,
5099               gen_helper_sve_st1dd_le_r_mte } },
5100           { { gen_helper_sve_st1bb_r_mte,
5101               gen_helper_sve_st1bh_r_mte,
5102               gen_helper_sve_st1bs_r_mte,
5103               gen_helper_sve_st1bd_r_mte },
5104             { NULL,
5105               gen_helper_sve_st1hh_be_r_mte,
5106               gen_helper_sve_st1hs_be_r_mte,
5107               gen_helper_sve_st1hd_be_r_mte },
5108             { NULL, NULL,
5109               gen_helper_sve_st1ss_be_r_mte,
5110               gen_helper_sve_st1sd_be_r_mte },
5111             { NULL, NULL, NULL,
5112               gen_helper_sve_st1dd_be_r_mte } } },
5113     };
5114     static gen_helper_gvec_mem * const fn_multiple[2][2][3][4] = {
5115         { { { gen_helper_sve_st2bb_r,
5116               gen_helper_sve_st2hh_le_r,
5117               gen_helper_sve_st2ss_le_r,
5118               gen_helper_sve_st2dd_le_r },
5119             { gen_helper_sve_st3bb_r,
5120               gen_helper_sve_st3hh_le_r,
5121               gen_helper_sve_st3ss_le_r,
5122               gen_helper_sve_st3dd_le_r },
5123             { gen_helper_sve_st4bb_r,
5124               gen_helper_sve_st4hh_le_r,
5125               gen_helper_sve_st4ss_le_r,
5126               gen_helper_sve_st4dd_le_r } },
5127           { { gen_helper_sve_st2bb_r,
5128               gen_helper_sve_st2hh_be_r,
5129               gen_helper_sve_st2ss_be_r,
5130               gen_helper_sve_st2dd_be_r },
5131             { gen_helper_sve_st3bb_r,
5132               gen_helper_sve_st3hh_be_r,
5133               gen_helper_sve_st3ss_be_r,
5134               gen_helper_sve_st3dd_be_r },
5135             { gen_helper_sve_st4bb_r,
5136               gen_helper_sve_st4hh_be_r,
5137               gen_helper_sve_st4ss_be_r,
5138               gen_helper_sve_st4dd_be_r } } },
5139         { { { gen_helper_sve_st2bb_r_mte,
5140               gen_helper_sve_st2hh_le_r_mte,
5141               gen_helper_sve_st2ss_le_r_mte,
5142               gen_helper_sve_st2dd_le_r_mte },
5143             { gen_helper_sve_st3bb_r_mte,
5144               gen_helper_sve_st3hh_le_r_mte,
5145               gen_helper_sve_st3ss_le_r_mte,
5146               gen_helper_sve_st3dd_le_r_mte },
5147             { gen_helper_sve_st4bb_r_mte,
5148               gen_helper_sve_st4hh_le_r_mte,
5149               gen_helper_sve_st4ss_le_r_mte,
5150               gen_helper_sve_st4dd_le_r_mte } },
5151           { { gen_helper_sve_st2bb_r_mte,
5152               gen_helper_sve_st2hh_be_r_mte,
5153               gen_helper_sve_st2ss_be_r_mte,
5154               gen_helper_sve_st2dd_be_r_mte },
5155             { gen_helper_sve_st3bb_r_mte,
5156               gen_helper_sve_st3hh_be_r_mte,
5157               gen_helper_sve_st3ss_be_r_mte,
5158               gen_helper_sve_st3dd_be_r_mte },
5159             { gen_helper_sve_st4bb_r_mte,
5160               gen_helper_sve_st4hh_be_r_mte,
5161               gen_helper_sve_st4ss_be_r_mte,
5162               gen_helper_sve_st4dd_be_r_mte } } },
5163     };
5164     gen_helper_gvec_mem *fn;
5165     int be = s->be_data == MO_BE;
5166 
5167     if (nreg == 0) {
5168         /* ST1 */
5169         fn = fn_single[s->mte_active[0]][be][msz][esz];
5170         nreg = 1;
5171     } else {
5172         /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
5173         assert(msz == esz);
5174         fn = fn_multiple[s->mte_active[0]][be][nreg - 1][msz];
5175     }
5176     assert(fn != NULL);
5177     do_mem_zpa(s, zt, pg, addr, msz_dtype(s, msz), nreg, true, fn);
5178 }
5179 
5180 static bool trans_ST_zprr(DisasContext *s, arg_rprr_store *a)
5181 {
5182     if (!dc_isar_feature(aa64_sve, s)) {
5183         return false;
5184     }
5185     if (a->rm == 31 || a->msz > a->esz) {
5186         return false;
5187     }
5188     if (sve_access_check(s)) {
5189         TCGv_i64 addr = tcg_temp_new_i64();
5190         tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), a->msz);
5191         tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
5192         do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg);
5193     }
5194     return true;
5195 }
5196 
5197 static bool trans_ST_zpri(DisasContext *s, arg_rpri_store *a)
5198 {
5199     if (!dc_isar_feature(aa64_sve, s)) {
5200         return false;
5201     }
5202     if (a->msz > a->esz) {
5203         return false;
5204     }
5205     if (sve_access_check(s)) {
5206         int vsz = vec_full_reg_size(s);
5207         int elements = vsz >> a->esz;
5208         TCGv_i64 addr = tcg_temp_new_i64();
5209 
5210         tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn),
5211                          (a->imm * elements * (a->nreg + 1)) << a->msz);
5212         do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg);
5213     }
5214     return true;
5215 }
5216 
5217 /*
5218  *** SVE gather loads / scatter stores
5219  */
5220 
5221 static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm,
5222                        int scale, TCGv_i64 scalar, int msz, bool is_write,
5223                        gen_helper_gvec_mem_scatter *fn)
5224 {
5225     unsigned vsz = vec_full_reg_size(s);
5226     TCGv_ptr t_zm = tcg_temp_new_ptr();
5227     TCGv_ptr t_pg = tcg_temp_new_ptr();
5228     TCGv_ptr t_zt = tcg_temp_new_ptr();
5229     int desc = 0;
5230 
5231     if (s->mte_active[0]) {
5232         desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
5233         desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
5234         desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
5235         desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
5236         desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (1 << msz) - 1);
5237         desc <<= SVE_MTEDESC_SHIFT;
5238     }
5239     desc = simd_desc(vsz, vsz, desc | scale);
5240 
5241     tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
5242     tcg_gen_addi_ptr(t_zm, cpu_env, vec_full_reg_offset(s, zm));
5243     tcg_gen_addi_ptr(t_zt, cpu_env, vec_full_reg_offset(s, zt));
5244     fn(cpu_env, t_zt, t_pg, t_zm, scalar, tcg_constant_i32(desc));
5245 }
5246 
5247 /* Indexed by [mte][be][ff][xs][u][msz].  */
5248 static gen_helper_gvec_mem_scatter * const
5249 gather_load_fn32[2][2][2][2][2][3] = {
5250     { /* MTE Inactive */
5251         { /* Little-endian */
5252             { { { gen_helper_sve_ldbss_zsu,
5253                   gen_helper_sve_ldhss_le_zsu,
5254                   NULL, },
5255                 { gen_helper_sve_ldbsu_zsu,
5256                   gen_helper_sve_ldhsu_le_zsu,
5257                   gen_helper_sve_ldss_le_zsu, } },
5258               { { gen_helper_sve_ldbss_zss,
5259                   gen_helper_sve_ldhss_le_zss,
5260                   NULL, },
5261                 { gen_helper_sve_ldbsu_zss,
5262                   gen_helper_sve_ldhsu_le_zss,
5263                   gen_helper_sve_ldss_le_zss, } } },
5264 
5265             /* First-fault */
5266             { { { gen_helper_sve_ldffbss_zsu,
5267                   gen_helper_sve_ldffhss_le_zsu,
5268                   NULL, },
5269                 { gen_helper_sve_ldffbsu_zsu,
5270                   gen_helper_sve_ldffhsu_le_zsu,
5271                   gen_helper_sve_ldffss_le_zsu, } },
5272               { { gen_helper_sve_ldffbss_zss,
5273                   gen_helper_sve_ldffhss_le_zss,
5274                   NULL, },
5275                 { gen_helper_sve_ldffbsu_zss,
5276                   gen_helper_sve_ldffhsu_le_zss,
5277                   gen_helper_sve_ldffss_le_zss, } } } },
5278 
5279         { /* Big-endian */
5280             { { { gen_helper_sve_ldbss_zsu,
5281                   gen_helper_sve_ldhss_be_zsu,
5282                   NULL, },
5283                 { gen_helper_sve_ldbsu_zsu,
5284                   gen_helper_sve_ldhsu_be_zsu,
5285                   gen_helper_sve_ldss_be_zsu, } },
5286               { { gen_helper_sve_ldbss_zss,
5287                   gen_helper_sve_ldhss_be_zss,
5288                   NULL, },
5289                 { gen_helper_sve_ldbsu_zss,
5290                   gen_helper_sve_ldhsu_be_zss,
5291                   gen_helper_sve_ldss_be_zss, } } },
5292 
5293             /* First-fault */
5294             { { { gen_helper_sve_ldffbss_zsu,
5295                   gen_helper_sve_ldffhss_be_zsu,
5296                   NULL, },
5297                 { gen_helper_sve_ldffbsu_zsu,
5298                   gen_helper_sve_ldffhsu_be_zsu,
5299                   gen_helper_sve_ldffss_be_zsu, } },
5300               { { gen_helper_sve_ldffbss_zss,
5301                   gen_helper_sve_ldffhss_be_zss,
5302                   NULL, },
5303                 { gen_helper_sve_ldffbsu_zss,
5304                   gen_helper_sve_ldffhsu_be_zss,
5305                   gen_helper_sve_ldffss_be_zss, } } } } },
5306     { /* MTE Active */
5307         { /* Little-endian */
5308             { { { gen_helper_sve_ldbss_zsu_mte,
5309                   gen_helper_sve_ldhss_le_zsu_mte,
5310                   NULL, },
5311                 { gen_helper_sve_ldbsu_zsu_mte,
5312                   gen_helper_sve_ldhsu_le_zsu_mte,
5313                   gen_helper_sve_ldss_le_zsu_mte, } },
5314               { { gen_helper_sve_ldbss_zss_mte,
5315                   gen_helper_sve_ldhss_le_zss_mte,
5316                   NULL, },
5317                 { gen_helper_sve_ldbsu_zss_mte,
5318                   gen_helper_sve_ldhsu_le_zss_mte,
5319                   gen_helper_sve_ldss_le_zss_mte, } } },
5320 
5321             /* First-fault */
5322             { { { gen_helper_sve_ldffbss_zsu_mte,
5323                   gen_helper_sve_ldffhss_le_zsu_mte,
5324                   NULL, },
5325                 { gen_helper_sve_ldffbsu_zsu_mte,
5326                   gen_helper_sve_ldffhsu_le_zsu_mte,
5327                   gen_helper_sve_ldffss_le_zsu_mte, } },
5328               { { gen_helper_sve_ldffbss_zss_mte,
5329                   gen_helper_sve_ldffhss_le_zss_mte,
5330                   NULL, },
5331                 { gen_helper_sve_ldffbsu_zss_mte,
5332                   gen_helper_sve_ldffhsu_le_zss_mte,
5333                   gen_helper_sve_ldffss_le_zss_mte, } } } },
5334 
5335         { /* Big-endian */
5336             { { { gen_helper_sve_ldbss_zsu_mte,
5337                   gen_helper_sve_ldhss_be_zsu_mte,
5338                   NULL, },
5339                 { gen_helper_sve_ldbsu_zsu_mte,
5340                   gen_helper_sve_ldhsu_be_zsu_mte,
5341                   gen_helper_sve_ldss_be_zsu_mte, } },
5342               { { gen_helper_sve_ldbss_zss_mte,
5343                   gen_helper_sve_ldhss_be_zss_mte,
5344                   NULL, },
5345                 { gen_helper_sve_ldbsu_zss_mte,
5346                   gen_helper_sve_ldhsu_be_zss_mte,
5347                   gen_helper_sve_ldss_be_zss_mte, } } },
5348 
5349             /* First-fault */
5350             { { { gen_helper_sve_ldffbss_zsu_mte,
5351                   gen_helper_sve_ldffhss_be_zsu_mte,
5352                   NULL, },
5353                 { gen_helper_sve_ldffbsu_zsu_mte,
5354                   gen_helper_sve_ldffhsu_be_zsu_mte,
5355                   gen_helper_sve_ldffss_be_zsu_mte, } },
5356               { { gen_helper_sve_ldffbss_zss_mte,
5357                   gen_helper_sve_ldffhss_be_zss_mte,
5358                   NULL, },
5359                 { gen_helper_sve_ldffbsu_zss_mte,
5360                   gen_helper_sve_ldffhsu_be_zss_mte,
5361                   gen_helper_sve_ldffss_be_zss_mte, } } } } },
5362 };
5363 
5364 /* Note that we overload xs=2 to indicate 64-bit offset.  */
5365 static gen_helper_gvec_mem_scatter * const
5366 gather_load_fn64[2][2][2][3][2][4] = {
5367     { /* MTE Inactive */
5368         { /* Little-endian */
5369             { { { gen_helper_sve_ldbds_zsu,
5370                   gen_helper_sve_ldhds_le_zsu,
5371                   gen_helper_sve_ldsds_le_zsu,
5372                   NULL, },
5373                 { gen_helper_sve_ldbdu_zsu,
5374                   gen_helper_sve_ldhdu_le_zsu,
5375                   gen_helper_sve_ldsdu_le_zsu,
5376                   gen_helper_sve_lddd_le_zsu, } },
5377               { { gen_helper_sve_ldbds_zss,
5378                   gen_helper_sve_ldhds_le_zss,
5379                   gen_helper_sve_ldsds_le_zss,
5380                   NULL, },
5381                 { gen_helper_sve_ldbdu_zss,
5382                   gen_helper_sve_ldhdu_le_zss,
5383                   gen_helper_sve_ldsdu_le_zss,
5384                   gen_helper_sve_lddd_le_zss, } },
5385               { { gen_helper_sve_ldbds_zd,
5386                   gen_helper_sve_ldhds_le_zd,
5387                   gen_helper_sve_ldsds_le_zd,
5388                   NULL, },
5389                 { gen_helper_sve_ldbdu_zd,
5390                   gen_helper_sve_ldhdu_le_zd,
5391                   gen_helper_sve_ldsdu_le_zd,
5392                   gen_helper_sve_lddd_le_zd, } } },
5393 
5394             /* First-fault */
5395             { { { gen_helper_sve_ldffbds_zsu,
5396                   gen_helper_sve_ldffhds_le_zsu,
5397                   gen_helper_sve_ldffsds_le_zsu,
5398                   NULL, },
5399                 { gen_helper_sve_ldffbdu_zsu,
5400                   gen_helper_sve_ldffhdu_le_zsu,
5401                   gen_helper_sve_ldffsdu_le_zsu,
5402                   gen_helper_sve_ldffdd_le_zsu, } },
5403               { { gen_helper_sve_ldffbds_zss,
5404                   gen_helper_sve_ldffhds_le_zss,
5405                   gen_helper_sve_ldffsds_le_zss,
5406                   NULL, },
5407                 { gen_helper_sve_ldffbdu_zss,
5408                   gen_helper_sve_ldffhdu_le_zss,
5409                   gen_helper_sve_ldffsdu_le_zss,
5410                   gen_helper_sve_ldffdd_le_zss, } },
5411               { { gen_helper_sve_ldffbds_zd,
5412                   gen_helper_sve_ldffhds_le_zd,
5413                   gen_helper_sve_ldffsds_le_zd,
5414                   NULL, },
5415                 { gen_helper_sve_ldffbdu_zd,
5416                   gen_helper_sve_ldffhdu_le_zd,
5417                   gen_helper_sve_ldffsdu_le_zd,
5418                   gen_helper_sve_ldffdd_le_zd, } } } },
5419         { /* Big-endian */
5420             { { { gen_helper_sve_ldbds_zsu,
5421                   gen_helper_sve_ldhds_be_zsu,
5422                   gen_helper_sve_ldsds_be_zsu,
5423                   NULL, },
5424                 { gen_helper_sve_ldbdu_zsu,
5425                   gen_helper_sve_ldhdu_be_zsu,
5426                   gen_helper_sve_ldsdu_be_zsu,
5427                   gen_helper_sve_lddd_be_zsu, } },
5428               { { gen_helper_sve_ldbds_zss,
5429                   gen_helper_sve_ldhds_be_zss,
5430                   gen_helper_sve_ldsds_be_zss,
5431                   NULL, },
5432                 { gen_helper_sve_ldbdu_zss,
5433                   gen_helper_sve_ldhdu_be_zss,
5434                   gen_helper_sve_ldsdu_be_zss,
5435                   gen_helper_sve_lddd_be_zss, } },
5436               { { gen_helper_sve_ldbds_zd,
5437                   gen_helper_sve_ldhds_be_zd,
5438                   gen_helper_sve_ldsds_be_zd,
5439                   NULL, },
5440                 { gen_helper_sve_ldbdu_zd,
5441                   gen_helper_sve_ldhdu_be_zd,
5442                   gen_helper_sve_ldsdu_be_zd,
5443                   gen_helper_sve_lddd_be_zd, } } },
5444 
5445             /* First-fault */
5446             { { { gen_helper_sve_ldffbds_zsu,
5447                   gen_helper_sve_ldffhds_be_zsu,
5448                   gen_helper_sve_ldffsds_be_zsu,
5449                   NULL, },
5450                 { gen_helper_sve_ldffbdu_zsu,
5451                   gen_helper_sve_ldffhdu_be_zsu,
5452                   gen_helper_sve_ldffsdu_be_zsu,
5453                   gen_helper_sve_ldffdd_be_zsu, } },
5454               { { gen_helper_sve_ldffbds_zss,
5455                   gen_helper_sve_ldffhds_be_zss,
5456                   gen_helper_sve_ldffsds_be_zss,
5457                   NULL, },
5458                 { gen_helper_sve_ldffbdu_zss,
5459                   gen_helper_sve_ldffhdu_be_zss,
5460                   gen_helper_sve_ldffsdu_be_zss,
5461                   gen_helper_sve_ldffdd_be_zss, } },
5462               { { gen_helper_sve_ldffbds_zd,
5463                   gen_helper_sve_ldffhds_be_zd,
5464                   gen_helper_sve_ldffsds_be_zd,
5465                   NULL, },
5466                 { gen_helper_sve_ldffbdu_zd,
5467                   gen_helper_sve_ldffhdu_be_zd,
5468                   gen_helper_sve_ldffsdu_be_zd,
5469                   gen_helper_sve_ldffdd_be_zd, } } } } },
5470     { /* MTE Active */
5471         { /* Little-endian */
5472             { { { gen_helper_sve_ldbds_zsu_mte,
5473                   gen_helper_sve_ldhds_le_zsu_mte,
5474                   gen_helper_sve_ldsds_le_zsu_mte,
5475                   NULL, },
5476                 { gen_helper_sve_ldbdu_zsu_mte,
5477                   gen_helper_sve_ldhdu_le_zsu_mte,
5478                   gen_helper_sve_ldsdu_le_zsu_mte,
5479                   gen_helper_sve_lddd_le_zsu_mte, } },
5480               { { gen_helper_sve_ldbds_zss_mte,
5481                   gen_helper_sve_ldhds_le_zss_mte,
5482                   gen_helper_sve_ldsds_le_zss_mte,
5483                   NULL, },
5484                 { gen_helper_sve_ldbdu_zss_mte,
5485                   gen_helper_sve_ldhdu_le_zss_mte,
5486                   gen_helper_sve_ldsdu_le_zss_mte,
5487                   gen_helper_sve_lddd_le_zss_mte, } },
5488               { { gen_helper_sve_ldbds_zd_mte,
5489                   gen_helper_sve_ldhds_le_zd_mte,
5490                   gen_helper_sve_ldsds_le_zd_mte,
5491                   NULL, },
5492                 { gen_helper_sve_ldbdu_zd_mte,
5493                   gen_helper_sve_ldhdu_le_zd_mte,
5494                   gen_helper_sve_ldsdu_le_zd_mte,
5495                   gen_helper_sve_lddd_le_zd_mte, } } },
5496 
5497             /* First-fault */
5498             { { { gen_helper_sve_ldffbds_zsu_mte,
5499                   gen_helper_sve_ldffhds_le_zsu_mte,
5500                   gen_helper_sve_ldffsds_le_zsu_mte,
5501                   NULL, },
5502                 { gen_helper_sve_ldffbdu_zsu_mte,
5503                   gen_helper_sve_ldffhdu_le_zsu_mte,
5504                   gen_helper_sve_ldffsdu_le_zsu_mte,
5505                   gen_helper_sve_ldffdd_le_zsu_mte, } },
5506               { { gen_helper_sve_ldffbds_zss_mte,
5507                   gen_helper_sve_ldffhds_le_zss_mte,
5508                   gen_helper_sve_ldffsds_le_zss_mte,
5509                   NULL, },
5510                 { gen_helper_sve_ldffbdu_zss_mte,
5511                   gen_helper_sve_ldffhdu_le_zss_mte,
5512                   gen_helper_sve_ldffsdu_le_zss_mte,
5513                   gen_helper_sve_ldffdd_le_zss_mte, } },
5514               { { gen_helper_sve_ldffbds_zd_mte,
5515                   gen_helper_sve_ldffhds_le_zd_mte,
5516                   gen_helper_sve_ldffsds_le_zd_mte,
5517                   NULL, },
5518                 { gen_helper_sve_ldffbdu_zd_mte,
5519                   gen_helper_sve_ldffhdu_le_zd_mte,
5520                   gen_helper_sve_ldffsdu_le_zd_mte,
5521                   gen_helper_sve_ldffdd_le_zd_mte, } } } },
5522         { /* Big-endian */
5523             { { { gen_helper_sve_ldbds_zsu_mte,
5524                   gen_helper_sve_ldhds_be_zsu_mte,
5525                   gen_helper_sve_ldsds_be_zsu_mte,
5526                   NULL, },
5527                 { gen_helper_sve_ldbdu_zsu_mte,
5528                   gen_helper_sve_ldhdu_be_zsu_mte,
5529                   gen_helper_sve_ldsdu_be_zsu_mte,
5530                   gen_helper_sve_lddd_be_zsu_mte, } },
5531               { { gen_helper_sve_ldbds_zss_mte,
5532                   gen_helper_sve_ldhds_be_zss_mte,
5533                   gen_helper_sve_ldsds_be_zss_mte,
5534                   NULL, },
5535                 { gen_helper_sve_ldbdu_zss_mte,
5536                   gen_helper_sve_ldhdu_be_zss_mte,
5537                   gen_helper_sve_ldsdu_be_zss_mte,
5538                   gen_helper_sve_lddd_be_zss_mte, } },
5539               { { gen_helper_sve_ldbds_zd_mte,
5540                   gen_helper_sve_ldhds_be_zd_mte,
5541                   gen_helper_sve_ldsds_be_zd_mte,
5542                   NULL, },
5543                 { gen_helper_sve_ldbdu_zd_mte,
5544                   gen_helper_sve_ldhdu_be_zd_mte,
5545                   gen_helper_sve_ldsdu_be_zd_mte,
5546                   gen_helper_sve_lddd_be_zd_mte, } } },
5547 
5548             /* First-fault */
5549             { { { gen_helper_sve_ldffbds_zsu_mte,
5550                   gen_helper_sve_ldffhds_be_zsu_mte,
5551                   gen_helper_sve_ldffsds_be_zsu_mte,
5552                   NULL, },
5553                 { gen_helper_sve_ldffbdu_zsu_mte,
5554                   gen_helper_sve_ldffhdu_be_zsu_mte,
5555                   gen_helper_sve_ldffsdu_be_zsu_mte,
5556                   gen_helper_sve_ldffdd_be_zsu_mte, } },
5557               { { gen_helper_sve_ldffbds_zss_mte,
5558                   gen_helper_sve_ldffhds_be_zss_mte,
5559                   gen_helper_sve_ldffsds_be_zss_mte,
5560                   NULL, },
5561                 { gen_helper_sve_ldffbdu_zss_mte,
5562                   gen_helper_sve_ldffhdu_be_zss_mte,
5563                   gen_helper_sve_ldffsdu_be_zss_mte,
5564                   gen_helper_sve_ldffdd_be_zss_mte, } },
5565               { { gen_helper_sve_ldffbds_zd_mte,
5566                   gen_helper_sve_ldffhds_be_zd_mte,
5567                   gen_helper_sve_ldffsds_be_zd_mte,
5568                   NULL, },
5569                 { gen_helper_sve_ldffbdu_zd_mte,
5570                   gen_helper_sve_ldffhdu_be_zd_mte,
5571                   gen_helper_sve_ldffsdu_be_zd_mte,
5572                   gen_helper_sve_ldffdd_be_zd_mte, } } } } },
5573 };
5574 
5575 static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a)
5576 {
5577     gen_helper_gvec_mem_scatter *fn = NULL;
5578     bool be = s->be_data == MO_BE;
5579     bool mte = s->mte_active[0];
5580 
5581     if (!dc_isar_feature(aa64_sve, s)) {
5582         return false;
5583     }
5584     s->is_nonstreaming = true;
5585     if (!sve_access_check(s)) {
5586         return true;
5587     }
5588 
5589     switch (a->esz) {
5590     case MO_32:
5591         fn = gather_load_fn32[mte][be][a->ff][a->xs][a->u][a->msz];
5592         break;
5593     case MO_64:
5594         fn = gather_load_fn64[mte][be][a->ff][a->xs][a->u][a->msz];
5595         break;
5596     }
5597     assert(fn != NULL);
5598 
5599     do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz,
5600                cpu_reg_sp(s, a->rn), a->msz, false, fn);
5601     return true;
5602 }
5603 
5604 static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a)
5605 {
5606     gen_helper_gvec_mem_scatter *fn = NULL;
5607     bool be = s->be_data == MO_BE;
5608     bool mte = s->mte_active[0];
5609 
5610     if (a->esz < a->msz || (a->esz == a->msz && !a->u)) {
5611         return false;
5612     }
5613     if (!dc_isar_feature(aa64_sve, s)) {
5614         return false;
5615     }
5616     s->is_nonstreaming = true;
5617     if (!sve_access_check(s)) {
5618         return true;
5619     }
5620 
5621     switch (a->esz) {
5622     case MO_32:
5623         fn = gather_load_fn32[mte][be][a->ff][0][a->u][a->msz];
5624         break;
5625     case MO_64:
5626         fn = gather_load_fn64[mte][be][a->ff][2][a->u][a->msz];
5627         break;
5628     }
5629     assert(fn != NULL);
5630 
5631     /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
5632      * by loading the immediate into the scalar parameter.
5633      */
5634     do_mem_zpz(s, a->rd, a->pg, a->rn, 0,
5635                tcg_constant_i64(a->imm << a->msz), a->msz, false, fn);
5636     return true;
5637 }
5638 
5639 static bool trans_LDNT1_zprz(DisasContext *s, arg_LD1_zprz *a)
5640 {
5641     gen_helper_gvec_mem_scatter *fn = NULL;
5642     bool be = s->be_data == MO_BE;
5643     bool mte = s->mte_active[0];
5644 
5645     if (a->esz < a->msz + !a->u) {
5646         return false;
5647     }
5648     if (!dc_isar_feature(aa64_sve2, s)) {
5649         return false;
5650     }
5651     s->is_nonstreaming = true;
5652     if (!sve_access_check(s)) {
5653         return true;
5654     }
5655 
5656     switch (a->esz) {
5657     case MO_32:
5658         fn = gather_load_fn32[mte][be][0][0][a->u][a->msz];
5659         break;
5660     case MO_64:
5661         fn = gather_load_fn64[mte][be][0][2][a->u][a->msz];
5662         break;
5663     }
5664     assert(fn != NULL);
5665 
5666     do_mem_zpz(s, a->rd, a->pg, a->rn, 0,
5667                cpu_reg(s, a->rm), a->msz, false, fn);
5668     return true;
5669 }
5670 
5671 /* Indexed by [mte][be][xs][msz].  */
5672 static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][2][2][3] = {
5673     { /* MTE Inactive */
5674         { /* Little-endian */
5675             { gen_helper_sve_stbs_zsu,
5676               gen_helper_sve_sths_le_zsu,
5677               gen_helper_sve_stss_le_zsu, },
5678             { gen_helper_sve_stbs_zss,
5679               gen_helper_sve_sths_le_zss,
5680               gen_helper_sve_stss_le_zss, } },
5681         { /* Big-endian */
5682             { gen_helper_sve_stbs_zsu,
5683               gen_helper_sve_sths_be_zsu,
5684               gen_helper_sve_stss_be_zsu, },
5685             { gen_helper_sve_stbs_zss,
5686               gen_helper_sve_sths_be_zss,
5687               gen_helper_sve_stss_be_zss, } } },
5688     { /* MTE Active */
5689         { /* Little-endian */
5690             { gen_helper_sve_stbs_zsu_mte,
5691               gen_helper_sve_sths_le_zsu_mte,
5692               gen_helper_sve_stss_le_zsu_mte, },
5693             { gen_helper_sve_stbs_zss_mte,
5694               gen_helper_sve_sths_le_zss_mte,
5695               gen_helper_sve_stss_le_zss_mte, } },
5696         { /* Big-endian */
5697             { gen_helper_sve_stbs_zsu_mte,
5698               gen_helper_sve_sths_be_zsu_mte,
5699               gen_helper_sve_stss_be_zsu_mte, },
5700             { gen_helper_sve_stbs_zss_mte,
5701               gen_helper_sve_sths_be_zss_mte,
5702               gen_helper_sve_stss_be_zss_mte, } } },
5703 };
5704 
5705 /* Note that we overload xs=2 to indicate 64-bit offset.  */
5706 static gen_helper_gvec_mem_scatter * const scatter_store_fn64[2][2][3][4] = {
5707     { /* MTE Inactive */
5708          { /* Little-endian */
5709              { gen_helper_sve_stbd_zsu,
5710                gen_helper_sve_sthd_le_zsu,
5711                gen_helper_sve_stsd_le_zsu,
5712                gen_helper_sve_stdd_le_zsu, },
5713              { gen_helper_sve_stbd_zss,
5714                gen_helper_sve_sthd_le_zss,
5715                gen_helper_sve_stsd_le_zss,
5716                gen_helper_sve_stdd_le_zss, },
5717              { gen_helper_sve_stbd_zd,
5718                gen_helper_sve_sthd_le_zd,
5719                gen_helper_sve_stsd_le_zd,
5720                gen_helper_sve_stdd_le_zd, } },
5721          { /* Big-endian */
5722              { gen_helper_sve_stbd_zsu,
5723                gen_helper_sve_sthd_be_zsu,
5724                gen_helper_sve_stsd_be_zsu,
5725                gen_helper_sve_stdd_be_zsu, },
5726              { gen_helper_sve_stbd_zss,
5727                gen_helper_sve_sthd_be_zss,
5728                gen_helper_sve_stsd_be_zss,
5729                gen_helper_sve_stdd_be_zss, },
5730              { gen_helper_sve_stbd_zd,
5731                gen_helper_sve_sthd_be_zd,
5732                gen_helper_sve_stsd_be_zd,
5733                gen_helper_sve_stdd_be_zd, } } },
5734     { /* MTE Inactive */
5735          { /* Little-endian */
5736              { gen_helper_sve_stbd_zsu_mte,
5737                gen_helper_sve_sthd_le_zsu_mte,
5738                gen_helper_sve_stsd_le_zsu_mte,
5739                gen_helper_sve_stdd_le_zsu_mte, },
5740              { gen_helper_sve_stbd_zss_mte,
5741                gen_helper_sve_sthd_le_zss_mte,
5742                gen_helper_sve_stsd_le_zss_mte,
5743                gen_helper_sve_stdd_le_zss_mte, },
5744              { gen_helper_sve_stbd_zd_mte,
5745                gen_helper_sve_sthd_le_zd_mte,
5746                gen_helper_sve_stsd_le_zd_mte,
5747                gen_helper_sve_stdd_le_zd_mte, } },
5748          { /* Big-endian */
5749              { gen_helper_sve_stbd_zsu_mte,
5750                gen_helper_sve_sthd_be_zsu_mte,
5751                gen_helper_sve_stsd_be_zsu_mte,
5752                gen_helper_sve_stdd_be_zsu_mte, },
5753              { gen_helper_sve_stbd_zss_mte,
5754                gen_helper_sve_sthd_be_zss_mte,
5755                gen_helper_sve_stsd_be_zss_mte,
5756                gen_helper_sve_stdd_be_zss_mte, },
5757              { gen_helper_sve_stbd_zd_mte,
5758                gen_helper_sve_sthd_be_zd_mte,
5759                gen_helper_sve_stsd_be_zd_mte,
5760                gen_helper_sve_stdd_be_zd_mte, } } },
5761 };
5762 
5763 static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a)
5764 {
5765     gen_helper_gvec_mem_scatter *fn;
5766     bool be = s->be_data == MO_BE;
5767     bool mte = s->mte_active[0];
5768 
5769     if (a->esz < a->msz || (a->msz == 0 && a->scale)) {
5770         return false;
5771     }
5772     if (!dc_isar_feature(aa64_sve, s)) {
5773         return false;
5774     }
5775     s->is_nonstreaming = true;
5776     if (!sve_access_check(s)) {
5777         return true;
5778     }
5779     switch (a->esz) {
5780     case MO_32:
5781         fn = scatter_store_fn32[mte][be][a->xs][a->msz];
5782         break;
5783     case MO_64:
5784         fn = scatter_store_fn64[mte][be][a->xs][a->msz];
5785         break;
5786     default:
5787         g_assert_not_reached();
5788     }
5789     do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz,
5790                cpu_reg_sp(s, a->rn), a->msz, true, fn);
5791     return true;
5792 }
5793 
5794 static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a)
5795 {
5796     gen_helper_gvec_mem_scatter *fn = NULL;
5797     bool be = s->be_data == MO_BE;
5798     bool mte = s->mte_active[0];
5799 
5800     if (a->esz < a->msz) {
5801         return false;
5802     }
5803     if (!dc_isar_feature(aa64_sve, s)) {
5804         return false;
5805     }
5806     s->is_nonstreaming = true;
5807     if (!sve_access_check(s)) {
5808         return true;
5809     }
5810 
5811     switch (a->esz) {
5812     case MO_32:
5813         fn = scatter_store_fn32[mte][be][0][a->msz];
5814         break;
5815     case MO_64:
5816         fn = scatter_store_fn64[mte][be][2][a->msz];
5817         break;
5818     }
5819     assert(fn != NULL);
5820 
5821     /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
5822      * by loading the immediate into the scalar parameter.
5823      */
5824     do_mem_zpz(s, a->rd, a->pg, a->rn, 0,
5825                tcg_constant_i64(a->imm << a->msz), a->msz, true, fn);
5826     return true;
5827 }
5828 
5829 static bool trans_STNT1_zprz(DisasContext *s, arg_ST1_zprz *a)
5830 {
5831     gen_helper_gvec_mem_scatter *fn;
5832     bool be = s->be_data == MO_BE;
5833     bool mte = s->mte_active[0];
5834 
5835     if (a->esz < a->msz) {
5836         return false;
5837     }
5838     if (!dc_isar_feature(aa64_sve2, s)) {
5839         return false;
5840     }
5841     s->is_nonstreaming = true;
5842     if (!sve_access_check(s)) {
5843         return true;
5844     }
5845 
5846     switch (a->esz) {
5847     case MO_32:
5848         fn = scatter_store_fn32[mte][be][0][a->msz];
5849         break;
5850     case MO_64:
5851         fn = scatter_store_fn64[mte][be][2][a->msz];
5852         break;
5853     default:
5854         g_assert_not_reached();
5855     }
5856 
5857     do_mem_zpz(s, a->rd, a->pg, a->rn, 0,
5858                cpu_reg(s, a->rm), a->msz, true, fn);
5859     return true;
5860 }
5861 
5862 /*
5863  * Prefetches
5864  */
5865 
5866 static bool trans_PRF(DisasContext *s, arg_PRF *a)
5867 {
5868     if (!dc_isar_feature(aa64_sve, s)) {
5869         return false;
5870     }
5871     /* Prefetch is a nop within QEMU.  */
5872     (void)sve_access_check(s);
5873     return true;
5874 }
5875 
5876 static bool trans_PRF_rr(DisasContext *s, arg_PRF_rr *a)
5877 {
5878     if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) {
5879         return false;
5880     }
5881     /* Prefetch is a nop within QEMU.  */
5882     (void)sve_access_check(s);
5883     return true;
5884 }
5885 
5886 static bool trans_PRF_ns(DisasContext *s, arg_PRF_ns *a)
5887 {
5888     if (!dc_isar_feature(aa64_sve, s)) {
5889         return false;
5890     }
5891     /* Prefetch is a nop within QEMU.  */
5892     s->is_nonstreaming = true;
5893     (void)sve_access_check(s);
5894     return true;
5895 }
5896 
5897 /*
5898  * Move Prefix
5899  *
5900  * TODO: The implementation so far could handle predicated merging movprfx.
5901  * The helper functions as written take an extra source register to
5902  * use in the operation, but the result is only written when predication
5903  * succeeds.  For unpredicated movprfx, we need to rearrange the helpers
5904  * to allow the final write back to the destination to be unconditional.
5905  * For predicated zeroing movprfx, we need to rearrange the helpers to
5906  * allow the final write back to zero inactives.
5907  *
5908  * In the meantime, just emit the moves.
5909  */
5910 
5911 TRANS_FEAT(MOVPRFX, aa64_sve, do_mov_z, a->rd, a->rn)
5912 TRANS_FEAT(MOVPRFX_m, aa64_sve, do_sel_z, a->rd, a->rn, a->rd, a->pg, a->esz)
5913 TRANS_FEAT(MOVPRFX_z, aa64_sve, do_movz_zpz, a->rd, a->rn, a->pg, a->esz, false)
5914 
5915 /*
5916  * SVE2 Integer Multiply - Unpredicated
5917  */
5918 
5919 TRANS_FEAT(MUL_zzz, aa64_sve2, gen_gvec_fn_arg_zzz, tcg_gen_gvec_mul, a)
5920 
5921 static gen_helper_gvec_3 * const smulh_zzz_fns[4] = {
5922     gen_helper_gvec_smulh_b, gen_helper_gvec_smulh_h,
5923     gen_helper_gvec_smulh_s, gen_helper_gvec_smulh_d,
5924 };
5925 TRANS_FEAT(SMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
5926            smulh_zzz_fns[a->esz], a, 0)
5927 
5928 static gen_helper_gvec_3 * const umulh_zzz_fns[4] = {
5929     gen_helper_gvec_umulh_b, gen_helper_gvec_umulh_h,
5930     gen_helper_gvec_umulh_s, gen_helper_gvec_umulh_d,
5931 };
5932 TRANS_FEAT(UMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
5933            umulh_zzz_fns[a->esz], a, 0)
5934 
5935 TRANS_FEAT(PMUL_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
5936            gen_helper_gvec_pmul_b, a, 0)
5937 
5938 static gen_helper_gvec_3 * const sqdmulh_zzz_fns[4] = {
5939     gen_helper_sve2_sqdmulh_b, gen_helper_sve2_sqdmulh_h,
5940     gen_helper_sve2_sqdmulh_s, gen_helper_sve2_sqdmulh_d,
5941 };
5942 TRANS_FEAT(SQDMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
5943            sqdmulh_zzz_fns[a->esz], a, 0)
5944 
5945 static gen_helper_gvec_3 * const sqrdmulh_zzz_fns[4] = {
5946     gen_helper_sve2_sqrdmulh_b, gen_helper_sve2_sqrdmulh_h,
5947     gen_helper_sve2_sqrdmulh_s, gen_helper_sve2_sqrdmulh_d,
5948 };
5949 TRANS_FEAT(SQRDMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
5950            sqrdmulh_zzz_fns[a->esz], a, 0)
5951 
5952 /*
5953  * SVE2 Integer - Predicated
5954  */
5955 
5956 static gen_helper_gvec_4 * const sadlp_fns[4] = {
5957     NULL,                          gen_helper_sve2_sadalp_zpzz_h,
5958     gen_helper_sve2_sadalp_zpzz_s, gen_helper_sve2_sadalp_zpzz_d,
5959 };
5960 TRANS_FEAT(SADALP_zpzz, aa64_sve2, gen_gvec_ool_arg_zpzz,
5961            sadlp_fns[a->esz], a, 0)
5962 
5963 static gen_helper_gvec_4 * const uadlp_fns[4] = {
5964     NULL,                          gen_helper_sve2_uadalp_zpzz_h,
5965     gen_helper_sve2_uadalp_zpzz_s, gen_helper_sve2_uadalp_zpzz_d,
5966 };
5967 TRANS_FEAT(UADALP_zpzz, aa64_sve2, gen_gvec_ool_arg_zpzz,
5968            uadlp_fns[a->esz], a, 0)
5969 
5970 /*
5971  * SVE2 integer unary operations (predicated)
5972  */
5973 
5974 TRANS_FEAT(URECPE, aa64_sve2, gen_gvec_ool_arg_zpz,
5975            a->esz == 2 ? gen_helper_sve2_urecpe_s : NULL, a, 0)
5976 
5977 TRANS_FEAT(URSQRTE, aa64_sve2, gen_gvec_ool_arg_zpz,
5978            a->esz == 2 ? gen_helper_sve2_ursqrte_s : NULL, a, 0)
5979 
5980 static gen_helper_gvec_3 * const sqabs_fns[4] = {
5981     gen_helper_sve2_sqabs_b, gen_helper_sve2_sqabs_h,
5982     gen_helper_sve2_sqabs_s, gen_helper_sve2_sqabs_d,
5983 };
5984 TRANS_FEAT(SQABS, aa64_sve2, gen_gvec_ool_arg_zpz, sqabs_fns[a->esz], a, 0)
5985 
5986 static gen_helper_gvec_3 * const sqneg_fns[4] = {
5987     gen_helper_sve2_sqneg_b, gen_helper_sve2_sqneg_h,
5988     gen_helper_sve2_sqneg_s, gen_helper_sve2_sqneg_d,
5989 };
5990 TRANS_FEAT(SQNEG, aa64_sve2, gen_gvec_ool_arg_zpz, sqneg_fns[a->esz], a, 0)
5991 
5992 DO_ZPZZ(SQSHL, aa64_sve2, sve2_sqshl)
5993 DO_ZPZZ(SQRSHL, aa64_sve2, sve2_sqrshl)
5994 DO_ZPZZ(SRSHL, aa64_sve2, sve2_srshl)
5995 
5996 DO_ZPZZ(UQSHL, aa64_sve2, sve2_uqshl)
5997 DO_ZPZZ(UQRSHL, aa64_sve2, sve2_uqrshl)
5998 DO_ZPZZ(URSHL, aa64_sve2, sve2_urshl)
5999 
6000 DO_ZPZZ(SHADD, aa64_sve2, sve2_shadd)
6001 DO_ZPZZ(SRHADD, aa64_sve2, sve2_srhadd)
6002 DO_ZPZZ(SHSUB, aa64_sve2, sve2_shsub)
6003 
6004 DO_ZPZZ(UHADD, aa64_sve2, sve2_uhadd)
6005 DO_ZPZZ(URHADD, aa64_sve2, sve2_urhadd)
6006 DO_ZPZZ(UHSUB, aa64_sve2, sve2_uhsub)
6007 
6008 DO_ZPZZ(ADDP, aa64_sve2, sve2_addp)
6009 DO_ZPZZ(SMAXP, aa64_sve2, sve2_smaxp)
6010 DO_ZPZZ(UMAXP, aa64_sve2, sve2_umaxp)
6011 DO_ZPZZ(SMINP, aa64_sve2, sve2_sminp)
6012 DO_ZPZZ(UMINP, aa64_sve2, sve2_uminp)
6013 
6014 DO_ZPZZ(SQADD_zpzz, aa64_sve2, sve2_sqadd)
6015 DO_ZPZZ(UQADD_zpzz, aa64_sve2, sve2_uqadd)
6016 DO_ZPZZ(SQSUB_zpzz, aa64_sve2, sve2_sqsub)
6017 DO_ZPZZ(UQSUB_zpzz, aa64_sve2, sve2_uqsub)
6018 DO_ZPZZ(SUQADD, aa64_sve2, sve2_suqadd)
6019 DO_ZPZZ(USQADD, aa64_sve2, sve2_usqadd)
6020 
6021 /*
6022  * SVE2 Widening Integer Arithmetic
6023  */
6024 
6025 static gen_helper_gvec_3 * const saddl_fns[4] = {
6026     NULL,                    gen_helper_sve2_saddl_h,
6027     gen_helper_sve2_saddl_s, gen_helper_sve2_saddl_d,
6028 };
6029 TRANS_FEAT(SADDLB, aa64_sve2, gen_gvec_ool_arg_zzz,
6030            saddl_fns[a->esz], a, 0)
6031 TRANS_FEAT(SADDLT, aa64_sve2, gen_gvec_ool_arg_zzz,
6032            saddl_fns[a->esz], a, 3)
6033 TRANS_FEAT(SADDLBT, aa64_sve2, gen_gvec_ool_arg_zzz,
6034            saddl_fns[a->esz], a, 2)
6035 
6036 static gen_helper_gvec_3 * const ssubl_fns[4] = {
6037     NULL,                    gen_helper_sve2_ssubl_h,
6038     gen_helper_sve2_ssubl_s, gen_helper_sve2_ssubl_d,
6039 };
6040 TRANS_FEAT(SSUBLB, aa64_sve2, gen_gvec_ool_arg_zzz,
6041            ssubl_fns[a->esz], a, 0)
6042 TRANS_FEAT(SSUBLT, aa64_sve2, gen_gvec_ool_arg_zzz,
6043            ssubl_fns[a->esz], a, 3)
6044 TRANS_FEAT(SSUBLBT, aa64_sve2, gen_gvec_ool_arg_zzz,
6045            ssubl_fns[a->esz], a, 2)
6046 TRANS_FEAT(SSUBLTB, aa64_sve2, gen_gvec_ool_arg_zzz,
6047            ssubl_fns[a->esz], a, 1)
6048 
6049 static gen_helper_gvec_3 * const sabdl_fns[4] = {
6050     NULL,                    gen_helper_sve2_sabdl_h,
6051     gen_helper_sve2_sabdl_s, gen_helper_sve2_sabdl_d,
6052 };
6053 TRANS_FEAT(SABDLB, aa64_sve2, gen_gvec_ool_arg_zzz,
6054            sabdl_fns[a->esz], a, 0)
6055 TRANS_FEAT(SABDLT, aa64_sve2, gen_gvec_ool_arg_zzz,
6056            sabdl_fns[a->esz], a, 3)
6057 
6058 static gen_helper_gvec_3 * const uaddl_fns[4] = {
6059     NULL,                    gen_helper_sve2_uaddl_h,
6060     gen_helper_sve2_uaddl_s, gen_helper_sve2_uaddl_d,
6061 };
6062 TRANS_FEAT(UADDLB, aa64_sve2, gen_gvec_ool_arg_zzz,
6063            uaddl_fns[a->esz], a, 0)
6064 TRANS_FEAT(UADDLT, aa64_sve2, gen_gvec_ool_arg_zzz,
6065            uaddl_fns[a->esz], a, 3)
6066 
6067 static gen_helper_gvec_3 * const usubl_fns[4] = {
6068     NULL,                    gen_helper_sve2_usubl_h,
6069     gen_helper_sve2_usubl_s, gen_helper_sve2_usubl_d,
6070 };
6071 TRANS_FEAT(USUBLB, aa64_sve2, gen_gvec_ool_arg_zzz,
6072            usubl_fns[a->esz], a, 0)
6073 TRANS_FEAT(USUBLT, aa64_sve2, gen_gvec_ool_arg_zzz,
6074            usubl_fns[a->esz], a, 3)
6075 
6076 static gen_helper_gvec_3 * const uabdl_fns[4] = {
6077     NULL,                    gen_helper_sve2_uabdl_h,
6078     gen_helper_sve2_uabdl_s, gen_helper_sve2_uabdl_d,
6079 };
6080 TRANS_FEAT(UABDLB, aa64_sve2, gen_gvec_ool_arg_zzz,
6081            uabdl_fns[a->esz], a, 0)
6082 TRANS_FEAT(UABDLT, aa64_sve2, gen_gvec_ool_arg_zzz,
6083            uabdl_fns[a->esz], a, 3)
6084 
6085 static gen_helper_gvec_3 * const sqdmull_fns[4] = {
6086     NULL,                          gen_helper_sve2_sqdmull_zzz_h,
6087     gen_helper_sve2_sqdmull_zzz_s, gen_helper_sve2_sqdmull_zzz_d,
6088 };
6089 TRANS_FEAT(SQDMULLB_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
6090            sqdmull_fns[a->esz], a, 0)
6091 TRANS_FEAT(SQDMULLT_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
6092            sqdmull_fns[a->esz], a, 3)
6093 
6094 static gen_helper_gvec_3 * const smull_fns[4] = {
6095     NULL,                        gen_helper_sve2_smull_zzz_h,
6096     gen_helper_sve2_smull_zzz_s, gen_helper_sve2_smull_zzz_d,
6097 };
6098 TRANS_FEAT(SMULLB_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
6099            smull_fns[a->esz], a, 0)
6100 TRANS_FEAT(SMULLT_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
6101            smull_fns[a->esz], a, 3)
6102 
6103 static gen_helper_gvec_3 * const umull_fns[4] = {
6104     NULL,                        gen_helper_sve2_umull_zzz_h,
6105     gen_helper_sve2_umull_zzz_s, gen_helper_sve2_umull_zzz_d,
6106 };
6107 TRANS_FEAT(UMULLB_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
6108            umull_fns[a->esz], a, 0)
6109 TRANS_FEAT(UMULLT_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
6110            umull_fns[a->esz], a, 3)
6111 
6112 static gen_helper_gvec_3 * const eoril_fns[4] = {
6113     gen_helper_sve2_eoril_b, gen_helper_sve2_eoril_h,
6114     gen_helper_sve2_eoril_s, gen_helper_sve2_eoril_d,
6115 };
6116 TRANS_FEAT(EORBT, aa64_sve2, gen_gvec_ool_arg_zzz, eoril_fns[a->esz], a, 2)
6117 TRANS_FEAT(EORTB, aa64_sve2, gen_gvec_ool_arg_zzz, eoril_fns[a->esz], a, 1)
6118 
6119 static bool do_trans_pmull(DisasContext *s, arg_rrr_esz *a, bool sel)
6120 {
6121     static gen_helper_gvec_3 * const fns[4] = {
6122         gen_helper_gvec_pmull_q, gen_helper_sve2_pmull_h,
6123         NULL,                    gen_helper_sve2_pmull_d,
6124     };
6125 
6126     if (a->esz == 0) {
6127         if (!dc_isar_feature(aa64_sve2_pmull128, s)) {
6128             return false;
6129         }
6130         s->is_nonstreaming = true;
6131     } else if (!dc_isar_feature(aa64_sve, s)) {
6132         return false;
6133     }
6134     return gen_gvec_ool_arg_zzz(s, fns[a->esz], a, sel);
6135 }
6136 
6137 TRANS_FEAT(PMULLB, aa64_sve2, do_trans_pmull, a, false)
6138 TRANS_FEAT(PMULLT, aa64_sve2, do_trans_pmull, a, true)
6139 
6140 static gen_helper_gvec_3 * const saddw_fns[4] = {
6141     NULL,                    gen_helper_sve2_saddw_h,
6142     gen_helper_sve2_saddw_s, gen_helper_sve2_saddw_d,
6143 };
6144 TRANS_FEAT(SADDWB, aa64_sve2, gen_gvec_ool_arg_zzz, saddw_fns[a->esz], a, 0)
6145 TRANS_FEAT(SADDWT, aa64_sve2, gen_gvec_ool_arg_zzz, saddw_fns[a->esz], a, 1)
6146 
6147 static gen_helper_gvec_3 * const ssubw_fns[4] = {
6148     NULL,                    gen_helper_sve2_ssubw_h,
6149     gen_helper_sve2_ssubw_s, gen_helper_sve2_ssubw_d,
6150 };
6151 TRANS_FEAT(SSUBWB, aa64_sve2, gen_gvec_ool_arg_zzz, ssubw_fns[a->esz], a, 0)
6152 TRANS_FEAT(SSUBWT, aa64_sve2, gen_gvec_ool_arg_zzz, ssubw_fns[a->esz], a, 1)
6153 
6154 static gen_helper_gvec_3 * const uaddw_fns[4] = {
6155     NULL,                    gen_helper_sve2_uaddw_h,
6156     gen_helper_sve2_uaddw_s, gen_helper_sve2_uaddw_d,
6157 };
6158 TRANS_FEAT(UADDWB, aa64_sve2, gen_gvec_ool_arg_zzz, uaddw_fns[a->esz], a, 0)
6159 TRANS_FEAT(UADDWT, aa64_sve2, gen_gvec_ool_arg_zzz, uaddw_fns[a->esz], a, 1)
6160 
6161 static gen_helper_gvec_3 * const usubw_fns[4] = {
6162     NULL,                    gen_helper_sve2_usubw_h,
6163     gen_helper_sve2_usubw_s, gen_helper_sve2_usubw_d,
6164 };
6165 TRANS_FEAT(USUBWB, aa64_sve2, gen_gvec_ool_arg_zzz, usubw_fns[a->esz], a, 0)
6166 TRANS_FEAT(USUBWT, aa64_sve2, gen_gvec_ool_arg_zzz, usubw_fns[a->esz], a, 1)
6167 
6168 static void gen_sshll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm)
6169 {
6170     int top = imm & 1;
6171     int shl = imm >> 1;
6172     int halfbits = 4 << vece;
6173 
6174     if (top) {
6175         if (shl == halfbits) {
6176             TCGv_vec t = tcg_temp_new_vec_matching(d);
6177             tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(halfbits, halfbits));
6178             tcg_gen_and_vec(vece, d, n, t);
6179         } else {
6180             tcg_gen_sari_vec(vece, d, n, halfbits);
6181             tcg_gen_shli_vec(vece, d, d, shl);
6182         }
6183     } else {
6184         tcg_gen_shli_vec(vece, d, n, halfbits);
6185         tcg_gen_sari_vec(vece, d, d, halfbits - shl);
6186     }
6187 }
6188 
6189 static void gen_ushll_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int imm)
6190 {
6191     int halfbits = 4 << vece;
6192     int top = imm & 1;
6193     int shl = (imm >> 1);
6194     int shift;
6195     uint64_t mask;
6196 
6197     mask = MAKE_64BIT_MASK(0, halfbits);
6198     mask <<= shl;
6199     mask = dup_const(vece, mask);
6200 
6201     shift = shl - top * halfbits;
6202     if (shift < 0) {
6203         tcg_gen_shri_i64(d, n, -shift);
6204     } else {
6205         tcg_gen_shli_i64(d, n, shift);
6206     }
6207     tcg_gen_andi_i64(d, d, mask);
6208 }
6209 
6210 static void gen_ushll16_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm)
6211 {
6212     gen_ushll_i64(MO_16, d, n, imm);
6213 }
6214 
6215 static void gen_ushll32_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm)
6216 {
6217     gen_ushll_i64(MO_32, d, n, imm);
6218 }
6219 
6220 static void gen_ushll64_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm)
6221 {
6222     gen_ushll_i64(MO_64, d, n, imm);
6223 }
6224 
6225 static void gen_ushll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm)
6226 {
6227     int halfbits = 4 << vece;
6228     int top = imm & 1;
6229     int shl = imm >> 1;
6230 
6231     if (top) {
6232         if (shl == halfbits) {
6233             TCGv_vec t = tcg_temp_new_vec_matching(d);
6234             tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(halfbits, halfbits));
6235             tcg_gen_and_vec(vece, d, n, t);
6236         } else {
6237             tcg_gen_shri_vec(vece, d, n, halfbits);
6238             tcg_gen_shli_vec(vece, d, d, shl);
6239         }
6240     } else {
6241         if (shl == 0) {
6242             TCGv_vec t = tcg_temp_new_vec_matching(d);
6243             tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
6244             tcg_gen_and_vec(vece, d, n, t);
6245         } else {
6246             tcg_gen_shli_vec(vece, d, n, halfbits);
6247             tcg_gen_shri_vec(vece, d, d, halfbits - shl);
6248         }
6249     }
6250 }
6251 
6252 static bool do_shll_tb(DisasContext *s, arg_rri_esz *a,
6253                        const GVecGen2i ops[3], bool sel)
6254 {
6255 
6256     if (a->esz < 0 || a->esz > 2) {
6257         return false;
6258     }
6259     if (sve_access_check(s)) {
6260         unsigned vsz = vec_full_reg_size(s);
6261         tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd),
6262                         vec_full_reg_offset(s, a->rn),
6263                         vsz, vsz, (a->imm << 1) | sel,
6264                         &ops[a->esz]);
6265     }
6266     return true;
6267 }
6268 
6269 static const TCGOpcode sshll_list[] = {
6270     INDEX_op_shli_vec, INDEX_op_sari_vec, 0
6271 };
6272 static const GVecGen2i sshll_ops[3] = {
6273     { .fniv = gen_sshll_vec,
6274       .opt_opc = sshll_list,
6275       .fno = gen_helper_sve2_sshll_h,
6276       .vece = MO_16 },
6277     { .fniv = gen_sshll_vec,
6278       .opt_opc = sshll_list,
6279       .fno = gen_helper_sve2_sshll_s,
6280       .vece = MO_32 },
6281     { .fniv = gen_sshll_vec,
6282       .opt_opc = sshll_list,
6283       .fno = gen_helper_sve2_sshll_d,
6284       .vece = MO_64 }
6285 };
6286 TRANS_FEAT(SSHLLB, aa64_sve2, do_shll_tb, a, sshll_ops, false)
6287 TRANS_FEAT(SSHLLT, aa64_sve2, do_shll_tb, a, sshll_ops, true)
6288 
6289 static const TCGOpcode ushll_list[] = {
6290     INDEX_op_shli_vec, INDEX_op_shri_vec, 0
6291 };
6292 static const GVecGen2i ushll_ops[3] = {
6293     { .fni8 = gen_ushll16_i64,
6294       .fniv = gen_ushll_vec,
6295       .opt_opc = ushll_list,
6296       .fno = gen_helper_sve2_ushll_h,
6297       .vece = MO_16 },
6298     { .fni8 = gen_ushll32_i64,
6299       .fniv = gen_ushll_vec,
6300       .opt_opc = ushll_list,
6301       .fno = gen_helper_sve2_ushll_s,
6302       .vece = MO_32 },
6303     { .fni8 = gen_ushll64_i64,
6304       .fniv = gen_ushll_vec,
6305       .opt_opc = ushll_list,
6306       .fno = gen_helper_sve2_ushll_d,
6307       .vece = MO_64 },
6308 };
6309 TRANS_FEAT(USHLLB, aa64_sve2, do_shll_tb, a, ushll_ops, false)
6310 TRANS_FEAT(USHLLT, aa64_sve2, do_shll_tb, a, ushll_ops, true)
6311 
6312 static gen_helper_gvec_3 * const bext_fns[4] = {
6313     gen_helper_sve2_bext_b, gen_helper_sve2_bext_h,
6314     gen_helper_sve2_bext_s, gen_helper_sve2_bext_d,
6315 };
6316 TRANS_FEAT_NONSTREAMING(BEXT, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz,
6317                         bext_fns[a->esz], a, 0)
6318 
6319 static gen_helper_gvec_3 * const bdep_fns[4] = {
6320     gen_helper_sve2_bdep_b, gen_helper_sve2_bdep_h,
6321     gen_helper_sve2_bdep_s, gen_helper_sve2_bdep_d,
6322 };
6323 TRANS_FEAT_NONSTREAMING(BDEP, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz,
6324                         bdep_fns[a->esz], a, 0)
6325 
6326 static gen_helper_gvec_3 * const bgrp_fns[4] = {
6327     gen_helper_sve2_bgrp_b, gen_helper_sve2_bgrp_h,
6328     gen_helper_sve2_bgrp_s, gen_helper_sve2_bgrp_d,
6329 };
6330 TRANS_FEAT_NONSTREAMING(BGRP, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz,
6331                         bgrp_fns[a->esz], a, 0)
6332 
6333 static gen_helper_gvec_3 * const cadd_fns[4] = {
6334     gen_helper_sve2_cadd_b, gen_helper_sve2_cadd_h,
6335     gen_helper_sve2_cadd_s, gen_helper_sve2_cadd_d,
6336 };
6337 TRANS_FEAT(CADD_rot90, aa64_sve2, gen_gvec_ool_arg_zzz,
6338            cadd_fns[a->esz], a, 0)
6339 TRANS_FEAT(CADD_rot270, aa64_sve2, gen_gvec_ool_arg_zzz,
6340            cadd_fns[a->esz], a, 1)
6341 
6342 static gen_helper_gvec_3 * const sqcadd_fns[4] = {
6343     gen_helper_sve2_sqcadd_b, gen_helper_sve2_sqcadd_h,
6344     gen_helper_sve2_sqcadd_s, gen_helper_sve2_sqcadd_d,
6345 };
6346 TRANS_FEAT(SQCADD_rot90, aa64_sve2, gen_gvec_ool_arg_zzz,
6347            sqcadd_fns[a->esz], a, 0)
6348 TRANS_FEAT(SQCADD_rot270, aa64_sve2, gen_gvec_ool_arg_zzz,
6349            sqcadd_fns[a->esz], a, 1)
6350 
6351 static gen_helper_gvec_4 * const sabal_fns[4] = {
6352     NULL,                    gen_helper_sve2_sabal_h,
6353     gen_helper_sve2_sabal_s, gen_helper_sve2_sabal_d,
6354 };
6355 TRANS_FEAT(SABALB, aa64_sve2, gen_gvec_ool_arg_zzzz, sabal_fns[a->esz], a, 0)
6356 TRANS_FEAT(SABALT, aa64_sve2, gen_gvec_ool_arg_zzzz, sabal_fns[a->esz], a, 1)
6357 
6358 static gen_helper_gvec_4 * const uabal_fns[4] = {
6359     NULL,                    gen_helper_sve2_uabal_h,
6360     gen_helper_sve2_uabal_s, gen_helper_sve2_uabal_d,
6361 };
6362 TRANS_FEAT(UABALB, aa64_sve2, gen_gvec_ool_arg_zzzz, uabal_fns[a->esz], a, 0)
6363 TRANS_FEAT(UABALT, aa64_sve2, gen_gvec_ool_arg_zzzz, uabal_fns[a->esz], a, 1)
6364 
6365 static bool do_adcl(DisasContext *s, arg_rrrr_esz *a, bool sel)
6366 {
6367     static gen_helper_gvec_4 * const fns[2] = {
6368         gen_helper_sve2_adcl_s,
6369         gen_helper_sve2_adcl_d,
6370     };
6371     /*
6372      * Note that in this case the ESZ field encodes both size and sign.
6373      * Split out 'subtract' into bit 1 of the data field for the helper.
6374      */
6375     return gen_gvec_ool_arg_zzzz(s, fns[a->esz & 1], a, (a->esz & 2) | sel);
6376 }
6377 
6378 TRANS_FEAT(ADCLB, aa64_sve2, do_adcl, a, false)
6379 TRANS_FEAT(ADCLT, aa64_sve2, do_adcl, a, true)
6380 
6381 TRANS_FEAT(SSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_ssra, a)
6382 TRANS_FEAT(USRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_usra, a)
6383 TRANS_FEAT(SRSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_srsra, a)
6384 TRANS_FEAT(URSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_ursra, a)
6385 TRANS_FEAT(SRI, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_sri, a)
6386 TRANS_FEAT(SLI, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_sli, a)
6387 
6388 TRANS_FEAT(SABA, aa64_sve2, gen_gvec_fn_arg_zzz, gen_gvec_saba, a)
6389 TRANS_FEAT(UABA, aa64_sve2, gen_gvec_fn_arg_zzz, gen_gvec_uaba, a)
6390 
6391 static bool do_narrow_extract(DisasContext *s, arg_rri_esz *a,
6392                               const GVecGen2 ops[3])
6393 {
6394     if (a->esz < 0 || a->esz > MO_32 || a->imm != 0) {
6395         return false;
6396     }
6397     if (sve_access_check(s)) {
6398         unsigned vsz = vec_full_reg_size(s);
6399         tcg_gen_gvec_2(vec_full_reg_offset(s, a->rd),
6400                         vec_full_reg_offset(s, a->rn),
6401                         vsz, vsz, &ops[a->esz]);
6402     }
6403     return true;
6404 }
6405 
6406 static const TCGOpcode sqxtn_list[] = {
6407     INDEX_op_shli_vec, INDEX_op_smin_vec, INDEX_op_smax_vec, 0
6408 };
6409 
6410 static void gen_sqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
6411 {
6412     TCGv_vec t = tcg_temp_new_vec_matching(d);
6413     int halfbits = 4 << vece;
6414     int64_t mask = (1ull << halfbits) - 1;
6415     int64_t min = -1ull << (halfbits - 1);
6416     int64_t max = -min - 1;
6417 
6418     tcg_gen_dupi_vec(vece, t, min);
6419     tcg_gen_smax_vec(vece, d, n, t);
6420     tcg_gen_dupi_vec(vece, t, max);
6421     tcg_gen_smin_vec(vece, d, d, t);
6422     tcg_gen_dupi_vec(vece, t, mask);
6423     tcg_gen_and_vec(vece, d, d, t);
6424 }
6425 
6426 static const GVecGen2 sqxtnb_ops[3] = {
6427     { .fniv = gen_sqxtnb_vec,
6428       .opt_opc = sqxtn_list,
6429       .fno = gen_helper_sve2_sqxtnb_h,
6430       .vece = MO_16 },
6431     { .fniv = gen_sqxtnb_vec,
6432       .opt_opc = sqxtn_list,
6433       .fno = gen_helper_sve2_sqxtnb_s,
6434       .vece = MO_32 },
6435     { .fniv = gen_sqxtnb_vec,
6436       .opt_opc = sqxtn_list,
6437       .fno = gen_helper_sve2_sqxtnb_d,
6438       .vece = MO_64 },
6439 };
6440 TRANS_FEAT(SQXTNB, aa64_sve2, do_narrow_extract, a, sqxtnb_ops)
6441 
6442 static void gen_sqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
6443 {
6444     TCGv_vec t = tcg_temp_new_vec_matching(d);
6445     int halfbits = 4 << vece;
6446     int64_t mask = (1ull << halfbits) - 1;
6447     int64_t min = -1ull << (halfbits - 1);
6448     int64_t max = -min - 1;
6449 
6450     tcg_gen_dupi_vec(vece, t, min);
6451     tcg_gen_smax_vec(vece, n, n, t);
6452     tcg_gen_dupi_vec(vece, t, max);
6453     tcg_gen_smin_vec(vece, n, n, t);
6454     tcg_gen_shli_vec(vece, n, n, halfbits);
6455     tcg_gen_dupi_vec(vece, t, mask);
6456     tcg_gen_bitsel_vec(vece, d, t, d, n);
6457 }
6458 
6459 static const GVecGen2 sqxtnt_ops[3] = {
6460     { .fniv = gen_sqxtnt_vec,
6461       .opt_opc = sqxtn_list,
6462       .load_dest = true,
6463       .fno = gen_helper_sve2_sqxtnt_h,
6464       .vece = MO_16 },
6465     { .fniv = gen_sqxtnt_vec,
6466       .opt_opc = sqxtn_list,
6467       .load_dest = true,
6468       .fno = gen_helper_sve2_sqxtnt_s,
6469       .vece = MO_32 },
6470     { .fniv = gen_sqxtnt_vec,
6471       .opt_opc = sqxtn_list,
6472       .load_dest = true,
6473       .fno = gen_helper_sve2_sqxtnt_d,
6474       .vece = MO_64 },
6475 };
6476 TRANS_FEAT(SQXTNT, aa64_sve2, do_narrow_extract, a, sqxtnt_ops)
6477 
6478 static const TCGOpcode uqxtn_list[] = {
6479     INDEX_op_shli_vec, INDEX_op_umin_vec, 0
6480 };
6481 
6482 static void gen_uqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
6483 {
6484     TCGv_vec t = tcg_temp_new_vec_matching(d);
6485     int halfbits = 4 << vece;
6486     int64_t max = (1ull << halfbits) - 1;
6487 
6488     tcg_gen_dupi_vec(vece, t, max);
6489     tcg_gen_umin_vec(vece, d, n, t);
6490 }
6491 
6492 static const GVecGen2 uqxtnb_ops[3] = {
6493     { .fniv = gen_uqxtnb_vec,
6494       .opt_opc = uqxtn_list,
6495       .fno = gen_helper_sve2_uqxtnb_h,
6496       .vece = MO_16 },
6497     { .fniv = gen_uqxtnb_vec,
6498       .opt_opc = uqxtn_list,
6499       .fno = gen_helper_sve2_uqxtnb_s,
6500       .vece = MO_32 },
6501     { .fniv = gen_uqxtnb_vec,
6502       .opt_opc = uqxtn_list,
6503       .fno = gen_helper_sve2_uqxtnb_d,
6504       .vece = MO_64 },
6505 };
6506 TRANS_FEAT(UQXTNB, aa64_sve2, do_narrow_extract, a, uqxtnb_ops)
6507 
6508 static void gen_uqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
6509 {
6510     TCGv_vec t = tcg_temp_new_vec_matching(d);
6511     int halfbits = 4 << vece;
6512     int64_t max = (1ull << halfbits) - 1;
6513 
6514     tcg_gen_dupi_vec(vece, t, max);
6515     tcg_gen_umin_vec(vece, n, n, t);
6516     tcg_gen_shli_vec(vece, n, n, halfbits);
6517     tcg_gen_bitsel_vec(vece, d, t, d, n);
6518 }
6519 
6520 static const GVecGen2 uqxtnt_ops[3] = {
6521     { .fniv = gen_uqxtnt_vec,
6522       .opt_opc = uqxtn_list,
6523       .load_dest = true,
6524       .fno = gen_helper_sve2_uqxtnt_h,
6525       .vece = MO_16 },
6526     { .fniv = gen_uqxtnt_vec,
6527       .opt_opc = uqxtn_list,
6528       .load_dest = true,
6529       .fno = gen_helper_sve2_uqxtnt_s,
6530       .vece = MO_32 },
6531     { .fniv = gen_uqxtnt_vec,
6532       .opt_opc = uqxtn_list,
6533       .load_dest = true,
6534       .fno = gen_helper_sve2_uqxtnt_d,
6535       .vece = MO_64 },
6536 };
6537 TRANS_FEAT(UQXTNT, aa64_sve2, do_narrow_extract, a, uqxtnt_ops)
6538 
6539 static const TCGOpcode sqxtun_list[] = {
6540     INDEX_op_shli_vec, INDEX_op_umin_vec, INDEX_op_smax_vec, 0
6541 };
6542 
6543 static void gen_sqxtunb_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
6544 {
6545     TCGv_vec t = tcg_temp_new_vec_matching(d);
6546     int halfbits = 4 << vece;
6547     int64_t max = (1ull << halfbits) - 1;
6548 
6549     tcg_gen_dupi_vec(vece, t, 0);
6550     tcg_gen_smax_vec(vece, d, n, t);
6551     tcg_gen_dupi_vec(vece, t, max);
6552     tcg_gen_umin_vec(vece, d, d, t);
6553 }
6554 
6555 static const GVecGen2 sqxtunb_ops[3] = {
6556     { .fniv = gen_sqxtunb_vec,
6557       .opt_opc = sqxtun_list,
6558       .fno = gen_helper_sve2_sqxtunb_h,
6559       .vece = MO_16 },
6560     { .fniv = gen_sqxtunb_vec,
6561       .opt_opc = sqxtun_list,
6562       .fno = gen_helper_sve2_sqxtunb_s,
6563       .vece = MO_32 },
6564     { .fniv = gen_sqxtunb_vec,
6565       .opt_opc = sqxtun_list,
6566       .fno = gen_helper_sve2_sqxtunb_d,
6567       .vece = MO_64 },
6568 };
6569 TRANS_FEAT(SQXTUNB, aa64_sve2, do_narrow_extract, a, sqxtunb_ops)
6570 
6571 static void gen_sqxtunt_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
6572 {
6573     TCGv_vec t = tcg_temp_new_vec_matching(d);
6574     int halfbits = 4 << vece;
6575     int64_t max = (1ull << halfbits) - 1;
6576 
6577     tcg_gen_dupi_vec(vece, t, 0);
6578     tcg_gen_smax_vec(vece, n, n, t);
6579     tcg_gen_dupi_vec(vece, t, max);
6580     tcg_gen_umin_vec(vece, n, n, t);
6581     tcg_gen_shli_vec(vece, n, n, halfbits);
6582     tcg_gen_bitsel_vec(vece, d, t, d, n);
6583 }
6584 
6585 static const GVecGen2 sqxtunt_ops[3] = {
6586     { .fniv = gen_sqxtunt_vec,
6587       .opt_opc = sqxtun_list,
6588       .load_dest = true,
6589       .fno = gen_helper_sve2_sqxtunt_h,
6590       .vece = MO_16 },
6591     { .fniv = gen_sqxtunt_vec,
6592       .opt_opc = sqxtun_list,
6593       .load_dest = true,
6594       .fno = gen_helper_sve2_sqxtunt_s,
6595       .vece = MO_32 },
6596     { .fniv = gen_sqxtunt_vec,
6597       .opt_opc = sqxtun_list,
6598       .load_dest = true,
6599       .fno = gen_helper_sve2_sqxtunt_d,
6600       .vece = MO_64 },
6601 };
6602 TRANS_FEAT(SQXTUNT, aa64_sve2, do_narrow_extract, a, sqxtunt_ops)
6603 
6604 static bool do_shr_narrow(DisasContext *s, arg_rri_esz *a,
6605                           const GVecGen2i ops[3])
6606 {
6607     if (a->esz < 0 || a->esz > MO_32) {
6608         return false;
6609     }
6610     assert(a->imm > 0 && a->imm <= (8 << a->esz));
6611     if (sve_access_check(s)) {
6612         unsigned vsz = vec_full_reg_size(s);
6613         tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd),
6614                         vec_full_reg_offset(s, a->rn),
6615                         vsz, vsz, a->imm, &ops[a->esz]);
6616     }
6617     return true;
6618 }
6619 
6620 static void gen_shrnb_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr)
6621 {
6622     int halfbits = 4 << vece;
6623     uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits));
6624 
6625     tcg_gen_shri_i64(d, n, shr);
6626     tcg_gen_andi_i64(d, d, mask);
6627 }
6628 
6629 static void gen_shrnb16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
6630 {
6631     gen_shrnb_i64(MO_16, d, n, shr);
6632 }
6633 
6634 static void gen_shrnb32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
6635 {
6636     gen_shrnb_i64(MO_32, d, n, shr);
6637 }
6638 
6639 static void gen_shrnb64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
6640 {
6641     gen_shrnb_i64(MO_64, d, n, shr);
6642 }
6643 
6644 static void gen_shrnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr)
6645 {
6646     TCGv_vec t = tcg_temp_new_vec_matching(d);
6647     int halfbits = 4 << vece;
6648     uint64_t mask = MAKE_64BIT_MASK(0, halfbits);
6649 
6650     tcg_gen_shri_vec(vece, n, n, shr);
6651     tcg_gen_dupi_vec(vece, t, mask);
6652     tcg_gen_and_vec(vece, d, n, t);
6653 }
6654 
6655 static const TCGOpcode shrnb_vec_list[] = { INDEX_op_shri_vec, 0 };
6656 static const GVecGen2i shrnb_ops[3] = {
6657     { .fni8 = gen_shrnb16_i64,
6658       .fniv = gen_shrnb_vec,
6659       .opt_opc = shrnb_vec_list,
6660       .fno = gen_helper_sve2_shrnb_h,
6661       .vece = MO_16 },
6662     { .fni8 = gen_shrnb32_i64,
6663       .fniv = gen_shrnb_vec,
6664       .opt_opc = shrnb_vec_list,
6665       .fno = gen_helper_sve2_shrnb_s,
6666       .vece = MO_32 },
6667     { .fni8 = gen_shrnb64_i64,
6668       .fniv = gen_shrnb_vec,
6669       .opt_opc = shrnb_vec_list,
6670       .fno = gen_helper_sve2_shrnb_d,
6671       .vece = MO_64 },
6672 };
6673 TRANS_FEAT(SHRNB, aa64_sve2, do_shr_narrow, a, shrnb_ops)
6674 
6675 static void gen_shrnt_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr)
6676 {
6677     int halfbits = 4 << vece;
6678     uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits));
6679 
6680     tcg_gen_shli_i64(n, n, halfbits - shr);
6681     tcg_gen_andi_i64(n, n, ~mask);
6682     tcg_gen_andi_i64(d, d, mask);
6683     tcg_gen_or_i64(d, d, n);
6684 }
6685 
6686 static void gen_shrnt16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
6687 {
6688     gen_shrnt_i64(MO_16, d, n, shr);
6689 }
6690 
6691 static void gen_shrnt32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
6692 {
6693     gen_shrnt_i64(MO_32, d, n, shr);
6694 }
6695 
6696 static void gen_shrnt64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
6697 {
6698     tcg_gen_shri_i64(n, n, shr);
6699     tcg_gen_deposit_i64(d, d, n, 32, 32);
6700 }
6701 
6702 static void gen_shrnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr)
6703 {
6704     TCGv_vec t = tcg_temp_new_vec_matching(d);
6705     int halfbits = 4 << vece;
6706     uint64_t mask = MAKE_64BIT_MASK(0, halfbits);
6707 
6708     tcg_gen_shli_vec(vece, n, n, halfbits - shr);
6709     tcg_gen_dupi_vec(vece, t, mask);
6710     tcg_gen_bitsel_vec(vece, d, t, d, n);
6711 }
6712 
6713 static const TCGOpcode shrnt_vec_list[] = { INDEX_op_shli_vec, 0 };
6714 static const GVecGen2i shrnt_ops[3] = {
6715     { .fni8 = gen_shrnt16_i64,
6716       .fniv = gen_shrnt_vec,
6717       .opt_opc = shrnt_vec_list,
6718       .load_dest = true,
6719       .fno = gen_helper_sve2_shrnt_h,
6720       .vece = MO_16 },
6721     { .fni8 = gen_shrnt32_i64,
6722       .fniv = gen_shrnt_vec,
6723       .opt_opc = shrnt_vec_list,
6724       .load_dest = true,
6725       .fno = gen_helper_sve2_shrnt_s,
6726       .vece = MO_32 },
6727     { .fni8 = gen_shrnt64_i64,
6728       .fniv = gen_shrnt_vec,
6729       .opt_opc = shrnt_vec_list,
6730       .load_dest = true,
6731       .fno = gen_helper_sve2_shrnt_d,
6732       .vece = MO_64 },
6733 };
6734 TRANS_FEAT(SHRNT, aa64_sve2, do_shr_narrow, a, shrnt_ops)
6735 
6736 static const GVecGen2i rshrnb_ops[3] = {
6737     { .fno = gen_helper_sve2_rshrnb_h },
6738     { .fno = gen_helper_sve2_rshrnb_s },
6739     { .fno = gen_helper_sve2_rshrnb_d },
6740 };
6741 TRANS_FEAT(RSHRNB, aa64_sve2, do_shr_narrow, a, rshrnb_ops)
6742 
6743 static const GVecGen2i rshrnt_ops[3] = {
6744     { .fno = gen_helper_sve2_rshrnt_h },
6745     { .fno = gen_helper_sve2_rshrnt_s },
6746     { .fno = gen_helper_sve2_rshrnt_d },
6747 };
6748 TRANS_FEAT(RSHRNT, aa64_sve2, do_shr_narrow, a, rshrnt_ops)
6749 
6750 static void gen_sqshrunb_vec(unsigned vece, TCGv_vec d,
6751                              TCGv_vec n, int64_t shr)
6752 {
6753     TCGv_vec t = tcg_temp_new_vec_matching(d);
6754     int halfbits = 4 << vece;
6755 
6756     tcg_gen_sari_vec(vece, n, n, shr);
6757     tcg_gen_dupi_vec(vece, t, 0);
6758     tcg_gen_smax_vec(vece, n, n, t);
6759     tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
6760     tcg_gen_umin_vec(vece, d, n, t);
6761 }
6762 
6763 static const TCGOpcode sqshrunb_vec_list[] = {
6764     INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_umin_vec, 0
6765 };
6766 static const GVecGen2i sqshrunb_ops[3] = {
6767     { .fniv = gen_sqshrunb_vec,
6768       .opt_opc = sqshrunb_vec_list,
6769       .fno = gen_helper_sve2_sqshrunb_h,
6770       .vece = MO_16 },
6771     { .fniv = gen_sqshrunb_vec,
6772       .opt_opc = sqshrunb_vec_list,
6773       .fno = gen_helper_sve2_sqshrunb_s,
6774       .vece = MO_32 },
6775     { .fniv = gen_sqshrunb_vec,
6776       .opt_opc = sqshrunb_vec_list,
6777       .fno = gen_helper_sve2_sqshrunb_d,
6778       .vece = MO_64 },
6779 };
6780 TRANS_FEAT(SQSHRUNB, aa64_sve2, do_shr_narrow, a, sqshrunb_ops)
6781 
6782 static void gen_sqshrunt_vec(unsigned vece, TCGv_vec d,
6783                              TCGv_vec n, int64_t shr)
6784 {
6785     TCGv_vec t = tcg_temp_new_vec_matching(d);
6786     int halfbits = 4 << vece;
6787 
6788     tcg_gen_sari_vec(vece, n, n, shr);
6789     tcg_gen_dupi_vec(vece, t, 0);
6790     tcg_gen_smax_vec(vece, n, n, t);
6791     tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
6792     tcg_gen_umin_vec(vece, n, n, t);
6793     tcg_gen_shli_vec(vece, n, n, halfbits);
6794     tcg_gen_bitsel_vec(vece, d, t, d, n);
6795 }
6796 
6797 static const TCGOpcode sqshrunt_vec_list[] = {
6798     INDEX_op_shli_vec, INDEX_op_sari_vec,
6799     INDEX_op_smax_vec, INDEX_op_umin_vec, 0
6800 };
6801 static const GVecGen2i sqshrunt_ops[3] = {
6802     { .fniv = gen_sqshrunt_vec,
6803       .opt_opc = sqshrunt_vec_list,
6804       .load_dest = true,
6805       .fno = gen_helper_sve2_sqshrunt_h,
6806       .vece = MO_16 },
6807     { .fniv = gen_sqshrunt_vec,
6808       .opt_opc = sqshrunt_vec_list,
6809       .load_dest = true,
6810       .fno = gen_helper_sve2_sqshrunt_s,
6811       .vece = MO_32 },
6812     { .fniv = gen_sqshrunt_vec,
6813       .opt_opc = sqshrunt_vec_list,
6814       .load_dest = true,
6815       .fno = gen_helper_sve2_sqshrunt_d,
6816       .vece = MO_64 },
6817 };
6818 TRANS_FEAT(SQSHRUNT, aa64_sve2, do_shr_narrow, a, sqshrunt_ops)
6819 
6820 static const GVecGen2i sqrshrunb_ops[3] = {
6821     { .fno = gen_helper_sve2_sqrshrunb_h },
6822     { .fno = gen_helper_sve2_sqrshrunb_s },
6823     { .fno = gen_helper_sve2_sqrshrunb_d },
6824 };
6825 TRANS_FEAT(SQRSHRUNB, aa64_sve2, do_shr_narrow, a, sqrshrunb_ops)
6826 
6827 static const GVecGen2i sqrshrunt_ops[3] = {
6828     { .fno = gen_helper_sve2_sqrshrunt_h },
6829     { .fno = gen_helper_sve2_sqrshrunt_s },
6830     { .fno = gen_helper_sve2_sqrshrunt_d },
6831 };
6832 TRANS_FEAT(SQRSHRUNT, aa64_sve2, do_shr_narrow, a, sqrshrunt_ops)
6833 
6834 static void gen_sqshrnb_vec(unsigned vece, TCGv_vec d,
6835                             TCGv_vec n, int64_t shr)
6836 {
6837     TCGv_vec t = tcg_temp_new_vec_matching(d);
6838     int halfbits = 4 << vece;
6839     int64_t max = MAKE_64BIT_MASK(0, halfbits - 1);
6840     int64_t min = -max - 1;
6841 
6842     tcg_gen_sari_vec(vece, n, n, shr);
6843     tcg_gen_dupi_vec(vece, t, min);
6844     tcg_gen_smax_vec(vece, n, n, t);
6845     tcg_gen_dupi_vec(vece, t, max);
6846     tcg_gen_smin_vec(vece, n, n, t);
6847     tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
6848     tcg_gen_and_vec(vece, d, n, t);
6849 }
6850 
6851 static const TCGOpcode sqshrnb_vec_list[] = {
6852     INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_smin_vec, 0
6853 };
6854 static const GVecGen2i sqshrnb_ops[3] = {
6855     { .fniv = gen_sqshrnb_vec,
6856       .opt_opc = sqshrnb_vec_list,
6857       .fno = gen_helper_sve2_sqshrnb_h,
6858       .vece = MO_16 },
6859     { .fniv = gen_sqshrnb_vec,
6860       .opt_opc = sqshrnb_vec_list,
6861       .fno = gen_helper_sve2_sqshrnb_s,
6862       .vece = MO_32 },
6863     { .fniv = gen_sqshrnb_vec,
6864       .opt_opc = sqshrnb_vec_list,
6865       .fno = gen_helper_sve2_sqshrnb_d,
6866       .vece = MO_64 },
6867 };
6868 TRANS_FEAT(SQSHRNB, aa64_sve2, do_shr_narrow, a, sqshrnb_ops)
6869 
6870 static void gen_sqshrnt_vec(unsigned vece, TCGv_vec d,
6871                              TCGv_vec n, int64_t shr)
6872 {
6873     TCGv_vec t = tcg_temp_new_vec_matching(d);
6874     int halfbits = 4 << vece;
6875     int64_t max = MAKE_64BIT_MASK(0, halfbits - 1);
6876     int64_t min = -max - 1;
6877 
6878     tcg_gen_sari_vec(vece, n, n, shr);
6879     tcg_gen_dupi_vec(vece, t, min);
6880     tcg_gen_smax_vec(vece, n, n, t);
6881     tcg_gen_dupi_vec(vece, t, max);
6882     tcg_gen_smin_vec(vece, n, n, t);
6883     tcg_gen_shli_vec(vece, n, n, halfbits);
6884     tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
6885     tcg_gen_bitsel_vec(vece, d, t, d, n);
6886 }
6887 
6888 static const TCGOpcode sqshrnt_vec_list[] = {
6889     INDEX_op_shli_vec, INDEX_op_sari_vec,
6890     INDEX_op_smax_vec, INDEX_op_smin_vec, 0
6891 };
6892 static const GVecGen2i sqshrnt_ops[3] = {
6893     { .fniv = gen_sqshrnt_vec,
6894       .opt_opc = sqshrnt_vec_list,
6895       .load_dest = true,
6896       .fno = gen_helper_sve2_sqshrnt_h,
6897       .vece = MO_16 },
6898     { .fniv = gen_sqshrnt_vec,
6899       .opt_opc = sqshrnt_vec_list,
6900       .load_dest = true,
6901       .fno = gen_helper_sve2_sqshrnt_s,
6902       .vece = MO_32 },
6903     { .fniv = gen_sqshrnt_vec,
6904       .opt_opc = sqshrnt_vec_list,
6905       .load_dest = true,
6906       .fno = gen_helper_sve2_sqshrnt_d,
6907       .vece = MO_64 },
6908 };
6909 TRANS_FEAT(SQSHRNT, aa64_sve2, do_shr_narrow, a, sqshrnt_ops)
6910 
6911 static const GVecGen2i sqrshrnb_ops[3] = {
6912     { .fno = gen_helper_sve2_sqrshrnb_h },
6913     { .fno = gen_helper_sve2_sqrshrnb_s },
6914     { .fno = gen_helper_sve2_sqrshrnb_d },
6915 };
6916 TRANS_FEAT(SQRSHRNB, aa64_sve2, do_shr_narrow, a, sqrshrnb_ops)
6917 
6918 static const GVecGen2i sqrshrnt_ops[3] = {
6919     { .fno = gen_helper_sve2_sqrshrnt_h },
6920     { .fno = gen_helper_sve2_sqrshrnt_s },
6921     { .fno = gen_helper_sve2_sqrshrnt_d },
6922 };
6923 TRANS_FEAT(SQRSHRNT, aa64_sve2, do_shr_narrow, a, sqrshrnt_ops)
6924 
6925 static void gen_uqshrnb_vec(unsigned vece, TCGv_vec d,
6926                             TCGv_vec n, int64_t shr)
6927 {
6928     TCGv_vec t = tcg_temp_new_vec_matching(d);
6929     int halfbits = 4 << vece;
6930 
6931     tcg_gen_shri_vec(vece, n, n, shr);
6932     tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
6933     tcg_gen_umin_vec(vece, d, n, t);
6934 }
6935 
6936 static const TCGOpcode uqshrnb_vec_list[] = {
6937     INDEX_op_shri_vec, INDEX_op_umin_vec, 0
6938 };
6939 static const GVecGen2i uqshrnb_ops[3] = {
6940     { .fniv = gen_uqshrnb_vec,
6941       .opt_opc = uqshrnb_vec_list,
6942       .fno = gen_helper_sve2_uqshrnb_h,
6943       .vece = MO_16 },
6944     { .fniv = gen_uqshrnb_vec,
6945       .opt_opc = uqshrnb_vec_list,
6946       .fno = gen_helper_sve2_uqshrnb_s,
6947       .vece = MO_32 },
6948     { .fniv = gen_uqshrnb_vec,
6949       .opt_opc = uqshrnb_vec_list,
6950       .fno = gen_helper_sve2_uqshrnb_d,
6951       .vece = MO_64 },
6952 };
6953 TRANS_FEAT(UQSHRNB, aa64_sve2, do_shr_narrow, a, uqshrnb_ops)
6954 
6955 static void gen_uqshrnt_vec(unsigned vece, TCGv_vec d,
6956                             TCGv_vec n, int64_t shr)
6957 {
6958     TCGv_vec t = tcg_temp_new_vec_matching(d);
6959     int halfbits = 4 << vece;
6960 
6961     tcg_gen_shri_vec(vece, n, n, shr);
6962     tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits));
6963     tcg_gen_umin_vec(vece, n, n, t);
6964     tcg_gen_shli_vec(vece, n, n, halfbits);
6965     tcg_gen_bitsel_vec(vece, d, t, d, n);
6966 }
6967 
6968 static const TCGOpcode uqshrnt_vec_list[] = {
6969     INDEX_op_shli_vec, INDEX_op_shri_vec, INDEX_op_umin_vec, 0
6970 };
6971 static const GVecGen2i uqshrnt_ops[3] = {
6972     { .fniv = gen_uqshrnt_vec,
6973       .opt_opc = uqshrnt_vec_list,
6974       .load_dest = true,
6975       .fno = gen_helper_sve2_uqshrnt_h,
6976       .vece = MO_16 },
6977     { .fniv = gen_uqshrnt_vec,
6978       .opt_opc = uqshrnt_vec_list,
6979       .load_dest = true,
6980       .fno = gen_helper_sve2_uqshrnt_s,
6981       .vece = MO_32 },
6982     { .fniv = gen_uqshrnt_vec,
6983       .opt_opc = uqshrnt_vec_list,
6984       .load_dest = true,
6985       .fno = gen_helper_sve2_uqshrnt_d,
6986       .vece = MO_64 },
6987 };
6988 TRANS_FEAT(UQSHRNT, aa64_sve2, do_shr_narrow, a, uqshrnt_ops)
6989 
6990 static const GVecGen2i uqrshrnb_ops[3] = {
6991     { .fno = gen_helper_sve2_uqrshrnb_h },
6992     { .fno = gen_helper_sve2_uqrshrnb_s },
6993     { .fno = gen_helper_sve2_uqrshrnb_d },
6994 };
6995 TRANS_FEAT(UQRSHRNB, aa64_sve2, do_shr_narrow, a, uqrshrnb_ops)
6996 
6997 static const GVecGen2i uqrshrnt_ops[3] = {
6998     { .fno = gen_helper_sve2_uqrshrnt_h },
6999     { .fno = gen_helper_sve2_uqrshrnt_s },
7000     { .fno = gen_helper_sve2_uqrshrnt_d },
7001 };
7002 TRANS_FEAT(UQRSHRNT, aa64_sve2, do_shr_narrow, a, uqrshrnt_ops)
7003 
7004 #define DO_SVE2_ZZZ_NARROW(NAME, name)                                    \
7005     static gen_helper_gvec_3 * const name##_fns[4] = {                    \
7006         NULL,                       gen_helper_sve2_##name##_h,           \
7007         gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d,           \
7008     };                                                                    \
7009     TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzz,                     \
7010                name##_fns[a->esz], a, 0)
7011 
7012 DO_SVE2_ZZZ_NARROW(ADDHNB, addhnb)
7013 DO_SVE2_ZZZ_NARROW(ADDHNT, addhnt)
7014 DO_SVE2_ZZZ_NARROW(RADDHNB, raddhnb)
7015 DO_SVE2_ZZZ_NARROW(RADDHNT, raddhnt)
7016 
7017 DO_SVE2_ZZZ_NARROW(SUBHNB, subhnb)
7018 DO_SVE2_ZZZ_NARROW(SUBHNT, subhnt)
7019 DO_SVE2_ZZZ_NARROW(RSUBHNB, rsubhnb)
7020 DO_SVE2_ZZZ_NARROW(RSUBHNT, rsubhnt)
7021 
7022 static gen_helper_gvec_flags_4 * const match_fns[4] = {
7023     gen_helper_sve2_match_ppzz_b, gen_helper_sve2_match_ppzz_h, NULL, NULL
7024 };
7025 TRANS_FEAT_NONSTREAMING(MATCH, aa64_sve2, do_ppzz_flags, a, match_fns[a->esz])
7026 
7027 static gen_helper_gvec_flags_4 * const nmatch_fns[4] = {
7028     gen_helper_sve2_nmatch_ppzz_b, gen_helper_sve2_nmatch_ppzz_h, NULL, NULL
7029 };
7030 TRANS_FEAT_NONSTREAMING(NMATCH, aa64_sve2, do_ppzz_flags, a, nmatch_fns[a->esz])
7031 
7032 static gen_helper_gvec_4 * const histcnt_fns[4] = {
7033     NULL, NULL, gen_helper_sve2_histcnt_s, gen_helper_sve2_histcnt_d
7034 };
7035 TRANS_FEAT_NONSTREAMING(HISTCNT, aa64_sve2, gen_gvec_ool_arg_zpzz,
7036                         histcnt_fns[a->esz], a, 0)
7037 
7038 TRANS_FEAT_NONSTREAMING(HISTSEG, aa64_sve2, gen_gvec_ool_arg_zzz,
7039                         a->esz == 0 ? gen_helper_sve2_histseg : NULL, a, 0)
7040 
7041 DO_ZPZZ_FP(FADDP, aa64_sve2, sve2_faddp_zpzz)
7042 DO_ZPZZ_FP(FMAXNMP, aa64_sve2, sve2_fmaxnmp_zpzz)
7043 DO_ZPZZ_FP(FMINNMP, aa64_sve2, sve2_fminnmp_zpzz)
7044 DO_ZPZZ_FP(FMAXP, aa64_sve2, sve2_fmaxp_zpzz)
7045 DO_ZPZZ_FP(FMINP, aa64_sve2, sve2_fminp_zpzz)
7046 
7047 /*
7048  * SVE Integer Multiply-Add (unpredicated)
7049  */
7050 
7051 TRANS_FEAT_NONSTREAMING(FMMLA_s, aa64_sve_f32mm, gen_gvec_fpst_zzzz,
7052                         gen_helper_fmmla_s, a->rd, a->rn, a->rm, a->ra,
7053                         0, FPST_FPCR)
7054 TRANS_FEAT_NONSTREAMING(FMMLA_d, aa64_sve_f64mm, gen_gvec_fpst_zzzz,
7055                         gen_helper_fmmla_d, a->rd, a->rn, a->rm, a->ra,
7056                         0, FPST_FPCR)
7057 
7058 static gen_helper_gvec_4 * const sqdmlal_zzzw_fns[] = {
7059     NULL,                           gen_helper_sve2_sqdmlal_zzzw_h,
7060     gen_helper_sve2_sqdmlal_zzzw_s, gen_helper_sve2_sqdmlal_zzzw_d,
7061 };
7062 TRANS_FEAT(SQDMLALB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7063            sqdmlal_zzzw_fns[a->esz], a, 0)
7064 TRANS_FEAT(SQDMLALT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7065            sqdmlal_zzzw_fns[a->esz], a, 3)
7066 TRANS_FEAT(SQDMLALBT, aa64_sve2, gen_gvec_ool_arg_zzzz,
7067            sqdmlal_zzzw_fns[a->esz], a, 2)
7068 
7069 static gen_helper_gvec_4 * const sqdmlsl_zzzw_fns[] = {
7070     NULL,                           gen_helper_sve2_sqdmlsl_zzzw_h,
7071     gen_helper_sve2_sqdmlsl_zzzw_s, gen_helper_sve2_sqdmlsl_zzzw_d,
7072 };
7073 TRANS_FEAT(SQDMLSLB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7074            sqdmlsl_zzzw_fns[a->esz], a, 0)
7075 TRANS_FEAT(SQDMLSLT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7076            sqdmlsl_zzzw_fns[a->esz], a, 3)
7077 TRANS_FEAT(SQDMLSLBT, aa64_sve2, gen_gvec_ool_arg_zzzz,
7078            sqdmlsl_zzzw_fns[a->esz], a, 2)
7079 
7080 static gen_helper_gvec_4 * const sqrdmlah_fns[] = {
7081     gen_helper_sve2_sqrdmlah_b, gen_helper_sve2_sqrdmlah_h,
7082     gen_helper_sve2_sqrdmlah_s, gen_helper_sve2_sqrdmlah_d,
7083 };
7084 TRANS_FEAT(SQRDMLAH_zzzz, aa64_sve2, gen_gvec_ool_arg_zzzz,
7085            sqrdmlah_fns[a->esz], a, 0)
7086 
7087 static gen_helper_gvec_4 * const sqrdmlsh_fns[] = {
7088     gen_helper_sve2_sqrdmlsh_b, gen_helper_sve2_sqrdmlsh_h,
7089     gen_helper_sve2_sqrdmlsh_s, gen_helper_sve2_sqrdmlsh_d,
7090 };
7091 TRANS_FEAT(SQRDMLSH_zzzz, aa64_sve2, gen_gvec_ool_arg_zzzz,
7092            sqrdmlsh_fns[a->esz], a, 0)
7093 
7094 static gen_helper_gvec_4 * const smlal_zzzw_fns[] = {
7095     NULL,                         gen_helper_sve2_smlal_zzzw_h,
7096     gen_helper_sve2_smlal_zzzw_s, gen_helper_sve2_smlal_zzzw_d,
7097 };
7098 TRANS_FEAT(SMLALB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7099            smlal_zzzw_fns[a->esz], a, 0)
7100 TRANS_FEAT(SMLALT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7101            smlal_zzzw_fns[a->esz], a, 1)
7102 
7103 static gen_helper_gvec_4 * const umlal_zzzw_fns[] = {
7104     NULL,                         gen_helper_sve2_umlal_zzzw_h,
7105     gen_helper_sve2_umlal_zzzw_s, gen_helper_sve2_umlal_zzzw_d,
7106 };
7107 TRANS_FEAT(UMLALB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7108            umlal_zzzw_fns[a->esz], a, 0)
7109 TRANS_FEAT(UMLALT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7110            umlal_zzzw_fns[a->esz], a, 1)
7111 
7112 static gen_helper_gvec_4 * const smlsl_zzzw_fns[] = {
7113     NULL,                         gen_helper_sve2_smlsl_zzzw_h,
7114     gen_helper_sve2_smlsl_zzzw_s, gen_helper_sve2_smlsl_zzzw_d,
7115 };
7116 TRANS_FEAT(SMLSLB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7117            smlsl_zzzw_fns[a->esz], a, 0)
7118 TRANS_FEAT(SMLSLT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7119            smlsl_zzzw_fns[a->esz], a, 1)
7120 
7121 static gen_helper_gvec_4 * const umlsl_zzzw_fns[] = {
7122     NULL,                         gen_helper_sve2_umlsl_zzzw_h,
7123     gen_helper_sve2_umlsl_zzzw_s, gen_helper_sve2_umlsl_zzzw_d,
7124 };
7125 TRANS_FEAT(UMLSLB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7126            umlsl_zzzw_fns[a->esz], a, 0)
7127 TRANS_FEAT(UMLSLT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7128            umlsl_zzzw_fns[a->esz], a, 1)
7129 
7130 static gen_helper_gvec_4 * const cmla_fns[] = {
7131     gen_helper_sve2_cmla_zzzz_b, gen_helper_sve2_cmla_zzzz_h,
7132     gen_helper_sve2_cmla_zzzz_s, gen_helper_sve2_cmla_zzzz_d,
7133 };
7134 TRANS_FEAT(CMLA_zzzz, aa64_sve2, gen_gvec_ool_zzzz,
7135            cmla_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot)
7136 
7137 static gen_helper_gvec_4 * const cdot_fns[] = {
7138     NULL, NULL, gen_helper_sve2_cdot_zzzz_s, gen_helper_sve2_cdot_zzzz_d
7139 };
7140 TRANS_FEAT(CDOT_zzzz, aa64_sve2, gen_gvec_ool_zzzz,
7141            cdot_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot)
7142 
7143 static gen_helper_gvec_4 * const sqrdcmlah_fns[] = {
7144     gen_helper_sve2_sqrdcmlah_zzzz_b, gen_helper_sve2_sqrdcmlah_zzzz_h,
7145     gen_helper_sve2_sqrdcmlah_zzzz_s, gen_helper_sve2_sqrdcmlah_zzzz_d,
7146 };
7147 TRANS_FEAT(SQRDCMLAH_zzzz, aa64_sve2, gen_gvec_ool_zzzz,
7148            sqrdcmlah_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot)
7149 
7150 TRANS_FEAT(USDOT_zzzz, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
7151            a->esz == 2 ? gen_helper_gvec_usdot_b : NULL, a, 0)
7152 
7153 TRANS_FEAT_NONSTREAMING(AESMC, aa64_sve2_aes, gen_gvec_ool_zz,
7154                         gen_helper_crypto_aesmc, a->rd, a->rd, 0)
7155 TRANS_FEAT_NONSTREAMING(AESIMC, aa64_sve2_aes, gen_gvec_ool_zz,
7156                         gen_helper_crypto_aesimc, a->rd, a->rd, 0)
7157 
7158 TRANS_FEAT_NONSTREAMING(AESE, aa64_sve2_aes, gen_gvec_ool_arg_zzz,
7159                         gen_helper_crypto_aese, a, 0)
7160 TRANS_FEAT_NONSTREAMING(AESD, aa64_sve2_aes, gen_gvec_ool_arg_zzz,
7161                         gen_helper_crypto_aesd, a, 0)
7162 
7163 TRANS_FEAT_NONSTREAMING(SM4E, aa64_sve2_sm4, gen_gvec_ool_arg_zzz,
7164                         gen_helper_crypto_sm4e, a, 0)
7165 TRANS_FEAT_NONSTREAMING(SM4EKEY, aa64_sve2_sm4, gen_gvec_ool_arg_zzz,
7166                         gen_helper_crypto_sm4ekey, a, 0)
7167 
7168 TRANS_FEAT_NONSTREAMING(RAX1, aa64_sve2_sha3, gen_gvec_fn_arg_zzz,
7169                         gen_gvec_rax1, a)
7170 
7171 TRANS_FEAT(FCVTNT_sh, aa64_sve2, gen_gvec_fpst_arg_zpz,
7172            gen_helper_sve2_fcvtnt_sh, a, 0, FPST_FPCR)
7173 TRANS_FEAT(FCVTNT_ds, aa64_sve2, gen_gvec_fpst_arg_zpz,
7174            gen_helper_sve2_fcvtnt_ds, a, 0, FPST_FPCR)
7175 
7176 TRANS_FEAT(BFCVTNT, aa64_sve_bf16, gen_gvec_fpst_arg_zpz,
7177            gen_helper_sve_bfcvtnt, a, 0, FPST_FPCR)
7178 
7179 TRANS_FEAT(FCVTLT_hs, aa64_sve2, gen_gvec_fpst_arg_zpz,
7180            gen_helper_sve2_fcvtlt_hs, a, 0, FPST_FPCR)
7181 TRANS_FEAT(FCVTLT_sd, aa64_sve2, gen_gvec_fpst_arg_zpz,
7182            gen_helper_sve2_fcvtlt_sd, a, 0, FPST_FPCR)
7183 
7184 TRANS_FEAT(FCVTX_ds, aa64_sve2, do_frint_mode, a,
7185            FPROUNDING_ODD, gen_helper_sve_fcvt_ds)
7186 TRANS_FEAT(FCVTXNT_ds, aa64_sve2, do_frint_mode, a,
7187            FPROUNDING_ODD, gen_helper_sve2_fcvtnt_ds)
7188 
7189 static gen_helper_gvec_3_ptr * const flogb_fns[] = {
7190     NULL,               gen_helper_flogb_h,
7191     gen_helper_flogb_s, gen_helper_flogb_d
7192 };
7193 TRANS_FEAT(FLOGB, aa64_sve2, gen_gvec_fpst_arg_zpz, flogb_fns[a->esz],
7194            a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
7195 
7196 static bool do_FMLAL_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sub, bool sel)
7197 {
7198     return gen_gvec_ptr_zzzz(s, gen_helper_sve2_fmlal_zzzw_s,
7199                              a->rd, a->rn, a->rm, a->ra,
7200                              (sel << 1) | sub, cpu_env);
7201 }
7202 
7203 TRANS_FEAT(FMLALB_zzzw, aa64_sve2, do_FMLAL_zzzw, a, false, false)
7204 TRANS_FEAT(FMLALT_zzzw, aa64_sve2, do_FMLAL_zzzw, a, false, true)
7205 TRANS_FEAT(FMLSLB_zzzw, aa64_sve2, do_FMLAL_zzzw, a, true, false)
7206 TRANS_FEAT(FMLSLT_zzzw, aa64_sve2, do_FMLAL_zzzw, a, true, true)
7207 
7208 static bool do_FMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a, bool sub, bool sel)
7209 {
7210     return gen_gvec_ptr_zzzz(s, gen_helper_sve2_fmlal_zzxw_s,
7211                              a->rd, a->rn, a->rm, a->ra,
7212                              (a->index << 2) | (sel << 1) | sub, cpu_env);
7213 }
7214 
7215 TRANS_FEAT(FMLALB_zzxw, aa64_sve2, do_FMLAL_zzxw, a, false, false)
7216 TRANS_FEAT(FMLALT_zzxw, aa64_sve2, do_FMLAL_zzxw, a, false, true)
7217 TRANS_FEAT(FMLSLB_zzxw, aa64_sve2, do_FMLAL_zzxw, a, true, false)
7218 TRANS_FEAT(FMLSLT_zzxw, aa64_sve2, do_FMLAL_zzxw, a, true, true)
7219 
7220 TRANS_FEAT_NONSTREAMING(SMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
7221                         gen_helper_gvec_smmla_b, a, 0)
7222 TRANS_FEAT_NONSTREAMING(USMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
7223                         gen_helper_gvec_usmmla_b, a, 0)
7224 TRANS_FEAT_NONSTREAMING(UMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
7225                         gen_helper_gvec_ummla_b, a, 0)
7226 
7227 TRANS_FEAT(BFDOT_zzzz, aa64_sve_bf16, gen_gvec_ool_arg_zzzz,
7228            gen_helper_gvec_bfdot, a, 0)
7229 TRANS_FEAT(BFDOT_zzxz, aa64_sve_bf16, gen_gvec_ool_arg_zzxz,
7230            gen_helper_gvec_bfdot_idx, a)
7231 
7232 TRANS_FEAT_NONSTREAMING(BFMMLA, aa64_sve_bf16, gen_gvec_ool_arg_zzzz,
7233                         gen_helper_gvec_bfmmla, a, 0)
7234 
7235 static bool do_BFMLAL_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel)
7236 {
7237     return gen_gvec_fpst_zzzz(s, gen_helper_gvec_bfmlal,
7238                               a->rd, a->rn, a->rm, a->ra, sel, FPST_FPCR);
7239 }
7240 
7241 TRANS_FEAT(BFMLALB_zzzw, aa64_sve_bf16, do_BFMLAL_zzzw, a, false)
7242 TRANS_FEAT(BFMLALT_zzzw, aa64_sve_bf16, do_BFMLAL_zzzw, a, true)
7243 
7244 static bool do_BFMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a, bool sel)
7245 {
7246     return gen_gvec_fpst_zzzz(s, gen_helper_gvec_bfmlal_idx,
7247                               a->rd, a->rn, a->rm, a->ra,
7248                               (a->index << 1) | sel, FPST_FPCR);
7249 }
7250 
7251 TRANS_FEAT(BFMLALB_zzxw, aa64_sve_bf16, do_BFMLAL_zzxw, a, false)
7252 TRANS_FEAT(BFMLALT_zzxw, aa64_sve_bf16, do_BFMLAL_zzxw, a, true)
7253 
7254 static bool trans_PSEL(DisasContext *s, arg_psel *a)
7255 {
7256     int vl = vec_full_reg_size(s);
7257     int pl = pred_gvec_reg_size(s);
7258     int elements = vl >> a->esz;
7259     TCGv_i64 tmp, didx, dbit;
7260     TCGv_ptr ptr;
7261 
7262     if (!dc_isar_feature(aa64_sme, s)) {
7263         return false;
7264     }
7265     if (!sve_access_check(s)) {
7266         return true;
7267     }
7268 
7269     tmp = tcg_temp_new_i64();
7270     dbit = tcg_temp_new_i64();
7271     didx = tcg_temp_new_i64();
7272     ptr = tcg_temp_new_ptr();
7273 
7274     /* Compute the predicate element. */
7275     tcg_gen_addi_i64(tmp, cpu_reg(s, a->rv), a->imm);
7276     if (is_power_of_2(elements)) {
7277         tcg_gen_andi_i64(tmp, tmp, elements - 1);
7278     } else {
7279         tcg_gen_remu_i64(tmp, tmp, tcg_constant_i64(elements));
7280     }
7281 
7282     /* Extract the predicate byte and bit indices. */
7283     tcg_gen_shli_i64(tmp, tmp, a->esz);
7284     tcg_gen_andi_i64(dbit, tmp, 7);
7285     tcg_gen_shri_i64(didx, tmp, 3);
7286     if (HOST_BIG_ENDIAN) {
7287         tcg_gen_xori_i64(didx, didx, 7);
7288     }
7289 
7290     /* Load the predicate word. */
7291     tcg_gen_trunc_i64_ptr(ptr, didx);
7292     tcg_gen_add_ptr(ptr, ptr, cpu_env);
7293     tcg_gen_ld8u_i64(tmp, ptr, pred_full_reg_offset(s, a->pm));
7294 
7295     /* Extract the predicate bit and replicate to MO_64. */
7296     tcg_gen_shr_i64(tmp, tmp, dbit);
7297     tcg_gen_andi_i64(tmp, tmp, 1);
7298     tcg_gen_neg_i64(tmp, tmp);
7299 
7300     /* Apply to either copy the source, or write zeros. */
7301     tcg_gen_gvec_ands(MO_64, pred_full_reg_offset(s, a->pd),
7302                       pred_full_reg_offset(s, a->pn), tmp, pl, pl);
7303     return true;
7304 }
7305 
7306 static void gen_sclamp_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_i32 a)
7307 {
7308     tcg_gen_smax_i32(d, a, n);
7309     tcg_gen_smin_i32(d, d, m);
7310 }
7311 
7312 static void gen_sclamp_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 a)
7313 {
7314     tcg_gen_smax_i64(d, a, n);
7315     tcg_gen_smin_i64(d, d, m);
7316 }
7317 
7318 static void gen_sclamp_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
7319                            TCGv_vec m, TCGv_vec a)
7320 {
7321     tcg_gen_smax_vec(vece, d, a, n);
7322     tcg_gen_smin_vec(vece, d, d, m);
7323 }
7324 
7325 static void gen_sclamp(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
7326                        uint32_t a, uint32_t oprsz, uint32_t maxsz)
7327 {
7328     static const TCGOpcode vecop[] = {
7329         INDEX_op_smin_vec, INDEX_op_smax_vec, 0
7330     };
7331     static const GVecGen4 ops[4] = {
7332         { .fniv = gen_sclamp_vec,
7333           .fno  = gen_helper_gvec_sclamp_b,
7334           .opt_opc = vecop,
7335           .vece = MO_8 },
7336         { .fniv = gen_sclamp_vec,
7337           .fno  = gen_helper_gvec_sclamp_h,
7338           .opt_opc = vecop,
7339           .vece = MO_16 },
7340         { .fni4 = gen_sclamp_i32,
7341           .fniv = gen_sclamp_vec,
7342           .fno  = gen_helper_gvec_sclamp_s,
7343           .opt_opc = vecop,
7344           .vece = MO_32 },
7345         { .fni8 = gen_sclamp_i64,
7346           .fniv = gen_sclamp_vec,
7347           .fno  = gen_helper_gvec_sclamp_d,
7348           .opt_opc = vecop,
7349           .vece = MO_64,
7350           .prefer_i64 = TCG_TARGET_REG_BITS == 64 }
7351     };
7352     tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &ops[vece]);
7353 }
7354 
7355 TRANS_FEAT(SCLAMP, aa64_sme, gen_gvec_fn_arg_zzzz, gen_sclamp, a)
7356 
7357 static void gen_uclamp_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_i32 a)
7358 {
7359     tcg_gen_umax_i32(d, a, n);
7360     tcg_gen_umin_i32(d, d, m);
7361 }
7362 
7363 static void gen_uclamp_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 a)
7364 {
7365     tcg_gen_umax_i64(d, a, n);
7366     tcg_gen_umin_i64(d, d, m);
7367 }
7368 
7369 static void gen_uclamp_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
7370                            TCGv_vec m, TCGv_vec a)
7371 {
7372     tcg_gen_umax_vec(vece, d, a, n);
7373     tcg_gen_umin_vec(vece, d, d, m);
7374 }
7375 
7376 static void gen_uclamp(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
7377                        uint32_t a, uint32_t oprsz, uint32_t maxsz)
7378 {
7379     static const TCGOpcode vecop[] = {
7380         INDEX_op_umin_vec, INDEX_op_umax_vec, 0
7381     };
7382     static const GVecGen4 ops[4] = {
7383         { .fniv = gen_uclamp_vec,
7384           .fno  = gen_helper_gvec_uclamp_b,
7385           .opt_opc = vecop,
7386           .vece = MO_8 },
7387         { .fniv = gen_uclamp_vec,
7388           .fno  = gen_helper_gvec_uclamp_h,
7389           .opt_opc = vecop,
7390           .vece = MO_16 },
7391         { .fni4 = gen_uclamp_i32,
7392           .fniv = gen_uclamp_vec,
7393           .fno  = gen_helper_gvec_uclamp_s,
7394           .opt_opc = vecop,
7395           .vece = MO_32 },
7396         { .fni8 = gen_uclamp_i64,
7397           .fniv = gen_uclamp_vec,
7398           .fno  = gen_helper_gvec_uclamp_d,
7399           .opt_opc = vecop,
7400           .vece = MO_64,
7401           .prefer_i64 = TCG_TARGET_REG_BITS == 64 }
7402     };
7403     tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &ops[vece]);
7404 }
7405 
7406 TRANS_FEAT(UCLAMP, aa64_sme, gen_gvec_fn_arg_zzzz, gen_uclamp, a)
7407