1/*
2 * QEMU TCG support -- s390x vector instruction translation functions
3 *
4 * Copyright (C) 2019 Red Hat Inc
5 *
6 * Authors:
7 *   David Hildenbrand <david@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
11 */
12
13/*
14 * For most instructions that use the same element size for reads and
15 * writes, we can use real gvec vector expansion, which potantially uses
16 * real host vector instructions. As they only work up to 64 bit elements,
17 * 128 bit elements (vector is a single element) have to be handled
18 * differently. Operations that are too complicated to encode via TCG ops
19 * are handled via gvec ool (out-of-line) handlers.
20 *
21 * As soon as instructions use different element sizes for reads and writes
22 * or access elements "out of their element scope" we expand them manually
23 * in fancy loops, as gvec expansion does not deal with actual element
24 * numbers and does also not support access to other elements.
25 *
26 * 128 bit elements:
27 *  As we only have i32/i64, such elements have to be loaded into two
28 *  i64 values and can then be processed e.g. by tcg_gen_add2_i64.
29 *
30 * Sizes:
31 *  On s390x, the operand size (oprsz) and the maximum size (maxsz) are
32 *  always 16 (128 bit). What gvec code calls "vece", s390x calls "es",
33 *  a.k.a. "element size". These values nicely map to MO_8 ... MO_64. Only
34 *  128 bit element size has to be treated in a special way (MO_64 + 1).
35 *  We will use ES_* instead of MO_* for this reason in this file.
36 *
37 * CC handling:
38 *  As gvec ool-helpers can currently not return values (besides via
39 *  pointers like vectors or tcg_env), whenever we have to set the CC and
40 *  can't conclude the value from the result vector, we will directly
41 *  set it in "env->cc_op" and mark it as static via set_cc_static()".
42 *  Whenever this is done, the helper writes globals (cc_op).
43 */
44
45#define NUM_VEC_ELEMENT_BYTES(es) (1 << (es))
46#define NUM_VEC_ELEMENTS(es) (16 / NUM_VEC_ELEMENT_BYTES(es))
47#define NUM_VEC_ELEMENT_BITS(es) (NUM_VEC_ELEMENT_BYTES(es) * BITS_PER_BYTE)
48
49#define ES_8    MO_8
50#define ES_16   MO_16
51#define ES_32   MO_32
52#define ES_64   MO_64
53#define ES_128  4
54
55/* Floating-Point Format */
56#define FPF_SHORT       2
57#define FPF_LONG        3
58#define FPF_EXT         4
59
60static inline bool valid_vec_element(uint16_t enr, MemOp es)
61{
62    return !(enr & ~(NUM_VEC_ELEMENTS(es) - 1));
63}
64
65static void read_vec_element_i64(TCGv_i64 dst, uint8_t reg, uint8_t enr,
66                                 MemOp memop)
67{
68    const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
69
70    switch ((unsigned)memop) {
71    case ES_8:
72        tcg_gen_ld8u_i64(dst, tcg_env, offs);
73        break;
74    case ES_16:
75        tcg_gen_ld16u_i64(dst, tcg_env, offs);
76        break;
77    case ES_32:
78        tcg_gen_ld32u_i64(dst, tcg_env, offs);
79        break;
80    case ES_8 | MO_SIGN:
81        tcg_gen_ld8s_i64(dst, tcg_env, offs);
82        break;
83    case ES_16 | MO_SIGN:
84        tcg_gen_ld16s_i64(dst, tcg_env, offs);
85        break;
86    case ES_32 | MO_SIGN:
87        tcg_gen_ld32s_i64(dst, tcg_env, offs);
88        break;
89    case ES_64:
90    case ES_64 | MO_SIGN:
91        tcg_gen_ld_i64(dst, tcg_env, offs);
92        break;
93    default:
94        g_assert_not_reached();
95    }
96}
97
98static void read_vec_element_i32(TCGv_i32 dst, uint8_t reg, uint8_t enr,
99                                 MemOp memop)
100{
101    const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
102
103    switch (memop) {
104    case ES_8:
105        tcg_gen_ld8u_i32(dst, tcg_env, offs);
106        break;
107    case ES_16:
108        tcg_gen_ld16u_i32(dst, tcg_env, offs);
109        break;
110    case ES_8 | MO_SIGN:
111        tcg_gen_ld8s_i32(dst, tcg_env, offs);
112        break;
113    case ES_16 | MO_SIGN:
114        tcg_gen_ld16s_i32(dst, tcg_env, offs);
115        break;
116    case ES_32:
117    case ES_32 | MO_SIGN:
118        tcg_gen_ld_i32(dst, tcg_env, offs);
119        break;
120    default:
121        g_assert_not_reached();
122    }
123}
124
125static void write_vec_element_i64(TCGv_i64 src, int reg, uint8_t enr,
126                                  MemOp memop)
127{
128    const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
129
130    switch (memop) {
131    case ES_8:
132        tcg_gen_st8_i64(src, tcg_env, offs);
133        break;
134    case ES_16:
135        tcg_gen_st16_i64(src, tcg_env, offs);
136        break;
137    case ES_32:
138        tcg_gen_st32_i64(src, tcg_env, offs);
139        break;
140    case ES_64:
141        tcg_gen_st_i64(src, tcg_env, offs);
142        break;
143    default:
144        g_assert_not_reached();
145    }
146}
147
148static void write_vec_element_i32(TCGv_i32 src, int reg, uint8_t enr,
149                                  MemOp memop)
150{
151    const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
152
153    switch (memop) {
154    case ES_8:
155        tcg_gen_st8_i32(src, tcg_env, offs);
156        break;
157    case ES_16:
158        tcg_gen_st16_i32(src, tcg_env, offs);
159        break;
160    case ES_32:
161        tcg_gen_st_i32(src, tcg_env, offs);
162        break;
163    default:
164        g_assert_not_reached();
165    }
166}
167
168static void get_vec_element_ptr_i64(TCGv_ptr ptr, uint8_t reg, TCGv_i64 enr,
169                                    uint8_t es)
170{
171    TCGv_i64 tmp = tcg_temp_new_i64();
172
173    /* mask off invalid parts from the element nr */
174    tcg_gen_andi_i64(tmp, enr, NUM_VEC_ELEMENTS(es) - 1);
175
176    /* convert it to an element offset relative to tcg_env (vec_reg_offset() */
177    tcg_gen_shli_i64(tmp, tmp, es);
178#if !HOST_BIG_ENDIAN
179    tcg_gen_xori_i64(tmp, tmp, 8 - NUM_VEC_ELEMENT_BYTES(es));
180#endif
181    tcg_gen_addi_i64(tmp, tmp, vec_full_reg_offset(reg));
182
183    /* generate the final ptr by adding tcg_env */
184    tcg_gen_trunc_i64_ptr(ptr, tmp);
185    tcg_gen_add_ptr(ptr, ptr, tcg_env);
186}
187
188#define gen_gvec_2(v1, v2, gen) \
189    tcg_gen_gvec_2(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
190                   16, 16, gen)
191#define gen_gvec_2s(v1, v2, c, gen) \
192    tcg_gen_gvec_2s(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
193                    16, 16, c, gen)
194#define gen_gvec_2_ool(v1, v2, data, fn) \
195    tcg_gen_gvec_2_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
196                       16, 16, data, fn)
197#define gen_gvec_2i_ool(v1, v2, c, data, fn) \
198    tcg_gen_gvec_2i_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
199                        c, 16, 16, data, fn)
200#define gen_gvec_2_ptr(v1, v2, ptr, data, fn) \
201    tcg_gen_gvec_2_ptr(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
202                       ptr, 16, 16, data, fn)
203#define gen_gvec_3(v1, v2, v3, gen) \
204    tcg_gen_gvec_3(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
205                   vec_full_reg_offset(v3), 16, 16, gen)
206#define gen_gvec_3_ool(v1, v2, v3, data, fn) \
207    tcg_gen_gvec_3_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
208                       vec_full_reg_offset(v3), 16, 16, data, fn)
209#define gen_gvec_3_ptr(v1, v2, v3, ptr, data, fn) \
210    tcg_gen_gvec_3_ptr(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
211                       vec_full_reg_offset(v3), ptr, 16, 16, data, fn)
212#define gen_gvec_3i(v1, v2, v3, c, gen) \
213    tcg_gen_gvec_3i(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
214                    vec_full_reg_offset(v3), 16, 16, c, gen)
215#define gen_gvec_4(v1, v2, v3, v4, gen) \
216    tcg_gen_gvec_4(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
217                   vec_full_reg_offset(v3), vec_full_reg_offset(v4), \
218                   16, 16, gen)
219#define gen_gvec_4_ool(v1, v2, v3, v4, data, fn) \
220    tcg_gen_gvec_4_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
221                       vec_full_reg_offset(v3), vec_full_reg_offset(v4), \
222                       16, 16, data, fn)
223#define gen_gvec_4_ptr(v1, v2, v3, v4, ptr, data, fn) \
224    tcg_gen_gvec_4_ptr(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
225                       vec_full_reg_offset(v3), vec_full_reg_offset(v4), \
226                       ptr, 16, 16, data, fn)
227#define gen_gvec_dup_i64(es, v1, c) \
228    tcg_gen_gvec_dup_i64(es, vec_full_reg_offset(v1), 16, 16, c)
229#define gen_gvec_mov(v1, v2) \
230    tcg_gen_gvec_mov(0, vec_full_reg_offset(v1), vec_full_reg_offset(v2), 16, \
231                     16)
232#define gen_gvec_dup_imm(es, v1, c) \
233    tcg_gen_gvec_dup_imm(es, vec_full_reg_offset(v1), 16, 16, c);
234#define gen_gvec_fn_2(fn, es, v1, v2) \
235    tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
236                      16, 16)
237#define gen_gvec_fn_2i(fn, es, v1, v2, c) \
238    tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
239                      c, 16, 16)
240#define gen_gvec_fn_2s(fn, es, v1, v2, s) \
241    tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
242                      s, 16, 16)
243#define gen_gvec_fn_3(fn, es, v1, v2, v3) \
244    tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
245                      vec_full_reg_offset(v3), 16, 16)
246#define gen_gvec_fn_4(fn, es, v1, v2, v3, v4) \
247    tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
248                      vec_full_reg_offset(v3), vec_full_reg_offset(v4), 16, 16)
249
250/*
251 * Helper to carry out a 128 bit vector computation using 2 i64 values per
252 * vector.
253 */
254typedef void (*gen_gvec128_3_i64_fn)(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
255                                     TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh);
256static void gen_gvec128_3_i64(gen_gvec128_3_i64_fn fn, uint8_t d, uint8_t a,
257                              uint8_t b)
258{
259        TCGv_i64 dh = tcg_temp_new_i64();
260        TCGv_i64 dl = tcg_temp_new_i64();
261        TCGv_i64 ah = tcg_temp_new_i64();
262        TCGv_i64 al = tcg_temp_new_i64();
263        TCGv_i64 bh = tcg_temp_new_i64();
264        TCGv_i64 bl = tcg_temp_new_i64();
265
266        read_vec_element_i64(ah, a, 0, ES_64);
267        read_vec_element_i64(al, a, 1, ES_64);
268        read_vec_element_i64(bh, b, 0, ES_64);
269        read_vec_element_i64(bl, b, 1, ES_64);
270        fn(dl, dh, al, ah, bl, bh);
271        write_vec_element_i64(dh, d, 0, ES_64);
272        write_vec_element_i64(dl, d, 1, ES_64);
273}
274
275typedef void (*gen_gvec128_4_i64_fn)(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
276                                     TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh,
277                                     TCGv_i64 cl, TCGv_i64 ch);
278static void gen_gvec128_4_i64(gen_gvec128_4_i64_fn fn, uint8_t d, uint8_t a,
279                              uint8_t b, uint8_t c)
280{
281        TCGv_i64 dh = tcg_temp_new_i64();
282        TCGv_i64 dl = tcg_temp_new_i64();
283        TCGv_i64 ah = tcg_temp_new_i64();
284        TCGv_i64 al = tcg_temp_new_i64();
285        TCGv_i64 bh = tcg_temp_new_i64();
286        TCGv_i64 bl = tcg_temp_new_i64();
287        TCGv_i64 ch = tcg_temp_new_i64();
288        TCGv_i64 cl = tcg_temp_new_i64();
289
290        read_vec_element_i64(ah, a, 0, ES_64);
291        read_vec_element_i64(al, a, 1, ES_64);
292        read_vec_element_i64(bh, b, 0, ES_64);
293        read_vec_element_i64(bl, b, 1, ES_64);
294        read_vec_element_i64(ch, c, 0, ES_64);
295        read_vec_element_i64(cl, c, 1, ES_64);
296        fn(dl, dh, al, ah, bl, bh, cl, ch);
297        write_vec_element_i64(dh, d, 0, ES_64);
298        write_vec_element_i64(dl, d, 1, ES_64);
299}
300
301static void gen_addi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
302                          uint64_t b)
303{
304    TCGv_i64 bl = tcg_constant_i64(b);
305    TCGv_i64 bh = tcg_constant_i64(0);
306
307    tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
308}
309
310static DisasJumpType op_vbperm(DisasContext *s, DisasOps *o)
311{
312    gen_gvec_3_ool(get_field(s, v1), get_field(s, v2), get_field(s, v3), 0,
313                   gen_helper_gvec_vbperm);
314
315    return DISAS_NEXT;
316}
317
318static DisasJumpType op_vge(DisasContext *s, DisasOps *o)
319{
320    const uint8_t es = s->insn->data;
321    const uint8_t enr = get_field(s, m3);
322    TCGv_i64 tmp;
323
324    if (!valid_vec_element(enr, es)) {
325        gen_program_exception(s, PGM_SPECIFICATION);
326        return DISAS_NORETURN;
327    }
328
329    tmp = tcg_temp_new_i64();
330    read_vec_element_i64(tmp, get_field(s, v2), enr, es);
331    tcg_gen_add_i64(o->addr1, o->addr1, tmp);
332    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 0);
333
334    tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
335    write_vec_element_i64(tmp, get_field(s, v1), enr, es);
336    return DISAS_NEXT;
337}
338
339static uint64_t generate_byte_mask(uint8_t mask)
340{
341    uint64_t r = 0;
342    int i;
343
344    for (i = 0; i < 8; i++) {
345        if ((mask >> i) & 1) {
346            r |= 0xffull << (i * 8);
347        }
348    }
349    return r;
350}
351
352static DisasJumpType op_vgbm(DisasContext *s, DisasOps *o)
353{
354    const uint16_t i2 = get_field(s, i2);
355
356    if (i2 == (i2 & 0xff) * 0x0101) {
357        /*
358         * Masks for both 64 bit elements of the vector are the same.
359         * Trust tcg to produce a good constant loading.
360         */
361        gen_gvec_dup_imm(ES_64, get_field(s, v1),
362                         generate_byte_mask(i2 & 0xff));
363    } else {
364        TCGv_i64 t = tcg_temp_new_i64();
365
366        tcg_gen_movi_i64(t, generate_byte_mask(i2 >> 8));
367        write_vec_element_i64(t, get_field(s, v1), 0, ES_64);
368        tcg_gen_movi_i64(t, generate_byte_mask(i2));
369        write_vec_element_i64(t, get_field(s, v1), 1, ES_64);
370    }
371    return DISAS_NEXT;
372}
373
374static DisasJumpType op_vgm(DisasContext *s, DisasOps *o)
375{
376    const uint8_t es = get_field(s, m4);
377    const uint8_t bits = NUM_VEC_ELEMENT_BITS(es);
378    const uint8_t i2 = get_field(s, i2) & (bits - 1);
379    const uint8_t i3 = get_field(s, i3) & (bits - 1);
380    uint64_t mask = 0;
381    int i;
382
383    if (es > ES_64) {
384        gen_program_exception(s, PGM_SPECIFICATION);
385        return DISAS_NORETURN;
386    }
387
388    /* generate the mask - take care of wrapping */
389    for (i = i2; ; i = (i + 1) % bits) {
390        mask |= 1ull << (bits - i - 1);
391        if (i == i3) {
392            break;
393        }
394    }
395
396    gen_gvec_dup_imm(es, get_field(s, v1), mask);
397    return DISAS_NEXT;
398}
399
400static DisasJumpType op_vl(DisasContext *s, DisasOps *o)
401{
402    TCGv_i64 t0 = tcg_temp_new_i64();
403    TCGv_i64 t1 = tcg_temp_new_i64();
404
405    tcg_gen_qemu_ld_i64(t0, o->addr1, get_mem_index(s), MO_TEUQ);
406    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
407    tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEUQ);
408    write_vec_element_i64(t0, get_field(s, v1), 0, ES_64);
409    write_vec_element_i64(t1, get_field(s, v1), 1, ES_64);
410    return DISAS_NEXT;
411}
412
413static DisasJumpType op_vlr(DisasContext *s, DisasOps *o)
414{
415    gen_gvec_mov(get_field(s, v1), get_field(s, v2));
416    return DISAS_NEXT;
417}
418
419static DisasJumpType op_vlrep(DisasContext *s, DisasOps *o)
420{
421    const uint8_t es = get_field(s, m3);
422    TCGv_i64 tmp;
423
424    if (es > ES_64) {
425        gen_program_exception(s, PGM_SPECIFICATION);
426        return DISAS_NORETURN;
427    }
428
429    tmp = tcg_temp_new_i64();
430    tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
431    gen_gvec_dup_i64(es, get_field(s, v1), tmp);
432    return DISAS_NEXT;
433}
434
435static DisasJumpType op_vlebr(DisasContext *s, DisasOps *o)
436{
437    const uint8_t es = s->insn->data;
438    const uint8_t enr = get_field(s, m3);
439    TCGv_i64 tmp;
440
441    if (!valid_vec_element(enr, es)) {
442        gen_program_exception(s, PGM_SPECIFICATION);
443        return DISAS_NORETURN;
444    }
445
446    tmp = tcg_temp_new_i64();
447    tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_LE | es);
448    write_vec_element_i64(tmp, get_field(s, v1), enr, es);
449    return DISAS_NEXT;
450}
451
452static DisasJumpType op_vlbrrep(DisasContext *s, DisasOps *o)
453{
454    const uint8_t es = get_field(s, m3);
455    TCGv_i64 tmp;
456
457    if (es < ES_16 || es > ES_64) {
458        gen_program_exception(s, PGM_SPECIFICATION);
459        return DISAS_NORETURN;
460    }
461
462    tmp = tcg_temp_new_i64();
463    tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_LE | es);
464    gen_gvec_dup_i64(es, get_field(s, v1), tmp);
465    return DISAS_NEXT;
466}
467
468static DisasJumpType op_vllebrz(DisasContext *s, DisasOps *o)
469{
470    const uint8_t m3 = get_field(s, m3);
471    TCGv_i64 tmp;
472    int es, lshift;
473
474    switch (m3) {
475    case ES_16:
476    case ES_32:
477    case ES_64:
478        es = m3;
479        lshift = 0;
480        break;
481    case 6:
482        es = ES_32;
483        lshift = 32;
484        break;
485    default:
486        gen_program_exception(s, PGM_SPECIFICATION);
487        return DISAS_NORETURN;
488    }
489
490    tmp = tcg_temp_new_i64();
491    tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_LE | es);
492    tcg_gen_shli_i64(tmp, tmp, lshift);
493
494    write_vec_element_i64(tmp, get_field(s, v1), 0, ES_64);
495    write_vec_element_i64(tcg_constant_i64(0), get_field(s, v1), 1, ES_64);
496    return DISAS_NEXT;
497}
498
499static DisasJumpType op_vlbr(DisasContext *s, DisasOps *o)
500{
501    const uint8_t es = get_field(s, m3);
502    TCGv_i64 t0, t1;
503
504    if (es < ES_16 || es > ES_128) {
505        gen_program_exception(s, PGM_SPECIFICATION);
506        return DISAS_NORETURN;
507    }
508
509    t0 = tcg_temp_new_i64();
510    t1 = tcg_temp_new_i64();
511
512
513    if (es == ES_128) {
514        tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_LEUQ);
515        gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
516        tcg_gen_qemu_ld_i64(t0, o->addr1, get_mem_index(s), MO_LEUQ);
517        goto write;
518    }
519
520    /* Begin with byte reversed doublewords... */
521    tcg_gen_qemu_ld_i64(t0, o->addr1, get_mem_index(s), MO_LEUQ);
522    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
523    tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_LEUQ);
524
525    /*
526     * For 16 and 32-bit elements, the doubleword bswap also reversed
527     * the order of the elements.  Perform a larger order swap to put
528     * them back into place.  For the 128-bit "element", finish the
529     * bswap by swapping the doublewords.
530     */
531    switch (es) {
532    case ES_16:
533        tcg_gen_hswap_i64(t0, t0);
534        tcg_gen_hswap_i64(t1, t1);
535        break;
536    case ES_32:
537        tcg_gen_wswap_i64(t0, t0);
538        tcg_gen_wswap_i64(t1, t1);
539        break;
540    case ES_64:
541        break;
542    default:
543        g_assert_not_reached();
544    }
545
546write:
547    write_vec_element_i64(t0, get_field(s, v1), 0, ES_64);
548    write_vec_element_i64(t1, get_field(s, v1), 1, ES_64);
549    return DISAS_NEXT;
550}
551
552static DisasJumpType op_vle(DisasContext *s, DisasOps *o)
553{
554    const uint8_t es = s->insn->data;
555    const uint8_t enr = get_field(s, m3);
556    TCGv_i64 tmp;
557
558    if (!valid_vec_element(enr, es)) {
559        gen_program_exception(s, PGM_SPECIFICATION);
560        return DISAS_NORETURN;
561    }
562
563    tmp = tcg_temp_new_i64();
564    tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
565    write_vec_element_i64(tmp, get_field(s, v1), enr, es);
566    return DISAS_NEXT;
567}
568
569static DisasJumpType op_vlei(DisasContext *s, DisasOps *o)
570{
571    const uint8_t es = s->insn->data;
572    const uint8_t enr = get_field(s, m3);
573    TCGv_i64 tmp;
574
575    if (!valid_vec_element(enr, es)) {
576        gen_program_exception(s, PGM_SPECIFICATION);
577        return DISAS_NORETURN;
578    }
579
580    tmp = tcg_constant_i64((int16_t)get_field(s, i2));
581    write_vec_element_i64(tmp, get_field(s, v1), enr, es);
582    return DISAS_NEXT;
583}
584
585static DisasJumpType op_vler(DisasContext *s, DisasOps *o)
586{
587    const uint8_t es = get_field(s, m3);
588
589    if (es < ES_16 || es > ES_64) {
590        gen_program_exception(s, PGM_SPECIFICATION);
591        return DISAS_NORETURN;
592    }
593
594    TCGv_i64 t0 = tcg_temp_new_i64();
595    TCGv_i64 t1 = tcg_temp_new_i64();
596
597    /* Begin with the two doublewords swapped... */
598    tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEUQ);
599    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
600    tcg_gen_qemu_ld_i64(t0, o->addr1, get_mem_index(s), MO_TEUQ);
601
602    /* ... then swap smaller elements within the doublewords as required. */
603    switch (es) {
604    case MO_16:
605        tcg_gen_hswap_i64(t1, t1);
606        tcg_gen_hswap_i64(t0, t0);
607        break;
608    case MO_32:
609        tcg_gen_wswap_i64(t1, t1);
610        tcg_gen_wswap_i64(t0, t0);
611        break;
612    case MO_64:
613        break;
614    default:
615        g_assert_not_reached();
616    }
617
618    write_vec_element_i64(t0, get_field(s, v1), 0, ES_64);
619    write_vec_element_i64(t1, get_field(s, v1), 1, ES_64);
620    return DISAS_NEXT;
621}
622
623static DisasJumpType op_vlgv(DisasContext *s, DisasOps *o)
624{
625    const uint8_t es = get_field(s, m4);
626    TCGv_ptr ptr;
627
628    if (es > ES_64) {
629        gen_program_exception(s, PGM_SPECIFICATION);
630        return DISAS_NORETURN;
631    }
632
633    /* fast path if we don't need the register content */
634    if (!get_field(s, b2)) {
635        uint8_t enr = get_field(s, d2) & (NUM_VEC_ELEMENTS(es) - 1);
636
637        read_vec_element_i64(o->out, get_field(s, v3), enr, es);
638        return DISAS_NEXT;
639    }
640
641    ptr = tcg_temp_new_ptr();
642    get_vec_element_ptr_i64(ptr, get_field(s, v3), o->addr1, es);
643    switch (es) {
644    case ES_8:
645        tcg_gen_ld8u_i64(o->out, ptr, 0);
646        break;
647    case ES_16:
648        tcg_gen_ld16u_i64(o->out, ptr, 0);
649        break;
650    case ES_32:
651        tcg_gen_ld32u_i64(o->out, ptr, 0);
652        break;
653    case ES_64:
654        tcg_gen_ld_i64(o->out, ptr, 0);
655        break;
656    default:
657        g_assert_not_reached();
658    }
659    return DISAS_NEXT;
660}
661
662static DisasJumpType op_vllez(DisasContext *s, DisasOps *o)
663{
664    uint8_t es = get_field(s, m3);
665    uint8_t enr;
666    TCGv_i64 t;
667
668    switch (es) {
669    /* rightmost sub-element of leftmost doubleword */
670    case ES_8:
671        enr = 7;
672        break;
673    case ES_16:
674        enr = 3;
675        break;
676    case ES_32:
677        enr = 1;
678        break;
679    case ES_64:
680        enr = 0;
681        break;
682    /* leftmost sub-element of leftmost doubleword */
683    case 6:
684        if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
685            es = ES_32;
686            enr = 0;
687            break;
688        }
689        /* fallthrough */
690    default:
691        gen_program_exception(s, PGM_SPECIFICATION);
692        return DISAS_NORETURN;
693    }
694
695    t = tcg_temp_new_i64();
696    tcg_gen_qemu_ld_i64(t, o->addr1, get_mem_index(s), MO_TE | es);
697    gen_gvec_dup_imm(es, get_field(s, v1), 0);
698    write_vec_element_i64(t, get_field(s, v1), enr, es);
699    return DISAS_NEXT;
700}
701
702static DisasJumpType op_vlm(DisasContext *s, DisasOps *o)
703{
704    const uint8_t v3 = get_field(s, v3);
705    uint8_t v1 = get_field(s, v1);
706    TCGv_i64 t0, t1;
707
708    if (v3 < v1 || (v3 - v1 + 1) > 16) {
709        gen_program_exception(s, PGM_SPECIFICATION);
710        return DISAS_NORETURN;
711    }
712
713    /*
714     * Check for possible access exceptions by trying to load the last
715     * element. The first element will be checked first next.
716     */
717    t0 = tcg_temp_new_i64();
718    t1 = tcg_temp_new_i64();
719    gen_addi_and_wrap_i64(s, t0, o->addr1, (v3 - v1) * 16 + 8);
720    tcg_gen_qemu_ld_i64(t0, t0, get_mem_index(s), MO_TEUQ);
721
722    for (;; v1++) {
723        tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEUQ);
724        write_vec_element_i64(t1, v1, 0, ES_64);
725        if (v1 == v3) {
726            break;
727        }
728        gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
729        tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEUQ);
730        write_vec_element_i64(t1, v1, 1, ES_64);
731        gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
732    }
733
734    /* Store the last element, loaded first */
735    write_vec_element_i64(t0, v1, 1, ES_64);
736    return DISAS_NEXT;
737}
738
739static DisasJumpType op_vlbb(DisasContext *s, DisasOps *o)
740{
741    const int64_t block_size = (1ull << (get_field(s, m3) + 6));
742    const int v1_offs = vec_full_reg_offset(get_field(s, v1));
743    TCGv_ptr a0;
744    TCGv_i64 bytes;
745
746    if (get_field(s, m3) > 6) {
747        gen_program_exception(s, PGM_SPECIFICATION);
748        return DISAS_NORETURN;
749    }
750
751    bytes = tcg_temp_new_i64();
752    a0 = tcg_temp_new_ptr();
753    /* calculate the number of bytes until the next block boundary */
754    tcg_gen_ori_i64(bytes, o->addr1, -block_size);
755    tcg_gen_neg_i64(bytes, bytes);
756
757    tcg_gen_addi_ptr(a0, tcg_env, v1_offs);
758    gen_helper_vll(tcg_env, a0, o->addr1, bytes);
759    return DISAS_NEXT;
760}
761
762static DisasJumpType op_vlvg(DisasContext *s, DisasOps *o)
763{
764    const uint8_t es = get_field(s, m4);
765    TCGv_ptr ptr;
766
767    if (es > ES_64) {
768        gen_program_exception(s, PGM_SPECIFICATION);
769        return DISAS_NORETURN;
770    }
771
772    /* fast path if we don't need the register content */
773    if (!get_field(s, b2)) {
774        uint8_t enr = get_field(s, d2) & (NUM_VEC_ELEMENTS(es) - 1);
775
776        write_vec_element_i64(o->in2, get_field(s, v1), enr, es);
777        return DISAS_NEXT;
778    }
779
780    ptr = tcg_temp_new_ptr();
781    get_vec_element_ptr_i64(ptr, get_field(s, v1), o->addr1, es);
782    switch (es) {
783    case ES_8:
784        tcg_gen_st8_i64(o->in2, ptr, 0);
785        break;
786    case ES_16:
787        tcg_gen_st16_i64(o->in2, ptr, 0);
788        break;
789    case ES_32:
790        tcg_gen_st32_i64(o->in2, ptr, 0);
791        break;
792    case ES_64:
793        tcg_gen_st_i64(o->in2, ptr, 0);
794        break;
795    default:
796        g_assert_not_reached();
797    }
798    return DISAS_NEXT;
799}
800
801static DisasJumpType op_vlvgp(DisasContext *s, DisasOps *o)
802{
803    write_vec_element_i64(o->in1, get_field(s, v1), 0, ES_64);
804    write_vec_element_i64(o->in2, get_field(s, v1), 1, ES_64);
805    return DISAS_NEXT;
806}
807
808static DisasJumpType op_vll(DisasContext *s, DisasOps *o)
809{
810    const int v1_offs = vec_full_reg_offset(get_field(s, v1));
811    TCGv_ptr a0 = tcg_temp_new_ptr();
812
813    /* convert highest index into an actual length */
814    tcg_gen_addi_i64(o->in2, o->in2, 1);
815    tcg_gen_addi_ptr(a0, tcg_env, v1_offs);
816    gen_helper_vll(tcg_env, a0, o->addr1, o->in2);
817    return DISAS_NEXT;
818}
819
820static DisasJumpType op_vmr(DisasContext *s, DisasOps *o)
821{
822    const uint8_t v1 = get_field(s, v1);
823    const uint8_t v2 = get_field(s, v2);
824    const uint8_t v3 = get_field(s, v3);
825    const uint8_t es = get_field(s, m4);
826    int dst_idx, src_idx;
827    TCGv_i64 tmp;
828
829    if (es > ES_64) {
830        gen_program_exception(s, PGM_SPECIFICATION);
831        return DISAS_NORETURN;
832    }
833
834    tmp = tcg_temp_new_i64();
835    if (s->fields.op2 == 0x61) {
836        /* iterate backwards to avoid overwriting data we might need later */
837        for (dst_idx = NUM_VEC_ELEMENTS(es) - 1; dst_idx >= 0; dst_idx--) {
838            src_idx = dst_idx / 2;
839            if (dst_idx % 2 == 0) {
840                read_vec_element_i64(tmp, v2, src_idx, es);
841            } else {
842                read_vec_element_i64(tmp, v3, src_idx, es);
843            }
844            write_vec_element_i64(tmp, v1, dst_idx, es);
845        }
846    } else {
847        /* iterate forward to avoid overwriting data we might need later */
848        for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(es); dst_idx++) {
849            src_idx = (dst_idx + NUM_VEC_ELEMENTS(es)) / 2;
850            if (dst_idx % 2 == 0) {
851                read_vec_element_i64(tmp, v2, src_idx, es);
852            } else {
853                read_vec_element_i64(tmp, v3, src_idx, es);
854            }
855            write_vec_element_i64(tmp, v1, dst_idx, es);
856        }
857    }
858    return DISAS_NEXT;
859}
860
861static DisasJumpType op_vpk(DisasContext *s, DisasOps *o)
862{
863    const uint8_t v1 = get_field(s, v1);
864    const uint8_t v2 = get_field(s, v2);
865    const uint8_t v3 = get_field(s, v3);
866    const uint8_t es = get_field(s, m4);
867    static gen_helper_gvec_3 * const vpk[3] = {
868        gen_helper_gvec_vpk16,
869        gen_helper_gvec_vpk32,
870        gen_helper_gvec_vpk64,
871    };
872     static gen_helper_gvec_3 * const vpks[3] = {
873        gen_helper_gvec_vpks16,
874        gen_helper_gvec_vpks32,
875        gen_helper_gvec_vpks64,
876    };
877    static gen_helper_gvec_3_ptr * const vpks_cc[3] = {
878        gen_helper_gvec_vpks_cc16,
879        gen_helper_gvec_vpks_cc32,
880        gen_helper_gvec_vpks_cc64,
881    };
882    static gen_helper_gvec_3 * const vpkls[3] = {
883        gen_helper_gvec_vpkls16,
884        gen_helper_gvec_vpkls32,
885        gen_helper_gvec_vpkls64,
886    };
887    static gen_helper_gvec_3_ptr * const vpkls_cc[3] = {
888        gen_helper_gvec_vpkls_cc16,
889        gen_helper_gvec_vpkls_cc32,
890        gen_helper_gvec_vpkls_cc64,
891    };
892
893    if (es == ES_8 || es > ES_64) {
894        gen_program_exception(s, PGM_SPECIFICATION);
895        return DISAS_NORETURN;
896    }
897
898    switch (s->fields.op2) {
899    case 0x97:
900        if (get_field(s, m5) & 0x1) {
901            gen_gvec_3_ptr(v1, v2, v3, tcg_env, 0, vpks_cc[es - 1]);
902            set_cc_static(s);
903        } else {
904            gen_gvec_3_ool(v1, v2, v3, 0, vpks[es - 1]);
905        }
906        break;
907    case 0x95:
908        if (get_field(s, m5) & 0x1) {
909            gen_gvec_3_ptr(v1, v2, v3, tcg_env, 0, vpkls_cc[es - 1]);
910            set_cc_static(s);
911        } else {
912            gen_gvec_3_ool(v1, v2, v3, 0, vpkls[es - 1]);
913        }
914        break;
915    case 0x94:
916        /* If sources and destination don't overlap -> fast path */
917        if (v1 != v2 && v1 != v3) {
918            const uint8_t src_es = get_field(s, m4);
919            const uint8_t dst_es = src_es - 1;
920            TCGv_i64 tmp = tcg_temp_new_i64();
921            int dst_idx, src_idx;
922
923            for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(dst_es); dst_idx++) {
924                src_idx = dst_idx;
925                if (src_idx < NUM_VEC_ELEMENTS(src_es)) {
926                    read_vec_element_i64(tmp, v2, src_idx, src_es);
927                } else {
928                    src_idx -= NUM_VEC_ELEMENTS(src_es);
929                    read_vec_element_i64(tmp, v3, src_idx, src_es);
930                }
931                write_vec_element_i64(tmp, v1, dst_idx, dst_es);
932            }
933        } else {
934            gen_gvec_3_ool(v1, v2, v3, 0, vpk[es - 1]);
935        }
936        break;
937    default:
938        g_assert_not_reached();
939    }
940    return DISAS_NEXT;
941}
942
943static DisasJumpType op_vperm(DisasContext *s, DisasOps *o)
944{
945    gen_gvec_4_ool(get_field(s, v1), get_field(s, v2),
946                   get_field(s, v3), get_field(s, v4),
947                   0, gen_helper_gvec_vperm);
948    return DISAS_NEXT;
949}
950
951static DisasJumpType op_vpdi(DisasContext *s, DisasOps *o)
952{
953    const uint8_t i2 = extract32(get_field(s, m4), 2, 1);
954    const uint8_t i3 = extract32(get_field(s, m4), 0, 1);
955    TCGv_i64 t0 = tcg_temp_new_i64();
956    TCGv_i64 t1 = tcg_temp_new_i64();
957
958    read_vec_element_i64(t0, get_field(s, v2), i2, ES_64);
959    read_vec_element_i64(t1, get_field(s, v3), i3, ES_64);
960    write_vec_element_i64(t0, get_field(s, v1), 0, ES_64);
961    write_vec_element_i64(t1, get_field(s, v1), 1, ES_64);
962    return DISAS_NEXT;
963}
964
965static DisasJumpType op_vrep(DisasContext *s, DisasOps *o)
966{
967    const uint16_t enr = get_field(s, i2);
968    const uint8_t es = get_field(s, m4);
969
970    if (es > ES_64 || !valid_vec_element(enr, es)) {
971        gen_program_exception(s, PGM_SPECIFICATION);
972        return DISAS_NORETURN;
973    }
974
975    tcg_gen_gvec_dup_mem(es, vec_full_reg_offset(get_field(s, v1)),
976                         vec_reg_offset(get_field(s, v3), enr, es),
977                         16, 16);
978    return DISAS_NEXT;
979}
980
981static DisasJumpType op_vrepi(DisasContext *s, DisasOps *o)
982{
983    const int64_t data = (int16_t)get_field(s, i2);
984    const uint8_t es = get_field(s, m3);
985
986    if (es > ES_64) {
987        gen_program_exception(s, PGM_SPECIFICATION);
988        return DISAS_NORETURN;
989    }
990
991    gen_gvec_dup_imm(es, get_field(s, v1), data);
992    return DISAS_NEXT;
993}
994
995static DisasJumpType op_vsce(DisasContext *s, DisasOps *o)
996{
997    const uint8_t es = s->insn->data;
998    const uint8_t enr = get_field(s, m3);
999    TCGv_i64 tmp;
1000
1001    if (!valid_vec_element(enr, es)) {
1002        gen_program_exception(s, PGM_SPECIFICATION);
1003        return DISAS_NORETURN;
1004    }
1005
1006    tmp = tcg_temp_new_i64();
1007    read_vec_element_i64(tmp, get_field(s, v2), enr, es);
1008    tcg_gen_add_i64(o->addr1, o->addr1, tmp);
1009    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 0);
1010
1011    read_vec_element_i64(tmp, get_field(s, v1), enr, es);
1012    tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
1013    return DISAS_NEXT;
1014}
1015
1016static DisasJumpType op_vsel(DisasContext *s, DisasOps *o)
1017{
1018    gen_gvec_fn_4(bitsel, ES_8, get_field(s, v1),
1019                  get_field(s, v4), get_field(s, v2),
1020                  get_field(s, v3));
1021    return DISAS_NEXT;
1022}
1023
1024static DisasJumpType op_vseg(DisasContext *s, DisasOps *o)
1025{
1026    const uint8_t es = get_field(s, m3);
1027    int idx1, idx2;
1028    TCGv_i64 tmp;
1029
1030    switch (es) {
1031    case ES_8:
1032        idx1 = 7;
1033        idx2 = 15;
1034        break;
1035    case ES_16:
1036        idx1 = 3;
1037        idx2 = 7;
1038        break;
1039    case ES_32:
1040        idx1 = 1;
1041        idx2 = 3;
1042        break;
1043    default:
1044        gen_program_exception(s, PGM_SPECIFICATION);
1045        return DISAS_NORETURN;
1046    }
1047
1048    tmp = tcg_temp_new_i64();
1049    read_vec_element_i64(tmp, get_field(s, v2), idx1, es | MO_SIGN);
1050    write_vec_element_i64(tmp, get_field(s, v1), 0, ES_64);
1051    read_vec_element_i64(tmp, get_field(s, v2), idx2, es | MO_SIGN);
1052    write_vec_element_i64(tmp, get_field(s, v1), 1, ES_64);
1053    return DISAS_NEXT;
1054}
1055
1056static DisasJumpType op_vst(DisasContext *s, DisasOps *o)
1057{
1058    TCGv_i64 tmp;
1059
1060    /* Probe write access before actually modifying memory */
1061    gen_helper_probe_write_access(tcg_env, o->addr1,
1062                                  tcg_constant_i64(16));
1063
1064    tmp = tcg_temp_new_i64();
1065    read_vec_element_i64(tmp,  get_field(s, v1), 0, ES_64);
1066    tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEUQ);
1067    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
1068    read_vec_element_i64(tmp,  get_field(s, v1), 1, ES_64);
1069    tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEUQ);
1070    return DISAS_NEXT;
1071}
1072
1073static DisasJumpType op_vstebr(DisasContext *s, DisasOps *o)
1074{
1075    const uint8_t es = s->insn->data;
1076    const uint8_t enr = get_field(s, m3);
1077    TCGv_i64 tmp;
1078
1079    if (!valid_vec_element(enr, es)) {
1080        gen_program_exception(s, PGM_SPECIFICATION);
1081        return DISAS_NORETURN;
1082    }
1083
1084    tmp = tcg_temp_new_i64();
1085    read_vec_element_i64(tmp, get_field(s, v1), enr, es);
1086    tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_LE | es);
1087    return DISAS_NEXT;
1088}
1089
1090static DisasJumpType op_vstbr(DisasContext *s, DisasOps *o)
1091{
1092    const uint8_t es = get_field(s, m3);
1093    TCGv_i64 t0, t1;
1094
1095    if (es < ES_16 || es > ES_128) {
1096        gen_program_exception(s, PGM_SPECIFICATION);
1097        return DISAS_NORETURN;
1098    }
1099
1100    /* Probe write access before actually modifying memory */
1101    gen_helper_probe_write_access(tcg_env, o->addr1, tcg_constant_i64(16));
1102
1103    t0 = tcg_temp_new_i64();
1104    t1 = tcg_temp_new_i64();
1105
1106
1107    if (es == ES_128) {
1108        read_vec_element_i64(t1, get_field(s, v1), 0, ES_64);
1109        read_vec_element_i64(t0, get_field(s, v1), 1, ES_64);
1110        goto write;
1111    }
1112
1113    read_vec_element_i64(t0, get_field(s, v1), 0, ES_64);
1114    read_vec_element_i64(t1, get_field(s, v1), 1, ES_64);
1115
1116    /*
1117     * For 16 and 32-bit elements, the doubleword bswap below will
1118     * reverse the order of the elements.  Perform a larger order
1119     * swap to put them back into place.  For the 128-bit "element",
1120     * finish the bswap by swapping the doublewords.
1121     */
1122    switch (es) {
1123    case MO_16:
1124        tcg_gen_hswap_i64(t0, t0);
1125        tcg_gen_hswap_i64(t1, t1);
1126        break;
1127    case MO_32:
1128        tcg_gen_wswap_i64(t0, t0);
1129        tcg_gen_wswap_i64(t1, t1);
1130        break;
1131    case MO_64:
1132        break;
1133    default:
1134        g_assert_not_reached();
1135    }
1136
1137write:
1138    tcg_gen_qemu_st_i64(t0, o->addr1, get_mem_index(s), MO_LEUQ);
1139    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
1140    tcg_gen_qemu_st_i64(t1, o->addr1, get_mem_index(s), MO_LEUQ);
1141    return DISAS_NEXT;
1142}
1143
1144static DisasJumpType op_vste(DisasContext *s, DisasOps *o)
1145{
1146    const uint8_t es = s->insn->data;
1147    const uint8_t enr = get_field(s, m3);
1148    TCGv_i64 tmp;
1149
1150    if (!valid_vec_element(enr, es)) {
1151        gen_program_exception(s, PGM_SPECIFICATION);
1152        return DISAS_NORETURN;
1153    }
1154
1155    tmp = tcg_temp_new_i64();
1156    read_vec_element_i64(tmp, get_field(s, v1), enr, es);
1157    tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
1158    return DISAS_NEXT;
1159}
1160
1161static DisasJumpType op_vster(DisasContext *s, DisasOps *o)
1162{
1163    const uint8_t es = get_field(s, m3);
1164    TCGv_i64 t0, t1;
1165
1166    if (es < ES_16 || es > ES_64) {
1167        gen_program_exception(s, PGM_SPECIFICATION);
1168        return DISAS_NORETURN;
1169    }
1170
1171    /* Probe write access before actually modifying memory */
1172    gen_helper_probe_write_access(tcg_env, o->addr1, tcg_constant_i64(16));
1173
1174    /* Begin with the two doublewords swapped... */
1175    t0 = tcg_temp_new_i64();
1176    t1 = tcg_temp_new_i64();
1177    read_vec_element_i64(t1,  get_field(s, v1), 0, ES_64);
1178    read_vec_element_i64(t0,  get_field(s, v1), 1, ES_64);
1179
1180    /* ... then swap smaller elements within the doublewords as required. */
1181    switch (es) {
1182    case MO_16:
1183        tcg_gen_hswap_i64(t1, t1);
1184        tcg_gen_hswap_i64(t0, t0);
1185        break;
1186    case MO_32:
1187        tcg_gen_wswap_i64(t1, t1);
1188        tcg_gen_wswap_i64(t0, t0);
1189        break;
1190    case MO_64:
1191        break;
1192    default:
1193        g_assert_not_reached();
1194    }
1195
1196    tcg_gen_qemu_st_i64(t0, o->addr1, get_mem_index(s), MO_TEUQ);
1197    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
1198    tcg_gen_qemu_st_i64(t1, o->addr1, get_mem_index(s), MO_TEUQ);
1199    return DISAS_NEXT;
1200}
1201
1202static DisasJumpType op_vstm(DisasContext *s, DisasOps *o)
1203{
1204    const uint8_t v3 = get_field(s, v3);
1205    uint8_t v1 = get_field(s, v1);
1206    TCGv_i64 tmp;
1207
1208    while (v3 < v1 || (v3 - v1 + 1) > 16) {
1209        gen_program_exception(s, PGM_SPECIFICATION);
1210        return DISAS_NORETURN;
1211    }
1212
1213    /* Probe write access before actually modifying memory */
1214    gen_helper_probe_write_access(tcg_env, o->addr1,
1215                                  tcg_constant_i64((v3 - v1 + 1) * 16));
1216
1217    tmp = tcg_temp_new_i64();
1218    for (;; v1++) {
1219        read_vec_element_i64(tmp, v1, 0, ES_64);
1220        tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEUQ);
1221        gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
1222        read_vec_element_i64(tmp, v1, 1, ES_64);
1223        tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEUQ);
1224        if (v1 == v3) {
1225            break;
1226        }
1227        gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
1228    }
1229    return DISAS_NEXT;
1230}
1231
1232static DisasJumpType op_vstl(DisasContext *s, DisasOps *o)
1233{
1234    const int v1_offs = vec_full_reg_offset(get_field(s, v1));
1235    TCGv_ptr a0 = tcg_temp_new_ptr();
1236
1237    /* convert highest index into an actual length */
1238    tcg_gen_addi_i64(o->in2, o->in2, 1);
1239    tcg_gen_addi_ptr(a0, tcg_env, v1_offs);
1240    gen_helper_vstl(tcg_env, a0, o->addr1, o->in2);
1241    return DISAS_NEXT;
1242}
1243
1244static DisasJumpType op_vup(DisasContext *s, DisasOps *o)
1245{
1246    const bool logical = s->fields.op2 == 0xd4 || s->fields.op2 == 0xd5;
1247    const uint8_t v1 = get_field(s, v1);
1248    const uint8_t v2 = get_field(s, v2);
1249    const uint8_t src_es = get_field(s, m3);
1250    const uint8_t dst_es = src_es + 1;
1251    int dst_idx, src_idx;
1252    TCGv_i64 tmp;
1253
1254    if (src_es > ES_32) {
1255        gen_program_exception(s, PGM_SPECIFICATION);
1256        return DISAS_NORETURN;
1257    }
1258
1259    tmp = tcg_temp_new_i64();
1260    if (s->fields.op2 == 0xd7 || s->fields.op2 == 0xd5) {
1261        /* iterate backwards to avoid overwriting data we might need later */
1262        for (dst_idx = NUM_VEC_ELEMENTS(dst_es) - 1; dst_idx >= 0; dst_idx--) {
1263            src_idx = dst_idx;
1264            read_vec_element_i64(tmp, v2, src_idx,
1265                                 src_es | (logical ? 0 : MO_SIGN));
1266            write_vec_element_i64(tmp, v1, dst_idx, dst_es);
1267        }
1268
1269    } else {
1270        /* iterate forward to avoid overwriting data we might need later */
1271        for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(dst_es); dst_idx++) {
1272            src_idx = dst_idx + NUM_VEC_ELEMENTS(src_es) / 2;
1273            read_vec_element_i64(tmp, v2, src_idx,
1274                                 src_es | (logical ? 0 : MO_SIGN));
1275            write_vec_element_i64(tmp, v1, dst_idx, dst_es);
1276        }
1277    }
1278    return DISAS_NEXT;
1279}
1280
1281static DisasJumpType op_va(DisasContext *s, DisasOps *o)
1282{
1283    const uint8_t es = get_field(s, m4);
1284
1285    if (es > ES_128) {
1286        gen_program_exception(s, PGM_SPECIFICATION);
1287        return DISAS_NORETURN;
1288    } else if (es == ES_128) {
1289        gen_gvec128_3_i64(tcg_gen_add2_i64, get_field(s, v1),
1290                          get_field(s, v2), get_field(s, v3));
1291        return DISAS_NEXT;
1292    }
1293    gen_gvec_fn_3(add, es, get_field(s, v1), get_field(s, v2),
1294                  get_field(s, v3));
1295    return DISAS_NEXT;
1296}
1297
1298static void gen_acc(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, uint8_t es)
1299{
1300    const uint8_t msb_bit_nr = NUM_VEC_ELEMENT_BITS(es) - 1;
1301    TCGv_i64 msb_mask = tcg_constant_i64(dup_const(es, 1ull << msb_bit_nr));
1302    TCGv_i64 t1 = tcg_temp_new_i64();
1303    TCGv_i64 t2 = tcg_temp_new_i64();
1304    TCGv_i64 t3 = tcg_temp_new_i64();
1305
1306    /* Calculate the carry into the MSB, ignoring the old MSBs */
1307    tcg_gen_andc_i64(t1, a, msb_mask);
1308    tcg_gen_andc_i64(t2, b, msb_mask);
1309    tcg_gen_add_i64(t1, t1, t2);
1310    /* Calculate the MSB without any carry into it */
1311    tcg_gen_xor_i64(t3, a, b);
1312    /* Calculate the carry out of the MSB in the MSB bit position */
1313    tcg_gen_and_i64(d, a, b);
1314    tcg_gen_and_i64(t1, t1, t3);
1315    tcg_gen_or_i64(d, d, t1);
1316    /* Isolate and shift the carry into position */
1317    tcg_gen_and_i64(d, d, msb_mask);
1318    tcg_gen_shri_i64(d, d, msb_bit_nr);
1319}
1320
1321static void gen_acc8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1322{
1323    gen_acc(d, a, b, ES_8);
1324}
1325
1326static void gen_acc16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1327{
1328    gen_acc(d, a, b, ES_16);
1329}
1330
1331static void gen_acc_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1332{
1333    TCGv_i32 t = tcg_temp_new_i32();
1334
1335    tcg_gen_add_i32(t, a, b);
1336    tcg_gen_setcond_i32(TCG_COND_LTU, d, t, b);
1337}
1338
1339static void gen_acc_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1340{
1341    TCGv_i64 t = tcg_temp_new_i64();
1342
1343    tcg_gen_add_i64(t, a, b);
1344    tcg_gen_setcond_i64(TCG_COND_LTU, d, t, b);
1345}
1346
1347static void gen_acc2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
1348                         TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
1349{
1350    TCGv_i64 th = tcg_temp_new_i64();
1351    TCGv_i64 tl = tcg_temp_new_i64();
1352    TCGv_i64 zero = tcg_constant_i64(0);
1353
1354    tcg_gen_add2_i64(tl, th, al, zero, bl, zero);
1355    tcg_gen_add2_i64(tl, th, th, zero, ah, zero);
1356    tcg_gen_add2_i64(tl, dl, tl, th, bh, zero);
1357    tcg_gen_mov_i64(dh, zero);
1358}
1359
1360static DisasJumpType op_vacc(DisasContext *s, DisasOps *o)
1361{
1362    const uint8_t es = get_field(s, m4);
1363    static const GVecGen3 g[4] = {
1364        { .fni8 = gen_acc8_i64, },
1365        { .fni8 = gen_acc16_i64, },
1366        { .fni4 = gen_acc_i32, },
1367        { .fni8 = gen_acc_i64, },
1368    };
1369
1370    if (es > ES_128) {
1371        gen_program_exception(s, PGM_SPECIFICATION);
1372        return DISAS_NORETURN;
1373    } else if (es == ES_128) {
1374        gen_gvec128_3_i64(gen_acc2_i64, get_field(s, v1),
1375                          get_field(s, v2), get_field(s, v3));
1376        return DISAS_NEXT;
1377    }
1378    gen_gvec_3(get_field(s, v1), get_field(s, v2),
1379               get_field(s, v3), &g[es]);
1380    return DISAS_NEXT;
1381}
1382
1383static void gen_ac2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
1384                        TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
1385{
1386    TCGv_i64 tl = tcg_temp_new_i64();
1387    TCGv_i64 zero = tcg_constant_i64(0);
1388
1389    /* extract the carry only */
1390    tcg_gen_extract_i64(tl, cl, 0, 1);
1391    tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
1392    tcg_gen_add2_i64(dl, dh, dl, dh, tl, zero);
1393}
1394
1395static DisasJumpType op_vac(DisasContext *s, DisasOps *o)
1396{
1397    if (get_field(s, m5) != ES_128) {
1398        gen_program_exception(s, PGM_SPECIFICATION);
1399        return DISAS_NORETURN;
1400    }
1401
1402    gen_gvec128_4_i64(gen_ac2_i64, get_field(s, v1),
1403                      get_field(s, v2), get_field(s, v3),
1404                      get_field(s, v4));
1405    return DISAS_NEXT;
1406}
1407
1408static void gen_accc2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
1409                          TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
1410{
1411    TCGv_i64 tl = tcg_temp_new_i64();
1412    TCGv_i64 th = tcg_temp_new_i64();
1413    TCGv_i64 zero = tcg_constant_i64(0);
1414
1415    tcg_gen_andi_i64(tl, cl, 1);
1416    tcg_gen_add2_i64(tl, th, tl, zero, al, zero);
1417    tcg_gen_add2_i64(tl, th, tl, th, bl, zero);
1418    tcg_gen_add2_i64(tl, th, th, zero, ah, zero);
1419    tcg_gen_add2_i64(tl, dl, tl, th, bh, zero);
1420    tcg_gen_mov_i64(dh, zero);
1421}
1422
1423static DisasJumpType op_vaccc(DisasContext *s, DisasOps *o)
1424{
1425    if (get_field(s, m5) != ES_128) {
1426        gen_program_exception(s, PGM_SPECIFICATION);
1427        return DISAS_NORETURN;
1428    }
1429
1430    gen_gvec128_4_i64(gen_accc2_i64, get_field(s, v1),
1431                      get_field(s, v2), get_field(s, v3),
1432                      get_field(s, v4));
1433    return DISAS_NEXT;
1434}
1435
1436static DisasJumpType op_vn(DisasContext *s, DisasOps *o)
1437{
1438    gen_gvec_fn_3(and, ES_8, get_field(s, v1), get_field(s, v2),
1439                  get_field(s, v3));
1440    return DISAS_NEXT;
1441}
1442
1443static DisasJumpType op_vnc(DisasContext *s, DisasOps *o)
1444{
1445    gen_gvec_fn_3(andc, ES_8, get_field(s, v1),
1446                  get_field(s, v2), get_field(s, v3));
1447    return DISAS_NEXT;
1448}
1449
1450static void gen_avg_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1451{
1452    TCGv_i64 t0 = tcg_temp_new_i64();
1453    TCGv_i64 t1 = tcg_temp_new_i64();
1454
1455    tcg_gen_ext_i32_i64(t0, a);
1456    tcg_gen_ext_i32_i64(t1, b);
1457    tcg_gen_add_i64(t0, t0, t1);
1458    tcg_gen_addi_i64(t0, t0, 1);
1459    tcg_gen_shri_i64(t0, t0, 1);
1460    tcg_gen_extrl_i64_i32(d, t0);
1461}
1462
1463static void gen_avg_i64(TCGv_i64 dl, TCGv_i64 al, TCGv_i64 bl)
1464{
1465    TCGv_i64 dh = tcg_temp_new_i64();
1466    TCGv_i64 ah = tcg_temp_new_i64();
1467    TCGv_i64 bh = tcg_temp_new_i64();
1468
1469    /* extending the sign by one bit is sufficient */
1470    tcg_gen_extract_i64(ah, al, 63, 1);
1471    tcg_gen_extract_i64(bh, bl, 63, 1);
1472    tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
1473    gen_addi2_i64(dl, dh, dl, dh, 1);
1474    tcg_gen_extract2_i64(dl, dl, dh, 1);
1475}
1476
1477static DisasJumpType op_vavg(DisasContext *s, DisasOps *o)
1478{
1479    const uint8_t es = get_field(s, m4);
1480    static const GVecGen3 g[4] = {
1481        { .fno = gen_helper_gvec_vavg8, },
1482        { .fno = gen_helper_gvec_vavg16, },
1483        { .fni4 = gen_avg_i32, },
1484        { .fni8 = gen_avg_i64, },
1485    };
1486
1487    if (es > ES_64) {
1488        gen_program_exception(s, PGM_SPECIFICATION);
1489        return DISAS_NORETURN;
1490    }
1491    gen_gvec_3(get_field(s, v1), get_field(s, v2),
1492               get_field(s, v3), &g[es]);
1493    return DISAS_NEXT;
1494}
1495
1496static void gen_avgl_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1497{
1498    TCGv_i64 t0 = tcg_temp_new_i64();
1499    TCGv_i64 t1 = tcg_temp_new_i64();
1500
1501    tcg_gen_extu_i32_i64(t0, a);
1502    tcg_gen_extu_i32_i64(t1, b);
1503    tcg_gen_add_i64(t0, t0, t1);
1504    tcg_gen_addi_i64(t0, t0, 1);
1505    tcg_gen_shri_i64(t0, t0, 1);
1506    tcg_gen_extrl_i64_i32(d, t0);
1507}
1508
1509static void gen_avgl_i64(TCGv_i64 dl, TCGv_i64 al, TCGv_i64 bl)
1510{
1511    TCGv_i64 dh = tcg_temp_new_i64();
1512    TCGv_i64 zero = tcg_constant_i64(0);
1513
1514    tcg_gen_add2_i64(dl, dh, al, zero, bl, zero);
1515    gen_addi2_i64(dl, dh, dl, dh, 1);
1516    tcg_gen_extract2_i64(dl, dl, dh, 1);
1517}
1518
1519static DisasJumpType op_vavgl(DisasContext *s, DisasOps *o)
1520{
1521    const uint8_t es = get_field(s, m4);
1522    static const GVecGen3 g[4] = {
1523        { .fno = gen_helper_gvec_vavgl8, },
1524        { .fno = gen_helper_gvec_vavgl16, },
1525        { .fni4 = gen_avgl_i32, },
1526        { .fni8 = gen_avgl_i64, },
1527    };
1528
1529    if (es > ES_64) {
1530        gen_program_exception(s, PGM_SPECIFICATION);
1531        return DISAS_NORETURN;
1532    }
1533    gen_gvec_3(get_field(s, v1), get_field(s, v2),
1534               get_field(s, v3), &g[es]);
1535    return DISAS_NEXT;
1536}
1537
1538static DisasJumpType op_vcksm(DisasContext *s, DisasOps *o)
1539{
1540    TCGv_i32 tmp = tcg_temp_new_i32();
1541    TCGv_i32 sum = tcg_temp_new_i32();
1542    int i;
1543
1544    read_vec_element_i32(sum, get_field(s, v3), 1, ES_32);
1545    for (i = 0; i < 4; i++) {
1546        read_vec_element_i32(tmp, get_field(s, v2), i, ES_32);
1547        tcg_gen_add2_i32(tmp, sum, sum, sum, tmp, tmp);
1548    }
1549    gen_gvec_dup_imm(ES_32, get_field(s, v1), 0);
1550    write_vec_element_i32(sum, get_field(s, v1), 1, ES_32);
1551    return DISAS_NEXT;
1552}
1553
1554static DisasJumpType op_vec(DisasContext *s, DisasOps *o)
1555{
1556    uint8_t es = get_field(s, m3);
1557    const uint8_t enr = NUM_VEC_ELEMENTS(es) / 2 - 1;
1558
1559    if (es > ES_64) {
1560        gen_program_exception(s, PGM_SPECIFICATION);
1561        return DISAS_NORETURN;
1562    }
1563    if (s->fields.op2 == 0xdb) {
1564        es |= MO_SIGN;
1565    }
1566
1567    o->in1 = tcg_temp_new_i64();
1568    o->in2 = tcg_temp_new_i64();
1569    read_vec_element_i64(o->in1, get_field(s, v1), enr, es);
1570    read_vec_element_i64(o->in2, get_field(s, v2), enr, es);
1571    return DISAS_NEXT;
1572}
1573
1574static DisasJumpType op_vc(DisasContext *s, DisasOps *o)
1575{
1576    const uint8_t es = get_field(s, m4);
1577    TCGCond cond = s->insn->data;
1578
1579    if (es > ES_64) {
1580        gen_program_exception(s, PGM_SPECIFICATION);
1581        return DISAS_NORETURN;
1582    }
1583
1584    tcg_gen_gvec_cmp(cond, es,
1585                     vec_full_reg_offset(get_field(s, v1)),
1586                     vec_full_reg_offset(get_field(s, v2)),
1587                     vec_full_reg_offset(get_field(s, v3)), 16, 16);
1588    if (get_field(s, m5) & 0x1) {
1589        TCGv_i64 low = tcg_temp_new_i64();
1590        TCGv_i64 high = tcg_temp_new_i64();
1591
1592        read_vec_element_i64(high, get_field(s, v1), 0, ES_64);
1593        read_vec_element_i64(low, get_field(s, v1), 1, ES_64);
1594        gen_op_update2_cc_i64(s, CC_OP_VC, low, high);
1595    }
1596    return DISAS_NEXT;
1597}
1598
1599static void gen_clz_i32(TCGv_i32 d, TCGv_i32 a)
1600{
1601    tcg_gen_clzi_i32(d, a, 32);
1602}
1603
1604static void gen_clz_i64(TCGv_i64 d, TCGv_i64 a)
1605{
1606    tcg_gen_clzi_i64(d, a, 64);
1607}
1608
1609static DisasJumpType op_vclz(DisasContext *s, DisasOps *o)
1610{
1611    const uint8_t es = get_field(s, m3);
1612    static const GVecGen2 g[4] = {
1613        { .fno = gen_helper_gvec_vclz8, },
1614        { .fno = gen_helper_gvec_vclz16, },
1615        { .fni4 = gen_clz_i32, },
1616        { .fni8 = gen_clz_i64, },
1617    };
1618
1619    if (es > ES_64) {
1620        gen_program_exception(s, PGM_SPECIFICATION);
1621        return DISAS_NORETURN;
1622    }
1623    gen_gvec_2(get_field(s, v1), get_field(s, v2), &g[es]);
1624    return DISAS_NEXT;
1625}
1626
1627static void gen_ctz_i32(TCGv_i32 d, TCGv_i32 a)
1628{
1629    tcg_gen_ctzi_i32(d, a, 32);
1630}
1631
1632static void gen_ctz_i64(TCGv_i64 d, TCGv_i64 a)
1633{
1634    tcg_gen_ctzi_i64(d, a, 64);
1635}
1636
1637static DisasJumpType op_vctz(DisasContext *s, DisasOps *o)
1638{
1639    const uint8_t es = get_field(s, m3);
1640    static const GVecGen2 g[4] = {
1641        { .fno = gen_helper_gvec_vctz8, },
1642        { .fno = gen_helper_gvec_vctz16, },
1643        { .fni4 = gen_ctz_i32, },
1644        { .fni8 = gen_ctz_i64, },
1645    };
1646
1647    if (es > ES_64) {
1648        gen_program_exception(s, PGM_SPECIFICATION);
1649        return DISAS_NORETURN;
1650    }
1651    gen_gvec_2(get_field(s, v1), get_field(s, v2), &g[es]);
1652    return DISAS_NEXT;
1653}
1654
1655static DisasJumpType op_vx(DisasContext *s, DisasOps *o)
1656{
1657    gen_gvec_fn_3(xor, ES_8, get_field(s, v1), get_field(s, v2),
1658                 get_field(s, v3));
1659    return DISAS_NEXT;
1660}
1661
1662static DisasJumpType op_vgfm(DisasContext *s, DisasOps *o)
1663{
1664    const uint8_t es = get_field(s, m4);
1665    static const GVecGen3 g[4] = {
1666        { .fno = gen_helper_gvec_vgfm8, },
1667        { .fno = gen_helper_gvec_vgfm16, },
1668        { .fno = gen_helper_gvec_vgfm32, },
1669        { .fno = gen_helper_gvec_vgfm64, },
1670    };
1671
1672    if (es > ES_64) {
1673        gen_program_exception(s, PGM_SPECIFICATION);
1674        return DISAS_NORETURN;
1675    }
1676    gen_gvec_3(get_field(s, v1), get_field(s, v2),
1677               get_field(s, v3), &g[es]);
1678    return DISAS_NEXT;
1679}
1680
1681static DisasJumpType op_vgfma(DisasContext *s, DisasOps *o)
1682{
1683    const uint8_t es = get_field(s, m5);
1684    static const GVecGen4 g[4] = {
1685        { .fno = gen_helper_gvec_vgfma8, },
1686        { .fno = gen_helper_gvec_vgfma16, },
1687        { .fno = gen_helper_gvec_vgfma32, },
1688        { .fno = gen_helper_gvec_vgfma64, },
1689    };
1690
1691    if (es > ES_64) {
1692        gen_program_exception(s, PGM_SPECIFICATION);
1693        return DISAS_NORETURN;
1694    }
1695    gen_gvec_4(get_field(s, v1), get_field(s, v2),
1696               get_field(s, v3), get_field(s, v4), &g[es]);
1697    return DISAS_NEXT;
1698}
1699
1700static DisasJumpType op_vlc(DisasContext *s, DisasOps *o)
1701{
1702    const uint8_t es = get_field(s, m3);
1703
1704    if (es > ES_64) {
1705        gen_program_exception(s, PGM_SPECIFICATION);
1706        return DISAS_NORETURN;
1707    }
1708
1709    gen_gvec_fn_2(neg, es, get_field(s, v1), get_field(s, v2));
1710    return DISAS_NEXT;
1711}
1712
1713static DisasJumpType op_vlp(DisasContext *s, DisasOps *o)
1714{
1715    const uint8_t es = get_field(s, m3);
1716
1717    if (es > ES_64) {
1718        gen_program_exception(s, PGM_SPECIFICATION);
1719        return DISAS_NORETURN;
1720    }
1721
1722    gen_gvec_fn_2(abs, es, get_field(s, v1), get_field(s, v2));
1723    return DISAS_NEXT;
1724}
1725
1726static DisasJumpType op_vmx(DisasContext *s, DisasOps *o)
1727{
1728    const uint8_t v1 = get_field(s, v1);
1729    const uint8_t v2 = get_field(s, v2);
1730    const uint8_t v3 = get_field(s, v3);
1731    const uint8_t es = get_field(s, m4);
1732
1733    if (es > ES_64) {
1734        gen_program_exception(s, PGM_SPECIFICATION);
1735        return DISAS_NORETURN;
1736    }
1737
1738    switch (s->fields.op2) {
1739    case 0xff:
1740        gen_gvec_fn_3(smax, es, v1, v2, v3);
1741        break;
1742    case 0xfd:
1743        gen_gvec_fn_3(umax, es, v1, v2, v3);
1744        break;
1745    case 0xfe:
1746        gen_gvec_fn_3(smin, es, v1, v2, v3);
1747        break;
1748    case 0xfc:
1749        gen_gvec_fn_3(umin, es, v1, v2, v3);
1750        break;
1751    default:
1752        g_assert_not_reached();
1753    }
1754    return DISAS_NEXT;
1755}
1756
1757static void gen_mal_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, TCGv_i32 c)
1758{
1759    TCGv_i32 t0 = tcg_temp_new_i32();
1760
1761    tcg_gen_mul_i32(t0, a, b);
1762    tcg_gen_add_i32(d, t0, c);
1763}
1764
1765static void gen_mah_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, TCGv_i32 c)
1766{
1767    TCGv_i64 t0 = tcg_temp_new_i64();
1768    TCGv_i64 t1 = tcg_temp_new_i64();
1769    TCGv_i64 t2 = tcg_temp_new_i64();
1770
1771    tcg_gen_ext_i32_i64(t0, a);
1772    tcg_gen_ext_i32_i64(t1, b);
1773    tcg_gen_ext_i32_i64(t2, c);
1774    tcg_gen_mul_i64(t0, t0, t1);
1775    tcg_gen_add_i64(t0, t0, t2);
1776    tcg_gen_extrh_i64_i32(d, t0);
1777}
1778
1779static void gen_malh_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, TCGv_i32 c)
1780{
1781    TCGv_i64 t0 = tcg_temp_new_i64();
1782    TCGv_i64 t1 = tcg_temp_new_i64();
1783    TCGv_i64 t2 = tcg_temp_new_i64();
1784
1785    tcg_gen_extu_i32_i64(t0, a);
1786    tcg_gen_extu_i32_i64(t1, b);
1787    tcg_gen_extu_i32_i64(t2, c);
1788    tcg_gen_mul_i64(t0, t0, t1);
1789    tcg_gen_add_i64(t0, t0, t2);
1790    tcg_gen_extrh_i64_i32(d, t0);
1791}
1792
1793static DisasJumpType op_vma(DisasContext *s, DisasOps *o)
1794{
1795    const uint8_t es = get_field(s, m5);
1796    static const GVecGen4 g_vmal[3] = {
1797        { .fno = gen_helper_gvec_vmal8, },
1798        { .fno = gen_helper_gvec_vmal16, },
1799        { .fni4 = gen_mal_i32, },
1800    };
1801    static const GVecGen4 g_vmah[3] = {
1802        { .fno = gen_helper_gvec_vmah8, },
1803        { .fno = gen_helper_gvec_vmah16, },
1804        { .fni4 = gen_mah_i32, },
1805    };
1806    static const GVecGen4 g_vmalh[3] = {
1807        { .fno = gen_helper_gvec_vmalh8, },
1808        { .fno = gen_helper_gvec_vmalh16, },
1809        { .fni4 = gen_malh_i32, },
1810    };
1811    static const GVecGen4 g_vmae[3] = {
1812        { .fno = gen_helper_gvec_vmae8, },
1813        { .fno = gen_helper_gvec_vmae16, },
1814        { .fno = gen_helper_gvec_vmae32, },
1815    };
1816    static const GVecGen4 g_vmale[3] = {
1817        { .fno = gen_helper_gvec_vmale8, },
1818        { .fno = gen_helper_gvec_vmale16, },
1819        { .fno = gen_helper_gvec_vmale32, },
1820    };
1821    static const GVecGen4 g_vmao[3] = {
1822        { .fno = gen_helper_gvec_vmao8, },
1823        { .fno = gen_helper_gvec_vmao16, },
1824        { .fno = gen_helper_gvec_vmao32, },
1825    };
1826    static const GVecGen4 g_vmalo[3] = {
1827        { .fno = gen_helper_gvec_vmalo8, },
1828        { .fno = gen_helper_gvec_vmalo16, },
1829        { .fno = gen_helper_gvec_vmalo32, },
1830    };
1831    const GVecGen4 *fn;
1832
1833    if (es > ES_32) {
1834        gen_program_exception(s, PGM_SPECIFICATION);
1835        return DISAS_NORETURN;
1836    }
1837
1838    switch (s->fields.op2) {
1839    case 0xaa:
1840        fn = &g_vmal[es];
1841        break;
1842    case 0xab:
1843        fn = &g_vmah[es];
1844        break;
1845    case 0xa9:
1846        fn = &g_vmalh[es];
1847        break;
1848    case 0xae:
1849        fn = &g_vmae[es];
1850        break;
1851    case 0xac:
1852        fn = &g_vmale[es];
1853        break;
1854    case 0xaf:
1855        fn = &g_vmao[es];
1856        break;
1857    case 0xad:
1858        fn = &g_vmalo[es];
1859        break;
1860    default:
1861        g_assert_not_reached();
1862    }
1863
1864    gen_gvec_4(get_field(s, v1), get_field(s, v2),
1865               get_field(s, v3), get_field(s, v4), fn);
1866    return DISAS_NEXT;
1867}
1868
1869static void gen_mh_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1870{
1871    TCGv_i32 t = tcg_temp_new_i32();
1872
1873    tcg_gen_muls2_i32(t, d, a, b);
1874}
1875
1876static void gen_mlh_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1877{
1878    TCGv_i32 t = tcg_temp_new_i32();
1879
1880    tcg_gen_mulu2_i32(t, d, a, b);
1881}
1882
1883static DisasJumpType op_vm(DisasContext *s, DisasOps *o)
1884{
1885    const uint8_t es = get_field(s, m4);
1886    static const GVecGen3 g_vmh[3] = {
1887        { .fno = gen_helper_gvec_vmh8, },
1888        { .fno = gen_helper_gvec_vmh16, },
1889        { .fni4 = gen_mh_i32, },
1890    };
1891    static const GVecGen3 g_vmlh[3] = {
1892        { .fno = gen_helper_gvec_vmlh8, },
1893        { .fno = gen_helper_gvec_vmlh16, },
1894        { .fni4 = gen_mlh_i32, },
1895    };
1896    static const GVecGen3 g_vme[3] = {
1897        { .fno = gen_helper_gvec_vme8, },
1898        { .fno = gen_helper_gvec_vme16, },
1899        { .fno = gen_helper_gvec_vme32, },
1900    };
1901    static const GVecGen3 g_vmle[3] = {
1902        { .fno = gen_helper_gvec_vmle8, },
1903        { .fno = gen_helper_gvec_vmle16, },
1904        { .fno = gen_helper_gvec_vmle32, },
1905    };
1906    static const GVecGen3 g_vmo[3] = {
1907        { .fno = gen_helper_gvec_vmo8, },
1908        { .fno = gen_helper_gvec_vmo16, },
1909        { .fno = gen_helper_gvec_vmo32, },
1910    };
1911    static const GVecGen3 g_vmlo[3] = {
1912        { .fno = gen_helper_gvec_vmlo8, },
1913        { .fno = gen_helper_gvec_vmlo16, },
1914        { .fno = gen_helper_gvec_vmlo32, },
1915    };
1916    const GVecGen3 *fn;
1917
1918    if (es > ES_32) {
1919        gen_program_exception(s, PGM_SPECIFICATION);
1920        return DISAS_NORETURN;
1921    }
1922
1923    switch (s->fields.op2) {
1924    case 0xa2:
1925        gen_gvec_fn_3(mul, es, get_field(s, v1),
1926                      get_field(s, v2), get_field(s, v3));
1927        return DISAS_NEXT;
1928    case 0xa3:
1929        fn = &g_vmh[es];
1930        break;
1931    case 0xa1:
1932        fn = &g_vmlh[es];
1933        break;
1934    case 0xa6:
1935        fn = &g_vme[es];
1936        break;
1937    case 0xa4:
1938        fn = &g_vmle[es];
1939        break;
1940    case 0xa7:
1941        fn = &g_vmo[es];
1942        break;
1943    case 0xa5:
1944        fn = &g_vmlo[es];
1945        break;
1946    default:
1947        g_assert_not_reached();
1948    }
1949
1950    gen_gvec_3(get_field(s, v1), get_field(s, v2),
1951               get_field(s, v3), fn);
1952    return DISAS_NEXT;
1953}
1954
1955static DisasJumpType op_vmsl(DisasContext *s, DisasOps *o)
1956{
1957    TCGv_i64 l1, h1, l2, h2;
1958
1959    if (get_field(s, m5) != ES_64) {
1960        gen_program_exception(s, PGM_SPECIFICATION);
1961        return DISAS_NORETURN;
1962    }
1963
1964    l1 = tcg_temp_new_i64();
1965    h1 = tcg_temp_new_i64();
1966    l2 = tcg_temp_new_i64();
1967    h2 = tcg_temp_new_i64();
1968
1969    /* Multiply both even elements from v2 and v3 */
1970    read_vec_element_i64(l1, get_field(s, v2), 0, ES_64);
1971    read_vec_element_i64(h1, get_field(s, v3), 0, ES_64);
1972    tcg_gen_mulu2_i64(l1, h1, l1, h1);
1973    /* Shift result left by one (x2) if requested */
1974    if (extract32(get_field(s, m6), 3, 1)) {
1975        tcg_gen_add2_i64(l1, h1, l1, h1, l1, h1);
1976    }
1977
1978    /* Multiply both odd elements from v2 and v3 */
1979    read_vec_element_i64(l2, get_field(s, v2), 1, ES_64);
1980    read_vec_element_i64(h2, get_field(s, v3), 1, ES_64);
1981    tcg_gen_mulu2_i64(l2, h2, l2, h2);
1982    /* Shift result left by one (x2) if requested */
1983    if (extract32(get_field(s, m6), 2, 1)) {
1984        tcg_gen_add2_i64(l2, h2, l2, h2, l2, h2);
1985    }
1986
1987    /* Add both intermediate results */
1988    tcg_gen_add2_i64(l1, h1, l1, h1, l2, h2);
1989    /* Add whole v4 */
1990    read_vec_element_i64(h2, get_field(s, v4), 0, ES_64);
1991    read_vec_element_i64(l2, get_field(s, v4), 1, ES_64);
1992    tcg_gen_add2_i64(l1, h1, l1, h1, l2, h2);
1993
1994    /* Store final result into v1. */
1995    write_vec_element_i64(h1, get_field(s, v1), 0, ES_64);
1996    write_vec_element_i64(l1, get_field(s, v1), 1, ES_64);
1997    return DISAS_NEXT;
1998}
1999
2000static DisasJumpType op_vnn(DisasContext *s, DisasOps *o)
2001{
2002    gen_gvec_fn_3(nand, ES_8, get_field(s, v1),
2003                  get_field(s, v2), get_field(s, v3));
2004    return DISAS_NEXT;
2005}
2006
2007static DisasJumpType op_vno(DisasContext *s, DisasOps *o)
2008{
2009    gen_gvec_fn_3(nor, ES_8, get_field(s, v1), get_field(s, v2),
2010                  get_field(s, v3));
2011    return DISAS_NEXT;
2012}
2013
2014static DisasJumpType op_vnx(DisasContext *s, DisasOps *o)
2015{
2016    gen_gvec_fn_3(eqv, ES_8, get_field(s, v1), get_field(s, v2),
2017                  get_field(s, v3));
2018    return DISAS_NEXT;
2019}
2020
2021static DisasJumpType op_vo(DisasContext *s, DisasOps *o)
2022{
2023    gen_gvec_fn_3(or, ES_8, get_field(s, v1), get_field(s, v2),
2024                  get_field(s, v3));
2025    return DISAS_NEXT;
2026}
2027
2028static DisasJumpType op_voc(DisasContext *s, DisasOps *o)
2029{
2030    gen_gvec_fn_3(orc, ES_8, get_field(s, v1), get_field(s, v2),
2031                  get_field(s, v3));
2032    return DISAS_NEXT;
2033}
2034
2035static DisasJumpType op_vpopct(DisasContext *s, DisasOps *o)
2036{
2037    const uint8_t es = get_field(s, m3);
2038    static const GVecGen2 g[4] = {
2039        { .fno = gen_helper_gvec_vpopct8, },
2040        { .fno = gen_helper_gvec_vpopct16, },
2041        { .fni4 = tcg_gen_ctpop_i32, },
2042        { .fni8 = tcg_gen_ctpop_i64, },
2043    };
2044
2045    if (es > ES_64 || (es != ES_8 && !s390_has_feat(S390_FEAT_VECTOR_ENH))) {
2046        gen_program_exception(s, PGM_SPECIFICATION);
2047        return DISAS_NORETURN;
2048    }
2049
2050    gen_gvec_2(get_field(s, v1), get_field(s, v2), &g[es]);
2051    return DISAS_NEXT;
2052}
2053
2054static void gen_rim_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, int32_t c)
2055{
2056    TCGv_i32 t = tcg_temp_new_i32();
2057
2058    tcg_gen_rotli_i32(t, a, c & 31);
2059    tcg_gen_and_i32(t, t, b);
2060    tcg_gen_andc_i32(d, d, b);
2061    tcg_gen_or_i32(d, d, t);
2062}
2063
2064static void gen_rim_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, int64_t c)
2065{
2066    TCGv_i64 t = tcg_temp_new_i64();
2067
2068    tcg_gen_rotli_i64(t, a, c & 63);
2069    tcg_gen_and_i64(t, t, b);
2070    tcg_gen_andc_i64(d, d, b);
2071    tcg_gen_or_i64(d, d, t);
2072}
2073
2074static DisasJumpType op_verim(DisasContext *s, DisasOps *o)
2075{
2076    const uint8_t es = get_field(s, m5);
2077    const uint8_t i4 = get_field(s, i4) &
2078                       (NUM_VEC_ELEMENT_BITS(es) - 1);
2079    static const GVecGen3i g[4] = {
2080        { .fno = gen_helper_gvec_verim8, },
2081        { .fno = gen_helper_gvec_verim16, },
2082        { .fni4 = gen_rim_i32,
2083          .load_dest = true, },
2084        { .fni8 = gen_rim_i64,
2085          .load_dest = true, },
2086    };
2087
2088    if (es > ES_64) {
2089        gen_program_exception(s, PGM_SPECIFICATION);
2090        return DISAS_NORETURN;
2091    }
2092
2093    gen_gvec_3i(get_field(s, v1), get_field(s, v2),
2094                get_field(s, v3), i4, &g[es]);
2095    return DISAS_NEXT;
2096}
2097
2098static DisasJumpType op_vesv(DisasContext *s, DisasOps *o)
2099{
2100    const uint8_t es = get_field(s, m4);
2101    const uint8_t v1 = get_field(s, v1);
2102    const uint8_t v2 = get_field(s, v2);
2103    const uint8_t v3 = get_field(s, v3);
2104
2105    if (es > ES_64) {
2106        gen_program_exception(s, PGM_SPECIFICATION);
2107        return DISAS_NORETURN;
2108    }
2109
2110    switch (s->fields.op2) {
2111    case 0x70:
2112        gen_gvec_fn_3(shlv, es, v1, v2, v3);
2113        break;
2114    case 0x73:
2115        gen_gvec_fn_3(rotlv, es, v1, v2, v3);
2116        break;
2117    case 0x7a:
2118        gen_gvec_fn_3(sarv, es, v1, v2, v3);
2119        break;
2120    case 0x78:
2121        gen_gvec_fn_3(shrv, es, v1, v2, v3);
2122        break;
2123    default:
2124        g_assert_not_reached();
2125    }
2126    return DISAS_NEXT;
2127}
2128
2129static DisasJumpType op_ves(DisasContext *s, DisasOps *o)
2130{
2131    const uint8_t es = get_field(s, m4);
2132    const uint8_t d2 = get_field(s, d2) &
2133                       (NUM_VEC_ELEMENT_BITS(es) - 1);
2134    const uint8_t v1 = get_field(s, v1);
2135    const uint8_t v3 = get_field(s, v3);
2136    TCGv_i32 shift;
2137
2138    if (es > ES_64) {
2139        gen_program_exception(s, PGM_SPECIFICATION);
2140        return DISAS_NORETURN;
2141    }
2142
2143    if (likely(!get_field(s, b2))) {
2144        switch (s->fields.op2) {
2145        case 0x30:
2146            gen_gvec_fn_2i(shli, es, v1, v3, d2);
2147            break;
2148        case 0x33:
2149            gen_gvec_fn_2i(rotli, es, v1, v3, d2);
2150            break;
2151        case 0x3a:
2152            gen_gvec_fn_2i(sari, es, v1, v3, d2);
2153            break;
2154        case 0x38:
2155            gen_gvec_fn_2i(shri, es, v1, v3, d2);
2156            break;
2157        default:
2158            g_assert_not_reached();
2159        }
2160    } else {
2161        shift = tcg_temp_new_i32();
2162        tcg_gen_extrl_i64_i32(shift, o->addr1);
2163        tcg_gen_andi_i32(shift, shift, NUM_VEC_ELEMENT_BITS(es) - 1);
2164        switch (s->fields.op2) {
2165        case 0x30:
2166            gen_gvec_fn_2s(shls, es, v1, v3, shift);
2167            break;
2168        case 0x33:
2169            gen_gvec_fn_2s(rotls, es, v1, v3, shift);
2170            break;
2171        case 0x3a:
2172            gen_gvec_fn_2s(sars, es, v1, v3, shift);
2173            break;
2174        case 0x38:
2175            gen_gvec_fn_2s(shrs, es, v1, v3, shift);
2176            break;
2177        default:
2178            g_assert_not_reached();
2179        }
2180    }
2181    return DISAS_NEXT;
2182}
2183
2184static DisasJumpType gen_vsh_by_byte(DisasContext *s, DisasOps *o,
2185                                      gen_helper_gvec_2i *gen,
2186                                      gen_helper_gvec_3 *gen_ve2)
2187{
2188    bool byte = s->insn->data;
2189
2190    if (!byte && s390_has_feat(S390_FEAT_VECTOR_ENH2)) {
2191        gen_gvec_3_ool(get_field(s, v1), get_field(s, v2),
2192                       get_field(s, v3), 0, gen_ve2);
2193    } else {
2194        TCGv_i64 shift = tcg_temp_new_i64();
2195
2196        read_vec_element_i64(shift, get_field(s, v3), 7, ES_8);
2197        tcg_gen_andi_i64(shift, shift, byte ? 0x78 : 7);
2198        gen_gvec_2i_ool(get_field(s, v1), get_field(s, v2), shift, 0, gen);
2199    }
2200    return DISAS_NEXT;
2201}
2202
2203static DisasJumpType op_vsl(DisasContext *s, DisasOps *o)
2204{
2205    return gen_vsh_by_byte(s, o, gen_helper_gvec_vsl,
2206                            gen_helper_gvec_vsl_ve2);
2207}
2208
2209static DisasJumpType op_vsra(DisasContext *s, DisasOps *o)
2210{
2211    return gen_vsh_by_byte(s, o, gen_helper_gvec_vsra,
2212                            gen_helper_gvec_vsra_ve2);
2213}
2214
2215static DisasJumpType op_vsrl(DisasContext *s, DisasOps *o)
2216{
2217    return gen_vsh_by_byte(s, o, gen_helper_gvec_vsrl,
2218                            gen_helper_gvec_vsrl_ve2);
2219}
2220
2221static DisasJumpType op_vsld(DisasContext *s, DisasOps *o)
2222{
2223    const bool byte = s->insn->data;
2224    const uint8_t mask = byte ? 15 : 7;
2225    const uint8_t mul  = byte ?  8 : 1;
2226    const uint8_t i4   = get_field(s, i4);
2227    const int right_shift = 64 - (i4 & 7) * mul;
2228    TCGv_i64 t0, t1, t2;
2229
2230    if (i4 & ~mask) {
2231        gen_program_exception(s, PGM_SPECIFICATION);
2232        return DISAS_NORETURN;
2233    }
2234
2235    t0 = tcg_temp_new_i64();
2236    t1 = tcg_temp_new_i64();
2237    t2 = tcg_temp_new_i64();
2238
2239    if ((i4 & 8) == 0) {
2240        read_vec_element_i64(t0, get_field(s, v2), 0, ES_64);
2241        read_vec_element_i64(t1, get_field(s, v2), 1, ES_64);
2242        read_vec_element_i64(t2, get_field(s, v3), 0, ES_64);
2243    } else {
2244        read_vec_element_i64(t0, get_field(s, v2), 1, ES_64);
2245        read_vec_element_i64(t1, get_field(s, v3), 0, ES_64);
2246        read_vec_element_i64(t2, get_field(s, v3), 1, ES_64);
2247    }
2248
2249    tcg_gen_extract2_i64(t0, t1, t0, right_shift);
2250    tcg_gen_extract2_i64(t1, t2, t1, right_shift);
2251
2252    write_vec_element_i64(t0, get_field(s, v1), 0, ES_64);
2253    write_vec_element_i64(t1, get_field(s, v1), 1, ES_64);
2254    return DISAS_NEXT;
2255}
2256
2257static DisasJumpType op_vsrd(DisasContext *s, DisasOps *o)
2258{
2259    const uint8_t i4 = get_field(s, i4);
2260    TCGv_i64 t0, t1, t2;
2261
2262    if (i4 & ~7) {
2263        gen_program_exception(s, PGM_SPECIFICATION);
2264        return DISAS_NORETURN;
2265    }
2266
2267    t0 = tcg_temp_new_i64();
2268    t1 = tcg_temp_new_i64();
2269    t2 = tcg_temp_new_i64();
2270
2271    read_vec_element_i64(t0, get_field(s, v2), 1, ES_64);
2272    read_vec_element_i64(t1, get_field(s, v3), 0, ES_64);
2273    read_vec_element_i64(t2, get_field(s, v3), 1, ES_64);
2274
2275    tcg_gen_extract2_i64(t0, t1, t0, i4);
2276    tcg_gen_extract2_i64(t1, t2, t1, i4);
2277
2278    write_vec_element_i64(t0, get_field(s, v1), 0, ES_64);
2279    write_vec_element_i64(t1, get_field(s, v1), 1, ES_64);
2280    return DISAS_NEXT;
2281}
2282
2283static DisasJumpType op_vs(DisasContext *s, DisasOps *o)
2284{
2285    const uint8_t es = get_field(s, m4);
2286
2287    if (es > ES_128) {
2288        gen_program_exception(s, PGM_SPECIFICATION);
2289        return DISAS_NORETURN;
2290    } else if (es == ES_128) {
2291        gen_gvec128_3_i64(tcg_gen_sub2_i64, get_field(s, v1),
2292                          get_field(s, v2), get_field(s, v3));
2293        return DISAS_NEXT;
2294    }
2295    gen_gvec_fn_3(sub, es, get_field(s, v1), get_field(s, v2),
2296                  get_field(s, v3));
2297    return DISAS_NEXT;
2298}
2299
2300static void gen_scbi_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2301{
2302    tcg_gen_setcond_i32(TCG_COND_GEU, d, a, b);
2303}
2304
2305static void gen_scbi_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2306{
2307    tcg_gen_setcond_i64(TCG_COND_GEU, d, a, b);
2308}
2309
2310static void gen_scbi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
2311                          TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2312{
2313    TCGv_i64 th = tcg_temp_new_i64();
2314    TCGv_i64 tl = tcg_temp_new_i64();
2315    TCGv_i64 zero = tcg_constant_i64(0);
2316
2317    tcg_gen_sub2_i64(tl, th, al, zero, bl, zero);
2318    tcg_gen_andi_i64(th, th, 1);
2319    tcg_gen_sub2_i64(tl, th, ah, zero, th, zero);
2320    tcg_gen_sub2_i64(tl, th, tl, th, bh, zero);
2321    /* "invert" the result: -1 -> 0; 0 -> 1 */
2322    tcg_gen_addi_i64(dl, th, 1);
2323    tcg_gen_mov_i64(dh, zero);
2324}
2325
2326static DisasJumpType op_vscbi(DisasContext *s, DisasOps *o)
2327{
2328    const uint8_t es = get_field(s, m4);
2329    static const GVecGen3 g[4] = {
2330        { .fno = gen_helper_gvec_vscbi8, },
2331        { .fno = gen_helper_gvec_vscbi16, },
2332        { .fni4 = gen_scbi_i32, },
2333        { .fni8 = gen_scbi_i64, },
2334    };
2335
2336    if (es > ES_128) {
2337        gen_program_exception(s, PGM_SPECIFICATION);
2338        return DISAS_NORETURN;
2339    } else if (es == ES_128) {
2340        gen_gvec128_3_i64(gen_scbi2_i64, get_field(s, v1),
2341                          get_field(s, v2), get_field(s, v3));
2342        return DISAS_NEXT;
2343    }
2344    gen_gvec_3(get_field(s, v1), get_field(s, v2),
2345               get_field(s, v3), &g[es]);
2346    return DISAS_NEXT;
2347}
2348
2349static void gen_sbi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
2350                         TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
2351{
2352    TCGv_i64 tl = tcg_temp_new_i64();
2353    TCGv_i64 th = tcg_temp_new_i64();
2354
2355    tcg_gen_not_i64(tl, bl);
2356    tcg_gen_not_i64(th, bh);
2357    gen_ac2_i64(dl, dh, al, ah, tl, th, cl, ch);
2358}
2359
2360static DisasJumpType op_vsbi(DisasContext *s, DisasOps *o)
2361{
2362    if (get_field(s, m5) != ES_128) {
2363        gen_program_exception(s, PGM_SPECIFICATION);
2364        return DISAS_NORETURN;
2365    }
2366
2367    gen_gvec128_4_i64(gen_sbi2_i64, get_field(s, v1),
2368                      get_field(s, v2), get_field(s, v3),
2369                      get_field(s, v4));
2370    return DISAS_NEXT;
2371}
2372
2373static void gen_sbcbi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
2374                           TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
2375{
2376    TCGv_i64 th = tcg_temp_new_i64();
2377    TCGv_i64 tl = tcg_temp_new_i64();
2378
2379    tcg_gen_not_i64(tl, bl);
2380    tcg_gen_not_i64(th, bh);
2381    gen_accc2_i64(dl, dh, al, ah, tl, th, cl, ch);
2382}
2383
2384static DisasJumpType op_vsbcbi(DisasContext *s, DisasOps *o)
2385{
2386    if (get_field(s, m5) != ES_128) {
2387        gen_program_exception(s, PGM_SPECIFICATION);
2388        return DISAS_NORETURN;
2389    }
2390
2391    gen_gvec128_4_i64(gen_sbcbi2_i64, get_field(s, v1),
2392                      get_field(s, v2), get_field(s, v3),
2393                      get_field(s, v4));
2394    return DISAS_NEXT;
2395}
2396
2397static DisasJumpType op_vsumg(DisasContext *s, DisasOps *o)
2398{
2399    const uint8_t es = get_field(s, m4);
2400    TCGv_i64 sum, tmp;
2401    uint8_t dst_idx;
2402
2403    if (es == ES_8 || es > ES_32) {
2404        gen_program_exception(s, PGM_SPECIFICATION);
2405        return DISAS_NORETURN;
2406    }
2407
2408    sum = tcg_temp_new_i64();
2409    tmp = tcg_temp_new_i64();
2410    for (dst_idx = 0; dst_idx < 2; dst_idx++) {
2411        uint8_t idx = dst_idx * NUM_VEC_ELEMENTS(es) / 2;
2412        const uint8_t max_idx = idx + NUM_VEC_ELEMENTS(es) / 2 - 1;
2413
2414        read_vec_element_i64(sum, get_field(s, v3), max_idx, es);
2415        for (; idx <= max_idx; idx++) {
2416            read_vec_element_i64(tmp, get_field(s, v2), idx, es);
2417            tcg_gen_add_i64(sum, sum, tmp);
2418        }
2419        write_vec_element_i64(sum, get_field(s, v1), dst_idx, ES_64);
2420    }
2421    return DISAS_NEXT;
2422}
2423
2424static DisasJumpType op_vsumq(DisasContext *s, DisasOps *o)
2425{
2426    const uint8_t es = get_field(s, m4);
2427    const uint8_t max_idx = NUM_VEC_ELEMENTS(es) - 1;
2428    TCGv_i64 sumh, suml, zero, tmpl;
2429    uint8_t idx;
2430
2431    if (es < ES_32 || es > ES_64) {
2432        gen_program_exception(s, PGM_SPECIFICATION);
2433        return DISAS_NORETURN;
2434    }
2435
2436    sumh = tcg_temp_new_i64();
2437    suml = tcg_temp_new_i64();
2438    zero = tcg_constant_i64(0);
2439    tmpl = tcg_temp_new_i64();
2440
2441    tcg_gen_mov_i64(sumh, zero);
2442    read_vec_element_i64(suml, get_field(s, v3), max_idx, es);
2443    for (idx = 0; idx <= max_idx; idx++) {
2444        read_vec_element_i64(tmpl, get_field(s, v2), idx, es);
2445        tcg_gen_add2_i64(suml, sumh, suml, sumh, tmpl, zero);
2446    }
2447    write_vec_element_i64(sumh, get_field(s, v1), 0, ES_64);
2448    write_vec_element_i64(suml, get_field(s, v1), 1, ES_64);
2449    return DISAS_NEXT;
2450}
2451
2452static DisasJumpType op_vsum(DisasContext *s, DisasOps *o)
2453{
2454    const uint8_t es = get_field(s, m4);
2455    TCGv_i32 sum, tmp;
2456    uint8_t dst_idx;
2457
2458    if (es > ES_16) {
2459        gen_program_exception(s, PGM_SPECIFICATION);
2460        return DISAS_NORETURN;
2461    }
2462
2463    sum = tcg_temp_new_i32();
2464    tmp = tcg_temp_new_i32();
2465    for (dst_idx = 0; dst_idx < 4; dst_idx++) {
2466        uint8_t idx = dst_idx * NUM_VEC_ELEMENTS(es) / 4;
2467        const uint8_t max_idx = idx + NUM_VEC_ELEMENTS(es) / 4 - 1;
2468
2469        read_vec_element_i32(sum, get_field(s, v3), max_idx, es);
2470        for (; idx <= max_idx; idx++) {
2471            read_vec_element_i32(tmp, get_field(s, v2), idx, es);
2472            tcg_gen_add_i32(sum, sum, tmp);
2473        }
2474        write_vec_element_i32(sum, get_field(s, v1), dst_idx, ES_32);
2475    }
2476    return DISAS_NEXT;
2477}
2478
2479static DisasJumpType op_vtm(DisasContext *s, DisasOps *o)
2480{
2481    gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2),
2482                   tcg_env, 0, gen_helper_gvec_vtm);
2483    set_cc_static(s);
2484    return DISAS_NEXT;
2485}
2486
2487static DisasJumpType op_vfae(DisasContext *s, DisasOps *o)
2488{
2489    const uint8_t es = get_field(s, m4);
2490    const uint8_t m5 = get_field(s, m5);
2491    static gen_helper_gvec_3 * const g[3] = {
2492        gen_helper_gvec_vfae8,
2493        gen_helper_gvec_vfae16,
2494        gen_helper_gvec_vfae32,
2495    };
2496    static gen_helper_gvec_3_ptr * const g_cc[3] = {
2497        gen_helper_gvec_vfae_cc8,
2498        gen_helper_gvec_vfae_cc16,
2499        gen_helper_gvec_vfae_cc32,
2500    };
2501    if (es > ES_32) {
2502        gen_program_exception(s, PGM_SPECIFICATION);
2503        return DISAS_NORETURN;
2504    }
2505
2506    if (extract32(m5, 0, 1)) {
2507        gen_gvec_3_ptr(get_field(s, v1), get_field(s, v2),
2508                       get_field(s, v3), tcg_env, m5, g_cc[es]);
2509        set_cc_static(s);
2510    } else {
2511        gen_gvec_3_ool(get_field(s, v1), get_field(s, v2),
2512                       get_field(s, v3), m5, g[es]);
2513    }
2514    return DISAS_NEXT;
2515}
2516
2517static DisasJumpType op_vfee(DisasContext *s, DisasOps *o)
2518{
2519    const uint8_t es = get_field(s, m4);
2520    const uint8_t m5 = get_field(s, m5);
2521    static gen_helper_gvec_3 * const g[3] = {
2522        gen_helper_gvec_vfee8,
2523        gen_helper_gvec_vfee16,
2524        gen_helper_gvec_vfee32,
2525    };
2526    static gen_helper_gvec_3_ptr * const g_cc[3] = {
2527        gen_helper_gvec_vfee_cc8,
2528        gen_helper_gvec_vfee_cc16,
2529        gen_helper_gvec_vfee_cc32,
2530    };
2531
2532    if (es > ES_32 || m5 & ~0x3) {
2533        gen_program_exception(s, PGM_SPECIFICATION);
2534        return DISAS_NORETURN;
2535    }
2536
2537    if (extract32(m5, 0, 1)) {
2538        gen_gvec_3_ptr(get_field(s, v1), get_field(s, v2),
2539                       get_field(s, v3), tcg_env, m5, g_cc[es]);
2540        set_cc_static(s);
2541    } else {
2542        gen_gvec_3_ool(get_field(s, v1), get_field(s, v2),
2543                       get_field(s, v3), m5, g[es]);
2544    }
2545    return DISAS_NEXT;
2546}
2547
2548static DisasJumpType op_vfene(DisasContext *s, DisasOps *o)
2549{
2550    const uint8_t es = get_field(s, m4);
2551    const uint8_t m5 = get_field(s, m5);
2552    static gen_helper_gvec_3 * const g[3] = {
2553        gen_helper_gvec_vfene8,
2554        gen_helper_gvec_vfene16,
2555        gen_helper_gvec_vfene32,
2556    };
2557    static gen_helper_gvec_3_ptr * const g_cc[3] = {
2558        gen_helper_gvec_vfene_cc8,
2559        gen_helper_gvec_vfene_cc16,
2560        gen_helper_gvec_vfene_cc32,
2561    };
2562
2563    if (es > ES_32 || m5 & ~0x3) {
2564        gen_program_exception(s, PGM_SPECIFICATION);
2565        return DISAS_NORETURN;
2566    }
2567
2568    if (extract32(m5, 0, 1)) {
2569        gen_gvec_3_ptr(get_field(s, v1), get_field(s, v2),
2570                       get_field(s, v3), tcg_env, m5, g_cc[es]);
2571        set_cc_static(s);
2572    } else {
2573        gen_gvec_3_ool(get_field(s, v1), get_field(s, v2),
2574                       get_field(s, v3), m5, g[es]);
2575    }
2576    return DISAS_NEXT;
2577}
2578
2579static DisasJumpType op_vistr(DisasContext *s, DisasOps *o)
2580{
2581    const uint8_t es = get_field(s, m3);
2582    const uint8_t m5 = get_field(s, m5);
2583    static gen_helper_gvec_2 * const g[3] = {
2584        gen_helper_gvec_vistr8,
2585        gen_helper_gvec_vistr16,
2586        gen_helper_gvec_vistr32,
2587    };
2588    static gen_helper_gvec_2_ptr * const g_cc[3] = {
2589        gen_helper_gvec_vistr_cc8,
2590        gen_helper_gvec_vistr_cc16,
2591        gen_helper_gvec_vistr_cc32,
2592    };
2593
2594    if (es > ES_32 || m5 & ~0x1) {
2595        gen_program_exception(s, PGM_SPECIFICATION);
2596        return DISAS_NORETURN;
2597    }
2598
2599    if (extract32(m5, 0, 1)) {
2600        gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2),
2601                       tcg_env, 0, g_cc[es]);
2602        set_cc_static(s);
2603    } else {
2604        gen_gvec_2_ool(get_field(s, v1), get_field(s, v2), 0,
2605                       g[es]);
2606    }
2607    return DISAS_NEXT;
2608}
2609
2610static DisasJumpType op_vstrc(DisasContext *s, DisasOps *o)
2611{
2612    const uint8_t es = get_field(s, m5);
2613    const uint8_t m6 = get_field(s, m6);
2614    static gen_helper_gvec_4 * const g[3] = {
2615        gen_helper_gvec_vstrc8,
2616        gen_helper_gvec_vstrc16,
2617        gen_helper_gvec_vstrc32,
2618    };
2619    static gen_helper_gvec_4 * const g_rt[3] = {
2620        gen_helper_gvec_vstrc_rt8,
2621        gen_helper_gvec_vstrc_rt16,
2622        gen_helper_gvec_vstrc_rt32,
2623    };
2624    static gen_helper_gvec_4_ptr * const g_cc[3] = {
2625        gen_helper_gvec_vstrc_cc8,
2626        gen_helper_gvec_vstrc_cc16,
2627        gen_helper_gvec_vstrc_cc32,
2628    };
2629    static gen_helper_gvec_4_ptr * const g_cc_rt[3] = {
2630        gen_helper_gvec_vstrc_cc_rt8,
2631        gen_helper_gvec_vstrc_cc_rt16,
2632        gen_helper_gvec_vstrc_cc_rt32,
2633    };
2634
2635    if (es > ES_32) {
2636        gen_program_exception(s, PGM_SPECIFICATION);
2637        return DISAS_NORETURN;
2638    }
2639
2640    if (extract32(m6, 0, 1)) {
2641        if (extract32(m6, 2, 1)) {
2642            gen_gvec_4_ptr(get_field(s, v1), get_field(s, v2),
2643                           get_field(s, v3), get_field(s, v4),
2644                           tcg_env, m6, g_cc_rt[es]);
2645        } else {
2646            gen_gvec_4_ptr(get_field(s, v1), get_field(s, v2),
2647                           get_field(s, v3), get_field(s, v4),
2648                           tcg_env, m6, g_cc[es]);
2649        }
2650        set_cc_static(s);
2651    } else {
2652        if (extract32(m6, 2, 1)) {
2653            gen_gvec_4_ool(get_field(s, v1), get_field(s, v2),
2654                           get_field(s, v3), get_field(s, v4),
2655                           m6, g_rt[es]);
2656        } else {
2657            gen_gvec_4_ool(get_field(s, v1), get_field(s, v2),
2658                           get_field(s, v3), get_field(s, v4),
2659                           m6, g[es]);
2660        }
2661    }
2662    return DISAS_NEXT;
2663}
2664
2665static DisasJumpType op_vstrs(DisasContext *s, DisasOps *o)
2666{
2667    typedef void (*helper_vstrs)(TCGv_ptr, TCGv_ptr, TCGv_ptr,
2668                                 TCGv_ptr, TCGv_ptr, TCGv_i32);
2669    static const helper_vstrs fns[3][2] = {
2670        { gen_helper_gvec_vstrs_8, gen_helper_gvec_vstrs_zs8 },
2671        { gen_helper_gvec_vstrs_16, gen_helper_gvec_vstrs_zs16 },
2672        { gen_helper_gvec_vstrs_32, gen_helper_gvec_vstrs_zs32 },
2673    };
2674    const uint8_t es = get_field(s, m5);
2675    const uint8_t m6 = get_field(s, m6);
2676    const bool zs = extract32(m6, 1, 1);
2677
2678    if (es > ES_32 || m6 & ~2) {
2679        gen_program_exception(s, PGM_SPECIFICATION);
2680        return DISAS_NORETURN;
2681    }
2682
2683    gen_gvec_4_ptr(get_field(s, v1), get_field(s, v2),
2684                   get_field(s, v3), get_field(s, v4),
2685                   tcg_env, 0, fns[es][zs]);
2686    set_cc_static(s);
2687    return DISAS_NEXT;
2688}
2689
2690static DisasJumpType op_vfa(DisasContext *s, DisasOps *o)
2691{
2692    const uint8_t fpf = get_field(s, m4);
2693    const uint8_t m5 = get_field(s, m5);
2694    gen_helper_gvec_3_ptr *fn = NULL;
2695
2696    switch (s->fields.op2) {
2697    case 0xe3:
2698        switch (fpf) {
2699        case FPF_SHORT:
2700            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
2701                fn = gen_helper_gvec_vfa32;
2702            }
2703            break;
2704        case FPF_LONG:
2705            fn = gen_helper_gvec_vfa64;
2706            break;
2707        case FPF_EXT:
2708            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
2709                fn = gen_helper_gvec_vfa128;
2710            }
2711            break;
2712        default:
2713            break;
2714        }
2715        break;
2716    case 0xe5:
2717        switch (fpf) {
2718        case FPF_SHORT:
2719            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
2720                fn = gen_helper_gvec_vfd32;
2721            }
2722            break;
2723        case FPF_LONG:
2724            fn = gen_helper_gvec_vfd64;
2725            break;
2726        case FPF_EXT:
2727            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
2728                fn = gen_helper_gvec_vfd128;
2729            }
2730            break;
2731        default:
2732            break;
2733        }
2734        break;
2735    case 0xe7:
2736        switch (fpf) {
2737        case FPF_SHORT:
2738            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
2739                fn = gen_helper_gvec_vfm32;
2740            }
2741            break;
2742        case FPF_LONG:
2743            fn = gen_helper_gvec_vfm64;
2744            break;
2745        case FPF_EXT:
2746            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
2747                fn = gen_helper_gvec_vfm128;
2748            }
2749            break;
2750        default:
2751            break;
2752        }
2753        break;
2754    case 0xe2:
2755        switch (fpf) {
2756        case FPF_SHORT:
2757            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
2758                fn = gen_helper_gvec_vfs32;
2759            }
2760            break;
2761        case FPF_LONG:
2762            fn = gen_helper_gvec_vfs64;
2763            break;
2764        case FPF_EXT:
2765            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
2766                fn = gen_helper_gvec_vfs128;
2767            }
2768            break;
2769        default:
2770            break;
2771        }
2772        break;
2773    default:
2774        g_assert_not_reached();
2775    }
2776
2777    if (!fn || extract32(m5, 0, 3)) {
2778        gen_program_exception(s, PGM_SPECIFICATION);
2779        return DISAS_NORETURN;
2780    }
2781
2782    gen_gvec_3_ptr(get_field(s, v1), get_field(s, v2),
2783                   get_field(s, v3), tcg_env, m5, fn);
2784    return DISAS_NEXT;
2785}
2786
2787static DisasJumpType op_wfc(DisasContext *s, DisasOps *o)
2788{
2789    const uint8_t fpf = get_field(s, m3);
2790    const uint8_t m4 = get_field(s, m4);
2791    gen_helper_gvec_2_ptr *fn = NULL;
2792
2793    switch (fpf) {
2794    case FPF_SHORT:
2795        if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
2796            fn = gen_helper_gvec_wfk32;
2797            if (s->fields.op2 == 0xcb) {
2798                fn = gen_helper_gvec_wfc32;
2799            }
2800        }
2801        break;
2802    case FPF_LONG:
2803        fn = gen_helper_gvec_wfk64;
2804        if (s->fields.op2 == 0xcb) {
2805            fn = gen_helper_gvec_wfc64;
2806        }
2807        break;
2808    case FPF_EXT:
2809        if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
2810            fn = gen_helper_gvec_wfk128;
2811            if (s->fields.op2 == 0xcb) {
2812                fn = gen_helper_gvec_wfc128;
2813            }
2814        }
2815        break;
2816    default:
2817        break;
2818    };
2819
2820    if (!fn || m4) {
2821        gen_program_exception(s, PGM_SPECIFICATION);
2822        return DISAS_NORETURN;
2823    }
2824
2825    gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2), tcg_env, 0, fn);
2826    set_cc_static(s);
2827    return DISAS_NEXT;
2828}
2829
2830static DisasJumpType op_vfc(DisasContext *s, DisasOps *o)
2831{
2832    const uint8_t fpf = get_field(s, m4);
2833    const uint8_t m5 = get_field(s, m5);
2834    const uint8_t m6 = get_field(s, m6);
2835    const bool cs = extract32(m6, 0, 1);
2836    const bool sq = extract32(m5, 2, 1);
2837    gen_helper_gvec_3_ptr *fn = NULL;
2838
2839    switch (s->fields.op2) {
2840    case 0xe8:
2841        switch (fpf) {
2842        case FPF_SHORT:
2843            fn = cs ? gen_helper_gvec_vfce32_cc : gen_helper_gvec_vfce32;
2844            break;
2845        case FPF_LONG:
2846            fn = cs ? gen_helper_gvec_vfce64_cc : gen_helper_gvec_vfce64;
2847            break;
2848        case FPF_EXT:
2849            fn = cs ? gen_helper_gvec_vfce128_cc : gen_helper_gvec_vfce128;
2850            break;
2851        default:
2852            break;
2853        }
2854        break;
2855    case 0xeb:
2856        switch (fpf) {
2857        case FPF_SHORT:
2858            fn = cs ? gen_helper_gvec_vfch32_cc : gen_helper_gvec_vfch32;
2859            break;
2860        case FPF_LONG:
2861            fn = cs ? gen_helper_gvec_vfch64_cc : gen_helper_gvec_vfch64;
2862            break;
2863        case FPF_EXT:
2864            fn = cs ? gen_helper_gvec_vfch128_cc : gen_helper_gvec_vfch128;
2865            break;
2866        default:
2867            break;
2868        }
2869        break;
2870    case 0xea:
2871        switch (fpf) {
2872        case FPF_SHORT:
2873            fn = cs ? gen_helper_gvec_vfche32_cc : gen_helper_gvec_vfche32;
2874            break;
2875        case FPF_LONG:
2876            fn = cs ? gen_helper_gvec_vfche64_cc : gen_helper_gvec_vfche64;
2877            break;
2878        case FPF_EXT:
2879            fn = cs ? gen_helper_gvec_vfche128_cc : gen_helper_gvec_vfche128;
2880            break;
2881        default:
2882            break;
2883        }
2884        break;
2885    default:
2886        g_assert_not_reached();
2887    }
2888
2889    if (!fn || extract32(m5, 0, 2) || extract32(m6, 1, 3) ||
2890        (!s390_has_feat(S390_FEAT_VECTOR_ENH) && (fpf != FPF_LONG || sq))) {
2891        gen_program_exception(s, PGM_SPECIFICATION);
2892        return DISAS_NORETURN;
2893    }
2894
2895    gen_gvec_3_ptr(get_field(s, v1), get_field(s, v2), get_field(s, v3),
2896                   tcg_env, m5, fn);
2897    if (cs) {
2898        set_cc_static(s);
2899    }
2900    return DISAS_NEXT;
2901}
2902
2903static DisasJumpType op_vcdg(DisasContext *s, DisasOps *o)
2904{
2905    const uint8_t fpf = get_field(s, m3);
2906    const uint8_t m4 = get_field(s, m4);
2907    const uint8_t erm = get_field(s, m5);
2908    gen_helper_gvec_2_ptr *fn = NULL;
2909
2910
2911    switch (s->fields.op2) {
2912    case 0xc3:
2913        switch (fpf) {
2914        case FPF_LONG:
2915            fn = gen_helper_gvec_vcdg64;
2916            break;
2917        case FPF_SHORT:
2918            if (s390_has_feat(S390_FEAT_VECTOR_ENH2)) {
2919                fn = gen_helper_gvec_vcdg32;
2920            }
2921            break;
2922        default:
2923            break;
2924        }
2925        break;
2926    case 0xc1:
2927        switch (fpf) {
2928        case FPF_LONG:
2929            fn = gen_helper_gvec_vcdlg64;
2930            break;
2931        case FPF_SHORT:
2932            if (s390_has_feat(S390_FEAT_VECTOR_ENH2)) {
2933                fn = gen_helper_gvec_vcdlg32;
2934            }
2935            break;
2936        default:
2937            break;
2938        }
2939        break;
2940    case 0xc2:
2941        switch (fpf) {
2942        case FPF_LONG:
2943            fn = gen_helper_gvec_vcgd64;
2944            break;
2945        case FPF_SHORT:
2946            if (s390_has_feat(S390_FEAT_VECTOR_ENH2)) {
2947                fn = gen_helper_gvec_vcgd32;
2948            }
2949            break;
2950        default:
2951            break;
2952        }
2953        break;
2954    case 0xc0:
2955        switch (fpf) {
2956        case FPF_LONG:
2957            fn = gen_helper_gvec_vclgd64;
2958            break;
2959        case FPF_SHORT:
2960            if (s390_has_feat(S390_FEAT_VECTOR_ENH2)) {
2961                fn = gen_helper_gvec_vclgd32;
2962            }
2963            break;
2964        default:
2965            break;
2966        }
2967        break;
2968    case 0xc7:
2969        switch (fpf) {
2970        case FPF_SHORT:
2971            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
2972                fn = gen_helper_gvec_vfi32;
2973            }
2974            break;
2975        case FPF_LONG:
2976            fn = gen_helper_gvec_vfi64;
2977            break;
2978        case FPF_EXT:
2979            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
2980                fn = gen_helper_gvec_vfi128;
2981            }
2982            break;
2983        default:
2984            break;
2985        }
2986        break;
2987    case 0xc5:
2988        switch (fpf) {
2989        case FPF_LONG:
2990            fn = gen_helper_gvec_vflr64;
2991            break;
2992        case FPF_EXT:
2993            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
2994                fn = gen_helper_gvec_vflr128;
2995            }
2996            break;
2997        default:
2998            break;
2999        }
3000        break;
3001    default:
3002        g_assert_not_reached();
3003    }
3004
3005    if (!fn || extract32(m4, 0, 2) || erm > 7 || erm == 2) {
3006        gen_program_exception(s, PGM_SPECIFICATION);
3007        return DISAS_NORETURN;
3008    }
3009
3010    gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2), tcg_env,
3011                   deposit32(m4, 4, 4, erm), fn);
3012    return DISAS_NEXT;
3013}
3014
3015static DisasJumpType op_vfll(DisasContext *s, DisasOps *o)
3016{
3017    const uint8_t fpf = get_field(s, m3);
3018    const uint8_t m4 = get_field(s, m4);
3019    gen_helper_gvec_2_ptr *fn = NULL;
3020
3021    switch (fpf) {
3022    case FPF_SHORT:
3023        fn = gen_helper_gvec_vfll32;
3024        break;
3025    case FPF_LONG:
3026        if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
3027            fn = gen_helper_gvec_vfll64;
3028        }
3029        break;
3030    default:
3031        break;
3032    }
3033
3034    if (!fn || extract32(m4, 0, 3)) {
3035        gen_program_exception(s, PGM_SPECIFICATION);
3036        return DISAS_NORETURN;
3037    }
3038
3039    gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2), tcg_env, m4, fn);
3040    return DISAS_NEXT;
3041}
3042
3043static DisasJumpType op_vfmax(DisasContext *s, DisasOps *o)
3044{
3045    const uint8_t fpf = get_field(s, m4);
3046    const uint8_t m6 = get_field(s, m6);
3047    const uint8_t m5 = get_field(s, m5);
3048    gen_helper_gvec_3_ptr *fn;
3049
3050    if (m6 == 5 || m6 == 6 || m6 == 7 || m6 >= 13 || (m5 & 7)) {
3051        gen_program_exception(s, PGM_SPECIFICATION);
3052        return DISAS_NORETURN;
3053    }
3054
3055    switch (fpf) {
3056    case FPF_SHORT:
3057        if (s->fields.op2 == 0xef) {
3058            fn = gen_helper_gvec_vfmax32;
3059        } else {
3060            fn = gen_helper_gvec_vfmin32;
3061        }
3062        break;
3063    case FPF_LONG:
3064        if (s->fields.op2 == 0xef) {
3065            fn = gen_helper_gvec_vfmax64;
3066        } else {
3067            fn = gen_helper_gvec_vfmin64;
3068        }
3069        break;
3070    case FPF_EXT:
3071        if (s->fields.op2 == 0xef) {
3072            fn = gen_helper_gvec_vfmax128;
3073        } else {
3074            fn = gen_helper_gvec_vfmin128;
3075        }
3076        break;
3077    default:
3078        gen_program_exception(s, PGM_SPECIFICATION);
3079        return DISAS_NORETURN;
3080    }
3081
3082    gen_gvec_3_ptr(get_field(s, v1), get_field(s, v2), get_field(s, v3),
3083                   tcg_env, deposit32(m5, 4, 4, m6), fn);
3084    return DISAS_NEXT;
3085}
3086
3087static DisasJumpType op_vfma(DisasContext *s, DisasOps *o)
3088{
3089    const uint8_t m5 = get_field(s, m5);
3090    const uint8_t fpf = get_field(s, m6);
3091    gen_helper_gvec_4_ptr *fn = NULL;
3092
3093    switch (s->fields.op2) {
3094    case 0x8f:
3095        switch (fpf) {
3096        case FPF_SHORT:
3097            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
3098                fn = gen_helper_gvec_vfma32;
3099            }
3100            break;
3101        case FPF_LONG:
3102            fn = gen_helper_gvec_vfma64;
3103            break;
3104        case FPF_EXT:
3105            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
3106                fn = gen_helper_gvec_vfma128;
3107            }
3108            break;
3109        default:
3110            break;
3111        }
3112        break;
3113    case 0x8e:
3114        switch (fpf) {
3115        case FPF_SHORT:
3116            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
3117                fn = gen_helper_gvec_vfms32;
3118            }
3119            break;
3120        case FPF_LONG:
3121            fn = gen_helper_gvec_vfms64;
3122            break;
3123        case FPF_EXT:
3124            if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
3125                fn = gen_helper_gvec_vfms128;
3126            }
3127            break;
3128        default:
3129            break;
3130        }
3131        break;
3132    case 0x9f:
3133        switch (fpf) {
3134        case FPF_SHORT:
3135            fn = gen_helper_gvec_vfnma32;
3136            break;
3137        case FPF_LONG:
3138            fn = gen_helper_gvec_vfnma64;
3139            break;
3140        case FPF_EXT:
3141            fn = gen_helper_gvec_vfnma128;
3142            break;
3143        default:
3144            break;
3145        }
3146        break;
3147    case 0x9e:
3148        switch (fpf) {
3149        case FPF_SHORT:
3150            fn = gen_helper_gvec_vfnms32;
3151            break;
3152        case FPF_LONG:
3153            fn = gen_helper_gvec_vfnms64;
3154            break;
3155        case FPF_EXT:
3156            fn = gen_helper_gvec_vfnms128;
3157            break;
3158        default:
3159            break;
3160        }
3161        break;
3162    default:
3163        g_assert_not_reached();
3164    }
3165
3166    if (!fn || extract32(m5, 0, 3)) {
3167        gen_program_exception(s, PGM_SPECIFICATION);
3168        return DISAS_NORETURN;
3169    }
3170
3171    gen_gvec_4_ptr(get_field(s, v1), get_field(s, v2),
3172                   get_field(s, v3), get_field(s, v4), tcg_env, m5, fn);
3173    return DISAS_NEXT;
3174}
3175
3176static DisasJumpType op_vfpso(DisasContext *s, DisasOps *o)
3177{
3178    const uint8_t v1 = get_field(s, v1);
3179    const uint8_t v2 = get_field(s, v2);
3180    const uint8_t fpf = get_field(s, m3);
3181    const uint8_t m4 = get_field(s, m4);
3182    const uint8_t m5 = get_field(s, m5);
3183    const bool se = extract32(m4, 3, 1);
3184    TCGv_i64 tmp;
3185
3186    if ((fpf != FPF_LONG && !s390_has_feat(S390_FEAT_VECTOR_ENH)) ||
3187        extract32(m4, 0, 3) || m5 > 2) {
3188        gen_program_exception(s, PGM_SPECIFICATION);
3189        return DISAS_NORETURN;
3190    }
3191
3192    switch (fpf) {
3193    case FPF_SHORT:
3194        if (!se) {
3195            switch (m5) {
3196            case 0:
3197                /* sign bit is inverted (complement) */
3198                gen_gvec_fn_2i(xori, ES_32, v1, v2, 1ull << 31);
3199                break;
3200            case 1:
3201                /* sign bit is set to one (negative) */
3202                gen_gvec_fn_2i(ori, ES_32, v1, v2, 1ull << 31);
3203                break;
3204            case 2:
3205                /* sign bit is set to zero (positive) */
3206                gen_gvec_fn_2i(andi, ES_32, v1, v2, (1ull << 31) - 1);
3207                break;
3208            }
3209            return DISAS_NEXT;
3210        }
3211        break;
3212    case FPF_LONG:
3213        if (!se) {
3214            switch (m5) {
3215            case 0:
3216                /* sign bit is inverted (complement) */
3217                gen_gvec_fn_2i(xori, ES_64, v1, v2, 1ull << 63);
3218                break;
3219            case 1:
3220                /* sign bit is set to one (negative) */
3221                gen_gvec_fn_2i(ori, ES_64, v1, v2, 1ull << 63);
3222                break;
3223            case 2:
3224                /* sign bit is set to zero (positive) */
3225                gen_gvec_fn_2i(andi, ES_64, v1, v2, (1ull << 63) - 1);
3226                break;
3227            }
3228            return DISAS_NEXT;
3229        }
3230        break;
3231    case FPF_EXT:
3232        /* Only a single element. */
3233        break;
3234    default:
3235        gen_program_exception(s, PGM_SPECIFICATION);
3236        return DISAS_NORETURN;
3237    }
3238
3239    /* With a single element, we are only interested in bit 0. */
3240    tmp = tcg_temp_new_i64();
3241    read_vec_element_i64(tmp, v2, 0, ES_64);
3242    switch (m5) {
3243    case 0:
3244        /* sign bit is inverted (complement) */
3245        tcg_gen_xori_i64(tmp, tmp, 1ull << 63);
3246        break;
3247    case 1:
3248        /* sign bit is set to one (negative) */
3249        tcg_gen_ori_i64(tmp, tmp, 1ull << 63);
3250        break;
3251    case 2:
3252        /* sign bit is set to zero (positive) */
3253        tcg_gen_andi_i64(tmp, tmp, (1ull << 63) - 1);
3254        break;
3255    }
3256    write_vec_element_i64(tmp, v1, 0, ES_64);
3257
3258    if (fpf == FPF_EXT) {
3259        read_vec_element_i64(tmp, v2, 1, ES_64);
3260        write_vec_element_i64(tmp, v1, 1, ES_64);
3261    }
3262    return DISAS_NEXT;
3263}
3264
3265static DisasJumpType op_vfsq(DisasContext *s, DisasOps *o)
3266{
3267    const uint8_t fpf = get_field(s, m3);
3268    const uint8_t m4 = get_field(s, m4);
3269    gen_helper_gvec_2_ptr *fn = NULL;
3270
3271    switch (fpf) {
3272    case FPF_SHORT:
3273        if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
3274            fn = gen_helper_gvec_vfsq32;
3275        }
3276        break;
3277    case FPF_LONG:
3278        fn = gen_helper_gvec_vfsq64;
3279        break;
3280    case FPF_EXT:
3281        if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
3282            fn = gen_helper_gvec_vfsq128;
3283        }
3284        break;
3285    default:
3286        break;
3287    }
3288
3289    if (!fn || extract32(m4, 0, 3)) {
3290        gen_program_exception(s, PGM_SPECIFICATION);
3291        return DISAS_NORETURN;
3292    }
3293
3294    gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2), tcg_env, m4, fn);
3295    return DISAS_NEXT;
3296}
3297
3298static DisasJumpType op_vftci(DisasContext *s, DisasOps *o)
3299{
3300    const uint16_t i3 = get_field(s, i3);
3301    const uint8_t fpf = get_field(s, m4);
3302    const uint8_t m5 = get_field(s, m5);
3303    gen_helper_gvec_2_ptr *fn = NULL;
3304
3305    switch (fpf) {
3306    case FPF_SHORT:
3307        if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
3308            fn = gen_helper_gvec_vftci32;
3309        }
3310        break;
3311    case FPF_LONG:
3312        fn = gen_helper_gvec_vftci64;
3313        break;
3314    case FPF_EXT:
3315        if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
3316            fn = gen_helper_gvec_vftci128;
3317        }
3318        break;
3319    default:
3320        break;
3321    }
3322
3323    if (!fn || extract32(m5, 0, 3)) {
3324        gen_program_exception(s, PGM_SPECIFICATION);
3325        return DISAS_NORETURN;
3326    }
3327
3328    gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2), tcg_env,
3329                   deposit32(m5, 4, 12, i3), fn);
3330    set_cc_static(s);
3331    return DISAS_NEXT;
3332}
3333