xref: /openbmc/qemu/target/s390x/tcg/translate.c (revision 3e964275)
1 /*
2  *  S/390 translation
3  *
4  *  Copyright (c) 2009 Ulrich Hecht
5  *  Copyright (c) 2010 Alexander Graf
6  *
7  * This library is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * This library is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19  */
20 
21 /* #define DEBUG_INLINE_BRANCHES */
22 #define S390X_DEBUG_DISAS
23 /* #define S390X_DEBUG_DISAS_VERBOSE */
24 
25 #ifdef S390X_DEBUG_DISAS_VERBOSE
26 #  define LOG_DISAS(...) qemu_log(__VA_ARGS__)
27 #else
28 #  define LOG_DISAS(...) do { } while (0)
29 #endif
30 
31 #include "qemu/osdep.h"
32 #include "cpu.h"
33 #include "s390x-internal.h"
34 #include "exec/exec-all.h"
35 #include "tcg/tcg-op.h"
36 #include "tcg/tcg-op-gvec.h"
37 #include "qemu/log.h"
38 #include "qemu/host-utils.h"
39 #include "exec/helper-proto.h"
40 #include "exec/helper-gen.h"
41 
42 #include "exec/translator.h"
43 #include "exec/log.h"
44 #include "qemu/atomic128.h"
45 
46 #define HELPER_H "helper.h"
47 #include "exec/helper-info.c.inc"
48 #undef  HELPER_H
49 
50 
51 /* Information that (most) every instruction needs to manipulate.  */
52 typedef struct DisasContext DisasContext;
53 typedef struct DisasInsn DisasInsn;
54 typedef struct DisasFields DisasFields;
55 
56 /*
57  * Define a structure to hold the decoded fields.  We'll store each inside
58  * an array indexed by an enum.  In order to conserve memory, we'll arrange
59  * for fields that do not exist at the same time to overlap, thus the "C"
60  * for compact.  For checking purposes there is an "O" for original index
61  * as well that will be applied to availability bitmaps.
62  */
63 
64 enum DisasFieldIndexO {
65     FLD_O_r1,
66     FLD_O_r2,
67     FLD_O_r3,
68     FLD_O_m1,
69     FLD_O_m3,
70     FLD_O_m4,
71     FLD_O_m5,
72     FLD_O_m6,
73     FLD_O_b1,
74     FLD_O_b2,
75     FLD_O_b4,
76     FLD_O_d1,
77     FLD_O_d2,
78     FLD_O_d4,
79     FLD_O_x2,
80     FLD_O_l1,
81     FLD_O_l2,
82     FLD_O_i1,
83     FLD_O_i2,
84     FLD_O_i3,
85     FLD_O_i4,
86     FLD_O_i5,
87     FLD_O_v1,
88     FLD_O_v2,
89     FLD_O_v3,
90     FLD_O_v4,
91 };
92 
93 enum DisasFieldIndexC {
94     FLD_C_r1 = 0,
95     FLD_C_m1 = 0,
96     FLD_C_b1 = 0,
97     FLD_C_i1 = 0,
98     FLD_C_v1 = 0,
99 
100     FLD_C_r2 = 1,
101     FLD_C_b2 = 1,
102     FLD_C_i2 = 1,
103 
104     FLD_C_r3 = 2,
105     FLD_C_m3 = 2,
106     FLD_C_i3 = 2,
107     FLD_C_v3 = 2,
108 
109     FLD_C_m4 = 3,
110     FLD_C_b4 = 3,
111     FLD_C_i4 = 3,
112     FLD_C_l1 = 3,
113     FLD_C_v4 = 3,
114 
115     FLD_C_i5 = 4,
116     FLD_C_d1 = 4,
117     FLD_C_m5 = 4,
118 
119     FLD_C_d2 = 5,
120     FLD_C_m6 = 5,
121 
122     FLD_C_d4 = 6,
123     FLD_C_x2 = 6,
124     FLD_C_l2 = 6,
125     FLD_C_v2 = 6,
126 
127     NUM_C_FIELD = 7
128 };
129 
130 struct DisasFields {
131     uint64_t raw_insn;
132     unsigned op:8;
133     unsigned op2:8;
134     unsigned presentC:16;
135     unsigned int presentO;
136     int c[NUM_C_FIELD];
137 };
138 
139 struct DisasContext {
140     DisasContextBase base;
141     const DisasInsn *insn;
142     DisasFields fields;
143     uint64_t ex_value;
144     /*
145      * During translate_one(), pc_tmp is used to determine the instruction
146      * to be executed after base.pc_next - e.g. next sequential instruction
147      * or a branch target.
148      */
149     uint64_t pc_tmp;
150     uint32_t ilen;
151     enum cc_op cc_op;
152     bool exit_to_mainloop;
153 };
154 
155 /* Information carried about a condition to be evaluated.  */
156 typedef struct {
157     TCGCond cond:8;
158     bool is_64;
159     union {
160         struct { TCGv_i64 a, b; } s64;
161         struct { TCGv_i32 a, b; } s32;
162     } u;
163 } DisasCompare;
164 
165 #ifdef DEBUG_INLINE_BRANCHES
166 static uint64_t inline_branch_hit[CC_OP_MAX];
167 static uint64_t inline_branch_miss[CC_OP_MAX];
168 #endif
169 
170 static void pc_to_link_info(TCGv_i64 out, DisasContext *s, uint64_t pc)
171 {
172     if (s->base.tb->flags & FLAG_MASK_32) {
173         if (s->base.tb->flags & FLAG_MASK_64) {
174             tcg_gen_movi_i64(out, pc);
175             return;
176         }
177         pc |= 0x80000000;
178     }
179     assert(!(s->base.tb->flags & FLAG_MASK_64));
180     tcg_gen_deposit_i64(out, out, tcg_constant_i64(pc), 0, 32);
181 }
182 
183 static TCGv_i64 psw_addr;
184 static TCGv_i64 psw_mask;
185 static TCGv_i64 gbea;
186 
187 static TCGv_i32 cc_op;
188 static TCGv_i64 cc_src;
189 static TCGv_i64 cc_dst;
190 static TCGv_i64 cc_vr;
191 
192 static char cpu_reg_names[16][4];
193 static TCGv_i64 regs[16];
194 
195 void s390x_translate_init(void)
196 {
197     int i;
198 
199     psw_addr = tcg_global_mem_new_i64(tcg_env,
200                                       offsetof(CPUS390XState, psw.addr),
201                                       "psw_addr");
202     psw_mask = tcg_global_mem_new_i64(tcg_env,
203                                       offsetof(CPUS390XState, psw.mask),
204                                       "psw_mask");
205     gbea = tcg_global_mem_new_i64(tcg_env,
206                                   offsetof(CPUS390XState, gbea),
207                                   "gbea");
208 
209     cc_op = tcg_global_mem_new_i32(tcg_env, offsetof(CPUS390XState, cc_op),
210                                    "cc_op");
211     cc_src = tcg_global_mem_new_i64(tcg_env, offsetof(CPUS390XState, cc_src),
212                                     "cc_src");
213     cc_dst = tcg_global_mem_new_i64(tcg_env, offsetof(CPUS390XState, cc_dst),
214                                     "cc_dst");
215     cc_vr = tcg_global_mem_new_i64(tcg_env, offsetof(CPUS390XState, cc_vr),
216                                    "cc_vr");
217 
218     for (i = 0; i < 16; i++) {
219         snprintf(cpu_reg_names[i], sizeof(cpu_reg_names[0]), "r%d", i);
220         regs[i] = tcg_global_mem_new(tcg_env,
221                                      offsetof(CPUS390XState, regs[i]),
222                                      cpu_reg_names[i]);
223     }
224 }
225 
226 static inline int vec_full_reg_offset(uint8_t reg)
227 {
228     g_assert(reg < 32);
229     return offsetof(CPUS390XState, vregs[reg][0]);
230 }
231 
232 static inline int vec_reg_offset(uint8_t reg, uint8_t enr, MemOp es)
233 {
234     /* Convert element size (es) - e.g. MO_8 - to bytes */
235     const uint8_t bytes = 1 << es;
236     int offs = enr * bytes;
237 
238     /*
239      * vregs[n][0] is the lowest 8 byte and vregs[n][1] the highest 8 byte
240      * of the 16 byte vector, on both, little and big endian systems.
241      *
242      * Big Endian (target/possible host)
243      * B:  [ 0][ 1][ 2][ 3][ 4][ 5][ 6][ 7] - [ 8][ 9][10][11][12][13][14][15]
244      * HW: [     0][     1][     2][     3] - [     4][     5][     6][     7]
245      * W:  [             0][             1] - [             2][             3]
246      * DW: [                             0] - [                             1]
247      *
248      * Little Endian (possible host)
249      * B:  [ 7][ 6][ 5][ 4][ 3][ 2][ 1][ 0] - [15][14][13][12][11][10][ 9][ 8]
250      * HW: [     3][     2][     1][     0] - [     7][     6][     5][     4]
251      * W:  [             1][             0] - [             3][             2]
252      * DW: [                             0] - [                             1]
253      *
254      * For 16 byte elements, the two 8 byte halves will not form a host
255      * int128 if the host is little endian, since they're in the wrong order.
256      * Some operations (e.g. xor) do not care. For operations like addition,
257      * the two 8 byte elements have to be loaded separately. Let's force all
258      * 16 byte operations to handle it in a special way.
259      */
260     g_assert(es <= MO_64);
261 #if !HOST_BIG_ENDIAN
262     offs ^= (8 - bytes);
263 #endif
264     return offs + vec_full_reg_offset(reg);
265 }
266 
267 static inline int freg64_offset(uint8_t reg)
268 {
269     g_assert(reg < 16);
270     return vec_reg_offset(reg, 0, MO_64);
271 }
272 
273 static inline int freg32_offset(uint8_t reg)
274 {
275     g_assert(reg < 16);
276     return vec_reg_offset(reg, 0, MO_32);
277 }
278 
279 static TCGv_i64 load_reg(int reg)
280 {
281     TCGv_i64 r = tcg_temp_new_i64();
282     tcg_gen_mov_i64(r, regs[reg]);
283     return r;
284 }
285 
286 static TCGv_i64 load_freg(int reg)
287 {
288     TCGv_i64 r = tcg_temp_new_i64();
289 
290     tcg_gen_ld_i64(r, tcg_env, freg64_offset(reg));
291     return r;
292 }
293 
294 static TCGv_i64 load_freg32_i64(int reg)
295 {
296     TCGv_i64 r = tcg_temp_new_i64();
297 
298     tcg_gen_ld32u_i64(r, tcg_env, freg32_offset(reg));
299     return r;
300 }
301 
302 static TCGv_i128 load_freg_128(int reg)
303 {
304     TCGv_i64 h = load_freg(reg);
305     TCGv_i64 l = load_freg(reg + 2);
306     TCGv_i128 r = tcg_temp_new_i128();
307 
308     tcg_gen_concat_i64_i128(r, l, h);
309     return r;
310 }
311 
312 static void store_reg(int reg, TCGv_i64 v)
313 {
314     tcg_gen_mov_i64(regs[reg], v);
315 }
316 
317 static void store_freg(int reg, TCGv_i64 v)
318 {
319     tcg_gen_st_i64(v, tcg_env, freg64_offset(reg));
320 }
321 
322 static void store_reg32_i64(int reg, TCGv_i64 v)
323 {
324     /* 32 bit register writes keep the upper half */
325     tcg_gen_deposit_i64(regs[reg], regs[reg], v, 0, 32);
326 }
327 
328 static void store_reg32h_i64(int reg, TCGv_i64 v)
329 {
330     tcg_gen_deposit_i64(regs[reg], regs[reg], v, 32, 32);
331 }
332 
333 static void store_freg32_i64(int reg, TCGv_i64 v)
334 {
335     tcg_gen_st32_i64(v, tcg_env, freg32_offset(reg));
336 }
337 
338 static void update_psw_addr(DisasContext *s)
339 {
340     /* psw.addr */
341     tcg_gen_movi_i64(psw_addr, s->base.pc_next);
342 }
343 
344 static void per_branch(DisasContext *s, TCGv_i64 dest)
345 {
346 #ifndef CONFIG_USER_ONLY
347     if (s->base.tb->flags & FLAG_MASK_PER_BRANCH) {
348         gen_helper_per_branch(tcg_env, dest, tcg_constant_i32(s->ilen));
349     }
350 #endif
351 }
352 
353 static void per_breaking_event(DisasContext *s)
354 {
355     tcg_gen_movi_i64(gbea, s->base.pc_next);
356 }
357 
358 static void update_cc_op(DisasContext *s)
359 {
360     if (s->cc_op != CC_OP_DYNAMIC && s->cc_op != CC_OP_STATIC) {
361         tcg_gen_movi_i32(cc_op, s->cc_op);
362     }
363 }
364 
365 static inline uint64_t ld_code2(CPUS390XState *env, DisasContext *s,
366                                 uint64_t pc)
367 {
368     return (uint64_t)translator_lduw(env, &s->base, pc);
369 }
370 
371 static inline uint64_t ld_code4(CPUS390XState *env, DisasContext *s,
372                                 uint64_t pc)
373 {
374     return (uint64_t)(uint32_t)translator_ldl(env, &s->base, pc);
375 }
376 
377 static int get_mem_index(DisasContext *s)
378 {
379 #ifdef CONFIG_USER_ONLY
380     return MMU_USER_IDX;
381 #else
382     if (!(s->base.tb->flags & FLAG_MASK_DAT)) {
383         return MMU_REAL_IDX;
384     }
385 
386     switch (s->base.tb->flags & FLAG_MASK_ASC) {
387     case PSW_ASC_PRIMARY >> FLAG_MASK_PSW_SHIFT:
388         return MMU_PRIMARY_IDX;
389     case PSW_ASC_SECONDARY >> FLAG_MASK_PSW_SHIFT:
390         return MMU_SECONDARY_IDX;
391     case PSW_ASC_HOME >> FLAG_MASK_PSW_SHIFT:
392         return MMU_HOME_IDX;
393     default:
394         g_assert_not_reached();
395         break;
396     }
397 #endif
398 }
399 
400 static void gen_exception(int excp)
401 {
402     gen_helper_exception(tcg_env, tcg_constant_i32(excp));
403 }
404 
405 static void gen_program_exception(DisasContext *s, int code)
406 {
407     /* Remember what pgm exception this was.  */
408     tcg_gen_st_i32(tcg_constant_i32(code), tcg_env,
409                    offsetof(CPUS390XState, int_pgm_code));
410 
411     tcg_gen_st_i32(tcg_constant_i32(s->ilen), tcg_env,
412                    offsetof(CPUS390XState, int_pgm_ilen));
413 
414     /* update the psw */
415     update_psw_addr(s);
416 
417     /* Save off cc.  */
418     update_cc_op(s);
419 
420     /* Trigger exception.  */
421     gen_exception(EXCP_PGM);
422 }
423 
424 static inline void gen_illegal_opcode(DisasContext *s)
425 {
426     gen_program_exception(s, PGM_OPERATION);
427 }
428 
429 static inline void gen_data_exception(uint8_t dxc)
430 {
431     gen_helper_data_exception(tcg_env, tcg_constant_i32(dxc));
432 }
433 
434 static inline void gen_trap(DisasContext *s)
435 {
436     /* Set DXC to 0xff */
437     gen_data_exception(0xff);
438 }
439 
440 static void gen_addi_and_wrap_i64(DisasContext *s, TCGv_i64 dst, TCGv_i64 src,
441                                   int64_t imm)
442 {
443     tcg_gen_addi_i64(dst, src, imm);
444     if (!(s->base.tb->flags & FLAG_MASK_64)) {
445         if (s->base.tb->flags & FLAG_MASK_32) {
446             tcg_gen_andi_i64(dst, dst, 0x7fffffff);
447         } else {
448             tcg_gen_andi_i64(dst, dst, 0x00ffffff);
449         }
450     }
451 }
452 
453 static TCGv_i64 get_address(DisasContext *s, int x2, int b2, int d2)
454 {
455     TCGv_i64 tmp = tcg_temp_new_i64();
456 
457     /*
458      * Note that d2 is limited to 20 bits, signed.  If we crop negative
459      * displacements early we create larger immediate addends.
460      */
461     if (b2 && x2) {
462         tcg_gen_add_i64(tmp, regs[b2], regs[x2]);
463         gen_addi_and_wrap_i64(s, tmp, tmp, d2);
464     } else if (b2) {
465         gen_addi_and_wrap_i64(s, tmp, regs[b2], d2);
466     } else if (x2) {
467         gen_addi_and_wrap_i64(s, tmp, regs[x2], d2);
468     } else if (!(s->base.tb->flags & FLAG_MASK_64)) {
469         if (s->base.tb->flags & FLAG_MASK_32) {
470             tcg_gen_movi_i64(tmp, d2 & 0x7fffffff);
471         } else {
472             tcg_gen_movi_i64(tmp, d2 & 0x00ffffff);
473         }
474     } else {
475         tcg_gen_movi_i64(tmp, d2);
476     }
477 
478     return tmp;
479 }
480 
481 static inline bool live_cc_data(DisasContext *s)
482 {
483     return (s->cc_op != CC_OP_DYNAMIC
484             && s->cc_op != CC_OP_STATIC
485             && s->cc_op > 3);
486 }
487 
488 static inline void gen_op_movi_cc(DisasContext *s, uint32_t val)
489 {
490     if (live_cc_data(s)) {
491         tcg_gen_discard_i64(cc_src);
492         tcg_gen_discard_i64(cc_dst);
493         tcg_gen_discard_i64(cc_vr);
494     }
495     s->cc_op = CC_OP_CONST0 + val;
496 }
497 
498 static void gen_op_update1_cc_i64(DisasContext *s, enum cc_op op, TCGv_i64 dst)
499 {
500     if (live_cc_data(s)) {
501         tcg_gen_discard_i64(cc_src);
502         tcg_gen_discard_i64(cc_vr);
503     }
504     tcg_gen_mov_i64(cc_dst, dst);
505     s->cc_op = op;
506 }
507 
508 static void gen_op_update2_cc_i64(DisasContext *s, enum cc_op op, TCGv_i64 src,
509                                   TCGv_i64 dst)
510 {
511     if (live_cc_data(s)) {
512         tcg_gen_discard_i64(cc_vr);
513     }
514     tcg_gen_mov_i64(cc_src, src);
515     tcg_gen_mov_i64(cc_dst, dst);
516     s->cc_op = op;
517 }
518 
519 static void gen_op_update3_cc_i64(DisasContext *s, enum cc_op op, TCGv_i64 src,
520                                   TCGv_i64 dst, TCGv_i64 vr)
521 {
522     tcg_gen_mov_i64(cc_src, src);
523     tcg_gen_mov_i64(cc_dst, dst);
524     tcg_gen_mov_i64(cc_vr, vr);
525     s->cc_op = op;
526 }
527 
528 static void set_cc_nz_u64(DisasContext *s, TCGv_i64 val)
529 {
530     gen_op_update1_cc_i64(s, CC_OP_NZ, val);
531 }
532 
533 /* CC value is in env->cc_op */
534 static void set_cc_static(DisasContext *s)
535 {
536     if (live_cc_data(s)) {
537         tcg_gen_discard_i64(cc_src);
538         tcg_gen_discard_i64(cc_dst);
539         tcg_gen_discard_i64(cc_vr);
540     }
541     s->cc_op = CC_OP_STATIC;
542 }
543 
544 /* calculates cc into cc_op */
545 static void gen_op_calc_cc(DisasContext *s)
546 {
547     TCGv_i32 local_cc_op = NULL;
548     TCGv_i64 dummy = NULL;
549 
550     switch (s->cc_op) {
551     default:
552         dummy = tcg_constant_i64(0);
553         /* FALLTHRU */
554     case CC_OP_ADD_64:
555     case CC_OP_SUB_64:
556     case CC_OP_ADD_32:
557     case CC_OP_SUB_32:
558         local_cc_op = tcg_constant_i32(s->cc_op);
559         break;
560     case CC_OP_CONST0:
561     case CC_OP_CONST1:
562     case CC_OP_CONST2:
563     case CC_OP_CONST3:
564     case CC_OP_STATIC:
565     case CC_OP_DYNAMIC:
566         break;
567     }
568 
569     switch (s->cc_op) {
570     case CC_OP_CONST0:
571     case CC_OP_CONST1:
572     case CC_OP_CONST2:
573     case CC_OP_CONST3:
574         /* s->cc_op is the cc value */
575         tcg_gen_movi_i32(cc_op, s->cc_op - CC_OP_CONST0);
576         break;
577     case CC_OP_STATIC:
578         /* env->cc_op already is the cc value */
579         break;
580     case CC_OP_NZ:
581         tcg_gen_setcondi_i64(TCG_COND_NE, cc_dst, cc_dst, 0);
582         tcg_gen_extrl_i64_i32(cc_op, cc_dst);
583         break;
584     case CC_OP_ABS_64:
585     case CC_OP_NABS_64:
586     case CC_OP_ABS_32:
587     case CC_OP_NABS_32:
588     case CC_OP_LTGT0_32:
589     case CC_OP_LTGT0_64:
590     case CC_OP_COMP_32:
591     case CC_OP_COMP_64:
592     case CC_OP_NZ_F32:
593     case CC_OP_NZ_F64:
594     case CC_OP_FLOGR:
595     case CC_OP_LCBB:
596     case CC_OP_MULS_32:
597         /* 1 argument */
598         gen_helper_calc_cc(cc_op, tcg_env, local_cc_op, dummy, cc_dst, dummy);
599         break;
600     case CC_OP_ADDU:
601     case CC_OP_ICM:
602     case CC_OP_LTGT_32:
603     case CC_OP_LTGT_64:
604     case CC_OP_LTUGTU_32:
605     case CC_OP_LTUGTU_64:
606     case CC_OP_TM_32:
607     case CC_OP_TM_64:
608     case CC_OP_SLA:
609     case CC_OP_SUBU:
610     case CC_OP_NZ_F128:
611     case CC_OP_VC:
612     case CC_OP_MULS_64:
613         /* 2 arguments */
614         gen_helper_calc_cc(cc_op, tcg_env, local_cc_op, cc_src, cc_dst, dummy);
615         break;
616     case CC_OP_ADD_64:
617     case CC_OP_SUB_64:
618     case CC_OP_ADD_32:
619     case CC_OP_SUB_32:
620         /* 3 arguments */
621         gen_helper_calc_cc(cc_op, tcg_env, local_cc_op, cc_src, cc_dst, cc_vr);
622         break;
623     case CC_OP_DYNAMIC:
624         /* unknown operation - assume 3 arguments and cc_op in env */
625         gen_helper_calc_cc(cc_op, tcg_env, cc_op, cc_src, cc_dst, cc_vr);
626         break;
627     default:
628         g_assert_not_reached();
629     }
630 
631     /* We now have cc in cc_op as constant */
632     set_cc_static(s);
633 }
634 
635 static bool use_goto_tb(DisasContext *s, uint64_t dest)
636 {
637     return translator_use_goto_tb(&s->base, dest);
638 }
639 
640 static void account_noninline_branch(DisasContext *s, int cc_op)
641 {
642 #ifdef DEBUG_INLINE_BRANCHES
643     inline_branch_miss[cc_op]++;
644 #endif
645 }
646 
647 static void account_inline_branch(DisasContext *s, int cc_op)
648 {
649 #ifdef DEBUG_INLINE_BRANCHES
650     inline_branch_hit[cc_op]++;
651 #endif
652 }
653 
654 /* Table of mask values to comparison codes, given a comparison as input.
655    For such, CC=3 should not be possible.  */
656 static const TCGCond ltgt_cond[16] = {
657     TCG_COND_NEVER,  TCG_COND_NEVER,     /*    |    |    | x */
658     TCG_COND_GT,     TCG_COND_GT,        /*    |    | GT | x */
659     TCG_COND_LT,     TCG_COND_LT,        /*    | LT |    | x */
660     TCG_COND_NE,     TCG_COND_NE,        /*    | LT | GT | x */
661     TCG_COND_EQ,     TCG_COND_EQ,        /* EQ |    |    | x */
662     TCG_COND_GE,     TCG_COND_GE,        /* EQ |    | GT | x */
663     TCG_COND_LE,     TCG_COND_LE,        /* EQ | LT |    | x */
664     TCG_COND_ALWAYS, TCG_COND_ALWAYS,    /* EQ | LT | GT | x */
665 };
666 
667 /* Table of mask values to comparison codes, given a logic op as input.
668    For such, only CC=0 and CC=1 should be possible.  */
669 static const TCGCond nz_cond[16] = {
670     TCG_COND_NEVER, TCG_COND_NEVER,      /*    |    | x | x */
671     TCG_COND_NEVER, TCG_COND_NEVER,
672     TCG_COND_NE, TCG_COND_NE,            /*    | NE | x | x */
673     TCG_COND_NE, TCG_COND_NE,
674     TCG_COND_EQ, TCG_COND_EQ,            /* EQ |    | x | x */
675     TCG_COND_EQ, TCG_COND_EQ,
676     TCG_COND_ALWAYS, TCG_COND_ALWAYS,    /* EQ | NE | x | x */
677     TCG_COND_ALWAYS, TCG_COND_ALWAYS,
678 };
679 
680 /* Interpret MASK in terms of S->CC_OP, and fill in C with all the
681    details required to generate a TCG comparison.  */
682 static void disas_jcc(DisasContext *s, DisasCompare *c, uint32_t mask)
683 {
684     TCGCond cond;
685     enum cc_op old_cc_op = s->cc_op;
686 
687     if (mask == 15 || mask == 0) {
688         c->cond = (mask ? TCG_COND_ALWAYS : TCG_COND_NEVER);
689         c->u.s32.a = cc_op;
690         c->u.s32.b = cc_op;
691         c->is_64 = false;
692         return;
693     }
694 
695     /* Find the TCG condition for the mask + cc op.  */
696     switch (old_cc_op) {
697     case CC_OP_LTGT0_32:
698     case CC_OP_LTGT0_64:
699     case CC_OP_LTGT_32:
700     case CC_OP_LTGT_64:
701         cond = ltgt_cond[mask];
702         if (cond == TCG_COND_NEVER) {
703             goto do_dynamic;
704         }
705         account_inline_branch(s, old_cc_op);
706         break;
707 
708     case CC_OP_LTUGTU_32:
709     case CC_OP_LTUGTU_64:
710         cond = tcg_unsigned_cond(ltgt_cond[mask]);
711         if (cond == TCG_COND_NEVER) {
712             goto do_dynamic;
713         }
714         account_inline_branch(s, old_cc_op);
715         break;
716 
717     case CC_OP_NZ:
718         cond = nz_cond[mask];
719         if (cond == TCG_COND_NEVER) {
720             goto do_dynamic;
721         }
722         account_inline_branch(s, old_cc_op);
723         break;
724 
725     case CC_OP_TM_32:
726     case CC_OP_TM_64:
727         switch (mask) {
728         case 8:
729             cond = TCG_COND_TSTEQ;
730             break;
731         case 4 | 2 | 1:
732             cond = TCG_COND_TSTNE;
733             break;
734         default:
735             goto do_dynamic;
736         }
737         account_inline_branch(s, old_cc_op);
738         break;
739 
740     case CC_OP_ICM:
741         switch (mask) {
742         case 8:
743             cond = TCG_COND_TSTEQ;
744             break;
745         case 4 | 2 | 1:
746         case 4 | 2:
747             cond = TCG_COND_TSTNE;
748             break;
749         default:
750             goto do_dynamic;
751         }
752         account_inline_branch(s, old_cc_op);
753         break;
754 
755     case CC_OP_FLOGR:
756         switch (mask & 0xa) {
757         case 8: /* src == 0 -> no one bit found */
758             cond = TCG_COND_EQ;
759             break;
760         case 2: /* src != 0 -> one bit found */
761             cond = TCG_COND_NE;
762             break;
763         default:
764             goto do_dynamic;
765         }
766         account_inline_branch(s, old_cc_op);
767         break;
768 
769     case CC_OP_ADDU:
770     case CC_OP_SUBU:
771         switch (mask) {
772         case 8 | 2: /* result == 0 */
773             cond = TCG_COND_EQ;
774             break;
775         case 4 | 1: /* result != 0 */
776             cond = TCG_COND_NE;
777             break;
778         case 8 | 4: /* !carry (borrow) */
779             cond = old_cc_op == CC_OP_ADDU ? TCG_COND_EQ : TCG_COND_NE;
780             break;
781         case 2 | 1: /* carry (!borrow) */
782             cond = old_cc_op == CC_OP_ADDU ? TCG_COND_NE : TCG_COND_EQ;
783             break;
784         default:
785             goto do_dynamic;
786         }
787         account_inline_branch(s, old_cc_op);
788         break;
789 
790     default:
791     do_dynamic:
792         /* Calculate cc value.  */
793         gen_op_calc_cc(s);
794         /* FALLTHRU */
795 
796     case CC_OP_STATIC:
797         /* Jump based on CC.  We'll load up the real cond below;
798            the assignment here merely avoids a compiler warning.  */
799         account_noninline_branch(s, old_cc_op);
800         old_cc_op = CC_OP_STATIC;
801         cond = TCG_COND_NEVER;
802         break;
803     }
804 
805     /* Load up the arguments of the comparison.  */
806     c->is_64 = true;
807     switch (old_cc_op) {
808     case CC_OP_LTGT0_32:
809         c->is_64 = false;
810         c->u.s32.a = tcg_temp_new_i32();
811         tcg_gen_extrl_i64_i32(c->u.s32.a, cc_dst);
812         c->u.s32.b = tcg_constant_i32(0);
813         break;
814     case CC_OP_LTGT_32:
815     case CC_OP_LTUGTU_32:
816         c->is_64 = false;
817         c->u.s32.a = tcg_temp_new_i32();
818         tcg_gen_extrl_i64_i32(c->u.s32.a, cc_src);
819         c->u.s32.b = tcg_temp_new_i32();
820         tcg_gen_extrl_i64_i32(c->u.s32.b, cc_dst);
821         break;
822 
823     case CC_OP_LTGT0_64:
824     case CC_OP_NZ:
825     case CC_OP_FLOGR:
826         c->u.s64.a = cc_dst;
827         c->u.s64.b = tcg_constant_i64(0);
828         break;
829 
830     case CC_OP_LTGT_64:
831     case CC_OP_LTUGTU_64:
832     case CC_OP_TM_32:
833     case CC_OP_TM_64:
834     case CC_OP_ICM:
835         c->u.s64.a = cc_src;
836         c->u.s64.b = cc_dst;
837         break;
838 
839     case CC_OP_ADDU:
840     case CC_OP_SUBU:
841         c->is_64 = true;
842         c->u.s64.b = tcg_constant_i64(0);
843         switch (mask) {
844         case 8 | 2:
845         case 4 | 1: /* result */
846             c->u.s64.a = cc_dst;
847             break;
848         case 8 | 4:
849         case 2 | 1: /* carry */
850             c->u.s64.a = cc_src;
851             break;
852         default:
853             g_assert_not_reached();
854         }
855         break;
856 
857     case CC_OP_STATIC:
858         c->is_64 = false;
859         c->u.s32.a = cc_op;
860 
861         /* Fold half of the cases using bit 3 to invert. */
862         switch (mask & 8 ? mask ^ 0xf : mask) {
863         case 0x1: /* cc == 3 */
864             cond = TCG_COND_EQ;
865             c->u.s32.b = tcg_constant_i32(3);
866             break;
867         case 0x2: /* cc == 2 */
868             cond = TCG_COND_EQ;
869             c->u.s32.b = tcg_constant_i32(2);
870             break;
871         case 0x4: /* cc == 1 */
872             cond = TCG_COND_EQ;
873             c->u.s32.b = tcg_constant_i32(1);
874             break;
875         case 0x2 | 0x1: /* cc == 2 || cc == 3 => cc > 1 */
876             cond = TCG_COND_GTU;
877             c->u.s32.b = tcg_constant_i32(1);
878             break;
879         case 0x4 | 0x1: /* cc == 1 || cc == 3 => (cc & 1) != 0 */
880             cond = TCG_COND_TSTNE;
881             c->u.s32.b = tcg_constant_i32(1);
882             break;
883         case 0x4 | 0x2: /* cc == 1 || cc == 2 => (cc - 1) <= 1 */
884             cond = TCG_COND_LEU;
885             c->u.s32.a = tcg_temp_new_i32();
886             c->u.s32.b = tcg_constant_i32(1);
887             tcg_gen_addi_i32(c->u.s32.a, cc_op, -1);
888             break;
889         case 0x4 | 0x2 | 0x1: /* cc != 0 */
890             cond = TCG_COND_NE;
891             c->u.s32.b = tcg_constant_i32(0);
892             break;
893         default:
894             /* case 0: never, handled above. */
895             g_assert_not_reached();
896         }
897         if (mask & 8) {
898             cond = tcg_invert_cond(cond);
899         }
900         break;
901 
902     default:
903         abort();
904     }
905     c->cond = cond;
906 }
907 
908 /* ====================================================================== */
909 /* Define the insn format enumeration.  */
910 #define F0(N)                         FMT_##N,
911 #define F1(N, X1)                     F0(N)
912 #define F2(N, X1, X2)                 F0(N)
913 #define F3(N, X1, X2, X3)             F0(N)
914 #define F4(N, X1, X2, X3, X4)         F0(N)
915 #define F5(N, X1, X2, X3, X4, X5)     F0(N)
916 #define F6(N, X1, X2, X3, X4, X5, X6) F0(N)
917 
918 typedef enum {
919 #include "insn-format.h.inc"
920 } DisasFormat;
921 
922 #undef F0
923 #undef F1
924 #undef F2
925 #undef F3
926 #undef F4
927 #undef F5
928 #undef F6
929 
930 /* This is the way fields are to be accessed out of DisasFields.  */
931 #define have_field(S, F)  have_field1((S), FLD_O_##F)
932 #define get_field(S, F)   get_field1((S), FLD_O_##F, FLD_C_##F)
933 
934 static bool have_field1(const DisasContext *s, enum DisasFieldIndexO c)
935 {
936     return (s->fields.presentO >> c) & 1;
937 }
938 
939 static int get_field1(const DisasContext *s, enum DisasFieldIndexO o,
940                       enum DisasFieldIndexC c)
941 {
942     assert(have_field1(s, o));
943     return s->fields.c[c];
944 }
945 
946 /* Describe the layout of each field in each format.  */
947 typedef struct DisasField {
948     unsigned int beg:8;
949     unsigned int size:8;
950     unsigned int type:2;
951     unsigned int indexC:6;
952     enum DisasFieldIndexO indexO:8;
953 } DisasField;
954 
955 typedef struct DisasFormatInfo {
956     DisasField op[NUM_C_FIELD];
957 } DisasFormatInfo;
958 
959 #define R(N, B)       {  B,  4, 0, FLD_C_r##N, FLD_O_r##N }
960 #define M(N, B)       {  B,  4, 0, FLD_C_m##N, FLD_O_m##N }
961 #define V(N, B)       {  B,  4, 3, FLD_C_v##N, FLD_O_v##N }
962 #define BD(N, BB, BD) { BB,  4, 0, FLD_C_b##N, FLD_O_b##N }, \
963                       { BD, 12, 0, FLD_C_d##N, FLD_O_d##N }
964 #define BXD(N)        { 16,  4, 0, FLD_C_b##N, FLD_O_b##N }, \
965                       { 12,  4, 0, FLD_C_x##N, FLD_O_x##N }, \
966                       { 20, 12, 0, FLD_C_d##N, FLD_O_d##N }
967 #define BDL(N)        { 16,  4, 0, FLD_C_b##N, FLD_O_b##N }, \
968                       { 20, 20, 2, FLD_C_d##N, FLD_O_d##N }
969 #define BXDL(N)       { 16,  4, 0, FLD_C_b##N, FLD_O_b##N }, \
970                       { 12,  4, 0, FLD_C_x##N, FLD_O_x##N }, \
971                       { 20, 20, 2, FLD_C_d##N, FLD_O_d##N }
972 #define I(N, B, S)    {  B,  S, 1, FLD_C_i##N, FLD_O_i##N }
973 #define L(N, B, S)    {  B,  S, 0, FLD_C_l##N, FLD_O_l##N }
974 
975 #define F0(N)                     { { } },
976 #define F1(N, X1)                 { { X1 } },
977 #define F2(N, X1, X2)             { { X1, X2 } },
978 #define F3(N, X1, X2, X3)         { { X1, X2, X3 } },
979 #define F4(N, X1, X2, X3, X4)     { { X1, X2, X3, X4 } },
980 #define F5(N, X1, X2, X3, X4, X5) { { X1, X2, X3, X4, X5 } },
981 #define F6(N, X1, X2, X3, X4, X5, X6)       { { X1, X2, X3, X4, X5, X6 } },
982 
983 static const DisasFormatInfo format_info[] = {
984 #include "insn-format.h.inc"
985 };
986 
987 #undef F0
988 #undef F1
989 #undef F2
990 #undef F3
991 #undef F4
992 #undef F5
993 #undef F6
994 #undef R
995 #undef M
996 #undef V
997 #undef BD
998 #undef BXD
999 #undef BDL
1000 #undef BXDL
1001 #undef I
1002 #undef L
1003 
1004 /* Generally, we'll extract operands into this structures, operate upon
1005    them, and store them back.  See the "in1", "in2", "prep", "wout" sets
1006    of routines below for more details.  */
1007 typedef struct {
1008     TCGv_i64 out, out2, in1, in2;
1009     TCGv_i64 addr1;
1010     TCGv_i128 out_128, in1_128, in2_128;
1011 } DisasOps;
1012 
1013 /* Instructions can place constraints on their operands, raising specification
1014    exceptions if they are violated.  To make this easy to automate, each "in1",
1015    "in2", "prep", "wout" helper will have a SPEC_<name> define that equals one
1016    of the following, or 0.  To make this easy to document, we'll put the
1017    SPEC_<name> defines next to <name>.  */
1018 
1019 #define SPEC_r1_even    1
1020 #define SPEC_r2_even    2
1021 #define SPEC_r3_even    4
1022 #define SPEC_r1_f128    8
1023 #define SPEC_r2_f128    16
1024 
1025 /* Return values from translate_one, indicating the state of the TB.  */
1026 
1027 /* We are not using a goto_tb (for whatever reason), but have updated
1028    the PC (for whatever reason), so there's no need to do it again on
1029    exiting the TB.  */
1030 #define DISAS_PC_UPDATED        DISAS_TARGET_0
1031 
1032 /* We have updated the PC and CC values.  */
1033 #define DISAS_PC_CC_UPDATED     DISAS_TARGET_2
1034 
1035 
1036 /* Instruction flags */
1037 #define IF_AFP1     0x0001      /* r1 is a fp reg for HFP/FPS instructions */
1038 #define IF_AFP2     0x0002      /* r2 is a fp reg for HFP/FPS instructions */
1039 #define IF_AFP3     0x0004      /* r3 is a fp reg for HFP/FPS instructions */
1040 #define IF_BFP      0x0008      /* binary floating point instruction */
1041 #define IF_DFP      0x0010      /* decimal floating point instruction */
1042 #define IF_PRIV     0x0020      /* privileged instruction */
1043 #define IF_VEC      0x0040      /* vector instruction */
1044 #define IF_IO       0x0080      /* input/output instruction */
1045 
1046 struct DisasInsn {
1047     unsigned opc:16;
1048     unsigned flags:16;
1049     DisasFormat fmt:8;
1050     unsigned fac:8;
1051     unsigned spec:8;
1052 
1053     const char *name;
1054 
1055     /* Pre-process arguments before HELP_OP.  */
1056     void (*help_in1)(DisasContext *, DisasOps *);
1057     void (*help_in2)(DisasContext *, DisasOps *);
1058     void (*help_prep)(DisasContext *, DisasOps *);
1059 
1060     /*
1061      * Post-process output after HELP_OP.
1062      * Note that these are not called if HELP_OP returns DISAS_NORETURN.
1063      */
1064     void (*help_wout)(DisasContext *, DisasOps *);
1065     void (*help_cout)(DisasContext *, DisasOps *);
1066 
1067     /* Implement the operation itself.  */
1068     DisasJumpType (*help_op)(DisasContext *, DisasOps *);
1069 
1070     uint64_t data;
1071 };
1072 
1073 /* ====================================================================== */
1074 /* Miscellaneous helpers, used by several operations.  */
1075 
1076 static DisasJumpType help_goto_direct(DisasContext *s, uint64_t dest)
1077 {
1078     update_cc_op(s);
1079     per_breaking_event(s);
1080     per_branch(s, tcg_constant_i64(dest));
1081 
1082     if (dest == s->pc_tmp) {
1083         return DISAS_NEXT;
1084     }
1085     if (use_goto_tb(s, dest)) {
1086         tcg_gen_goto_tb(0);
1087         tcg_gen_movi_i64(psw_addr, dest);
1088         tcg_gen_exit_tb(s->base.tb, 0);
1089         return DISAS_NORETURN;
1090     } else {
1091         tcg_gen_movi_i64(psw_addr, dest);
1092         return DISAS_PC_CC_UPDATED;
1093     }
1094 }
1095 
1096 static DisasJumpType help_goto_indirect(DisasContext *s, TCGv_i64 dest)
1097 {
1098     update_cc_op(s);
1099     per_breaking_event(s);
1100     tcg_gen_mov_i64(psw_addr, dest);
1101     per_branch(s, psw_addr);
1102     return DISAS_PC_CC_UPDATED;
1103 }
1104 
1105 static DisasJumpType help_branch(DisasContext *s, DisasCompare *c,
1106                                  bool is_imm, int imm, TCGv_i64 cdest)
1107 {
1108     uint64_t dest = s->base.pc_next + (int64_t)imm * 2;
1109     TCGLabel *lab;
1110 
1111     /* Take care of the special cases first.  */
1112     if (c->cond == TCG_COND_NEVER) {
1113         return DISAS_NEXT;
1114     }
1115     if (is_imm) {
1116         /*
1117          * Do not optimize a conditional branch if PER enabled, because we
1118          * still need a conditional call to helper_per_branch.
1119          */
1120         if (c->cond == TCG_COND_ALWAYS
1121             || (dest == s->pc_tmp &&
1122                 !(s->base.tb->flags & FLAG_MASK_PER_BRANCH))) {
1123             return help_goto_direct(s, dest);
1124         }
1125     } else {
1126         if (!cdest) {
1127             /* E.g. bcr %r0 -> no branch.  */
1128             return DISAS_NEXT;
1129         }
1130         if (c->cond == TCG_COND_ALWAYS) {
1131             return help_goto_indirect(s, cdest);
1132         }
1133     }
1134 
1135     update_cc_op(s);
1136 
1137     /*
1138      * Ensure the taken branch is fall-through of the tcg branch.
1139      * This keeps @cdest usage within the extended basic block,
1140      * which avoids an otherwise unnecessary spill to the stack.
1141      */
1142     lab = gen_new_label();
1143     if (c->is_64) {
1144         tcg_gen_brcond_i64(tcg_invert_cond(c->cond),
1145                            c->u.s64.a, c->u.s64.b, lab);
1146     } else {
1147         tcg_gen_brcond_i32(tcg_invert_cond(c->cond),
1148                            c->u.s32.a, c->u.s32.b, lab);
1149     }
1150 
1151     /* Branch taken.  */
1152     per_breaking_event(s);
1153     if (is_imm) {
1154         tcg_gen_movi_i64(psw_addr, dest);
1155     } else {
1156         tcg_gen_mov_i64(psw_addr, cdest);
1157     }
1158     per_branch(s, psw_addr);
1159 
1160     if (is_imm && use_goto_tb(s, dest)) {
1161         tcg_gen_goto_tb(0);
1162         tcg_gen_exit_tb(s->base.tb, 0);
1163     } else {
1164         tcg_gen_lookup_and_goto_ptr();
1165     }
1166 
1167     gen_set_label(lab);
1168 
1169     /* Branch not taken.  */
1170     tcg_gen_movi_i64(psw_addr, s->pc_tmp);
1171     if (use_goto_tb(s, s->pc_tmp)) {
1172         tcg_gen_goto_tb(1);
1173         tcg_gen_exit_tb(s->base.tb, 1);
1174         return DISAS_NORETURN;
1175     }
1176     return DISAS_PC_CC_UPDATED;
1177 }
1178 
1179 /* ====================================================================== */
1180 /* The operations.  These perform the bulk of the work for any insn,
1181    usually after the operands have been loaded and output initialized.  */
1182 
1183 static DisasJumpType op_abs(DisasContext *s, DisasOps *o)
1184 {
1185     tcg_gen_abs_i64(o->out, o->in2);
1186     return DISAS_NEXT;
1187 }
1188 
1189 static DisasJumpType op_absf32(DisasContext *s, DisasOps *o)
1190 {
1191     tcg_gen_andi_i64(o->out, o->in2, 0x7fffffffull);
1192     return DISAS_NEXT;
1193 }
1194 
1195 static DisasJumpType op_absf64(DisasContext *s, DisasOps *o)
1196 {
1197     tcg_gen_andi_i64(o->out, o->in2, 0x7fffffffffffffffull);
1198     return DISAS_NEXT;
1199 }
1200 
1201 static DisasJumpType op_absf128(DisasContext *s, DisasOps *o)
1202 {
1203     tcg_gen_andi_i64(o->out, o->in1, 0x7fffffffffffffffull);
1204     tcg_gen_mov_i64(o->out2, o->in2);
1205     return DISAS_NEXT;
1206 }
1207 
1208 static DisasJumpType op_add(DisasContext *s, DisasOps *o)
1209 {
1210     tcg_gen_add_i64(o->out, o->in1, o->in2);
1211     return DISAS_NEXT;
1212 }
1213 
1214 static DisasJumpType op_addu64(DisasContext *s, DisasOps *o)
1215 {
1216     tcg_gen_movi_i64(cc_src, 0);
1217     tcg_gen_add2_i64(o->out, cc_src, o->in1, cc_src, o->in2, cc_src);
1218     return DISAS_NEXT;
1219 }
1220 
1221 /* Compute carry into cc_src. */
1222 static void compute_carry(DisasContext *s)
1223 {
1224     switch (s->cc_op) {
1225     case CC_OP_ADDU:
1226         /* The carry value is already in cc_src (1,0). */
1227         break;
1228     case CC_OP_SUBU:
1229         tcg_gen_addi_i64(cc_src, cc_src, 1);
1230         break;
1231     default:
1232         gen_op_calc_cc(s);
1233         /* fall through */
1234     case CC_OP_STATIC:
1235         /* The carry flag is the msb of CC; compute into cc_src. */
1236         tcg_gen_extu_i32_i64(cc_src, cc_op);
1237         tcg_gen_shri_i64(cc_src, cc_src, 1);
1238         break;
1239     }
1240 }
1241 
1242 static DisasJumpType op_addc32(DisasContext *s, DisasOps *o)
1243 {
1244     compute_carry(s);
1245     tcg_gen_add_i64(o->out, o->in1, o->in2);
1246     tcg_gen_add_i64(o->out, o->out, cc_src);
1247     return DISAS_NEXT;
1248 }
1249 
1250 static DisasJumpType op_addc64(DisasContext *s, DisasOps *o)
1251 {
1252     compute_carry(s);
1253 
1254     TCGv_i64 zero = tcg_constant_i64(0);
1255     tcg_gen_add2_i64(o->out, cc_src, o->in1, zero, cc_src, zero);
1256     tcg_gen_add2_i64(o->out, cc_src, o->out, cc_src, o->in2, zero);
1257 
1258     return DISAS_NEXT;
1259 }
1260 
1261 static DisasJumpType op_asi(DisasContext *s, DisasOps *o)
1262 {
1263     bool non_atomic = !s390_has_feat(S390_FEAT_STFLE_45);
1264 
1265     o->in1 = tcg_temp_new_i64();
1266     if (non_atomic) {
1267         tcg_gen_qemu_ld_tl(o->in1, o->addr1, get_mem_index(s), s->insn->data);
1268     } else {
1269         /* Perform the atomic addition in memory. */
1270         tcg_gen_atomic_fetch_add_i64(o->in1, o->addr1, o->in2, get_mem_index(s),
1271                                      s->insn->data);
1272     }
1273 
1274     /* Recompute also for atomic case: needed for setting CC. */
1275     tcg_gen_add_i64(o->out, o->in1, o->in2);
1276 
1277     if (non_atomic) {
1278         tcg_gen_qemu_st_tl(o->out, o->addr1, get_mem_index(s), s->insn->data);
1279     }
1280     return DISAS_NEXT;
1281 }
1282 
1283 static DisasJumpType op_asiu64(DisasContext *s, DisasOps *o)
1284 {
1285     bool non_atomic = !s390_has_feat(S390_FEAT_STFLE_45);
1286 
1287     o->in1 = tcg_temp_new_i64();
1288     if (non_atomic) {
1289         tcg_gen_qemu_ld_tl(o->in1, o->addr1, get_mem_index(s), s->insn->data);
1290     } else {
1291         /* Perform the atomic addition in memory. */
1292         tcg_gen_atomic_fetch_add_i64(o->in1, o->addr1, o->in2, get_mem_index(s),
1293                                      s->insn->data);
1294     }
1295 
1296     /* Recompute also for atomic case: needed for setting CC. */
1297     tcg_gen_movi_i64(cc_src, 0);
1298     tcg_gen_add2_i64(o->out, cc_src, o->in1, cc_src, o->in2, cc_src);
1299 
1300     if (non_atomic) {
1301         tcg_gen_qemu_st_tl(o->out, o->addr1, get_mem_index(s), s->insn->data);
1302     }
1303     return DISAS_NEXT;
1304 }
1305 
1306 static DisasJumpType op_aeb(DisasContext *s, DisasOps *o)
1307 {
1308     gen_helper_aeb(o->out, tcg_env, o->in1, o->in2);
1309     return DISAS_NEXT;
1310 }
1311 
1312 static DisasJumpType op_adb(DisasContext *s, DisasOps *o)
1313 {
1314     gen_helper_adb(o->out, tcg_env, o->in1, o->in2);
1315     return DISAS_NEXT;
1316 }
1317 
1318 static DisasJumpType op_axb(DisasContext *s, DisasOps *o)
1319 {
1320     gen_helper_axb(o->out_128, tcg_env, o->in1_128, o->in2_128);
1321     return DISAS_NEXT;
1322 }
1323 
1324 static DisasJumpType op_and(DisasContext *s, DisasOps *o)
1325 {
1326     tcg_gen_and_i64(o->out, o->in1, o->in2);
1327     return DISAS_NEXT;
1328 }
1329 
1330 static DisasJumpType op_andi(DisasContext *s, DisasOps *o)
1331 {
1332     int shift = s->insn->data & 0xff;
1333     int size = s->insn->data >> 8;
1334     uint64_t mask = ((1ull << size) - 1) << shift;
1335     TCGv_i64 t = tcg_temp_new_i64();
1336 
1337     tcg_gen_shli_i64(t, o->in2, shift);
1338     tcg_gen_ori_i64(t, t, ~mask);
1339     tcg_gen_and_i64(o->out, o->in1, t);
1340 
1341     /* Produce the CC from only the bits manipulated.  */
1342     tcg_gen_andi_i64(cc_dst, o->out, mask);
1343     set_cc_nz_u64(s, cc_dst);
1344     return DISAS_NEXT;
1345 }
1346 
1347 static DisasJumpType op_andc(DisasContext *s, DisasOps *o)
1348 {
1349     tcg_gen_andc_i64(o->out, o->in1, o->in2);
1350     return DISAS_NEXT;
1351 }
1352 
1353 static DisasJumpType op_orc(DisasContext *s, DisasOps *o)
1354 {
1355     tcg_gen_orc_i64(o->out, o->in1, o->in2);
1356     return DISAS_NEXT;
1357 }
1358 
1359 static DisasJumpType op_nand(DisasContext *s, DisasOps *o)
1360 {
1361     tcg_gen_nand_i64(o->out, o->in1, o->in2);
1362     return DISAS_NEXT;
1363 }
1364 
1365 static DisasJumpType op_nor(DisasContext *s, DisasOps *o)
1366 {
1367     tcg_gen_nor_i64(o->out, o->in1, o->in2);
1368     return DISAS_NEXT;
1369 }
1370 
1371 static DisasJumpType op_nxor(DisasContext *s, DisasOps *o)
1372 {
1373     tcg_gen_eqv_i64(o->out, o->in1, o->in2);
1374     return DISAS_NEXT;
1375 }
1376 
1377 static DisasJumpType op_ni(DisasContext *s, DisasOps *o)
1378 {
1379     o->in1 = tcg_temp_new_i64();
1380 
1381     if (!s390_has_feat(S390_FEAT_INTERLOCKED_ACCESS_2)) {
1382         tcg_gen_qemu_ld_tl(o->in1, o->addr1, get_mem_index(s), s->insn->data);
1383     } else {
1384         /* Perform the atomic operation in memory. */
1385         tcg_gen_atomic_fetch_and_i64(o->in1, o->addr1, o->in2, get_mem_index(s),
1386                                      s->insn->data);
1387     }
1388 
1389     /* Recompute also for atomic case: needed for setting CC. */
1390     tcg_gen_and_i64(o->out, o->in1, o->in2);
1391 
1392     if (!s390_has_feat(S390_FEAT_INTERLOCKED_ACCESS_2)) {
1393         tcg_gen_qemu_st_tl(o->out, o->addr1, get_mem_index(s), s->insn->data);
1394     }
1395     return DISAS_NEXT;
1396 }
1397 
1398 static DisasJumpType op_bas(DisasContext *s, DisasOps *o)
1399 {
1400     pc_to_link_info(o->out, s, s->pc_tmp);
1401     if (o->in2) {
1402         return help_goto_indirect(s, o->in2);
1403     } else {
1404         return DISAS_NEXT;
1405     }
1406 }
1407 
1408 static void save_link_info(DisasContext *s, DisasOps *o)
1409 {
1410     TCGv_i64 t;
1411 
1412     if (s->base.tb->flags & (FLAG_MASK_32 | FLAG_MASK_64)) {
1413         pc_to_link_info(o->out, s, s->pc_tmp);
1414         return;
1415     }
1416     gen_op_calc_cc(s);
1417     tcg_gen_andi_i64(o->out, o->out, 0xffffffff00000000ull);
1418     tcg_gen_ori_i64(o->out, o->out, ((s->ilen / 2) << 30) | s->pc_tmp);
1419     t = tcg_temp_new_i64();
1420     tcg_gen_shri_i64(t, psw_mask, 16);
1421     tcg_gen_andi_i64(t, t, 0x0f000000);
1422     tcg_gen_or_i64(o->out, o->out, t);
1423     tcg_gen_extu_i32_i64(t, cc_op);
1424     tcg_gen_shli_i64(t, t, 28);
1425     tcg_gen_or_i64(o->out, o->out, t);
1426 }
1427 
1428 static DisasJumpType op_bal(DisasContext *s, DisasOps *o)
1429 {
1430     save_link_info(s, o);
1431     if (o->in2) {
1432         return help_goto_indirect(s, o->in2);
1433     } else {
1434         return DISAS_NEXT;
1435     }
1436 }
1437 
1438 /*
1439  * Disassemble the target of a branch. The results are returned in a form
1440  * suitable for passing into help_branch():
1441  *
1442  * - bool IS_IMM reflects whether the target is fixed or computed. Non-EXECUTEd
1443  *   branches, whose DisasContext *S contains the relative immediate field RI,
1444  *   are considered fixed. All the other branches are considered computed.
1445  * - int IMM is the value of RI.
1446  * - TCGv_i64 CDEST is the address of the computed target.
1447  */
1448 #define disas_jdest(s, ri, is_imm, imm, cdest) do {                            \
1449     if (have_field(s, ri)) {                                                   \
1450         if (unlikely(s->ex_value)) {                                           \
1451             cdest = tcg_temp_new_i64();                                        \
1452             tcg_gen_ld_i64(cdest, tcg_env, offsetof(CPUS390XState, ex_target));\
1453             tcg_gen_addi_i64(cdest, cdest, (int64_t)get_field(s, ri) * 2);     \
1454             is_imm = false;                                                    \
1455         } else {                                                               \
1456             is_imm = true;                                                     \
1457         }                                                                      \
1458     } else {                                                                   \
1459         is_imm = false;                                                        \
1460     }                                                                          \
1461     imm = is_imm ? get_field(s, ri) : 0;                                       \
1462 } while (false)
1463 
1464 static DisasJumpType op_basi(DisasContext *s, DisasOps *o)
1465 {
1466     DisasCompare c;
1467     bool is_imm;
1468     int imm;
1469 
1470     pc_to_link_info(o->out, s, s->pc_tmp);
1471 
1472     disas_jdest(s, i2, is_imm, imm, o->in2);
1473     disas_jcc(s, &c, 0xf);
1474     return help_branch(s, &c, is_imm, imm, o->in2);
1475 }
1476 
1477 static DisasJumpType op_bc(DisasContext *s, DisasOps *o)
1478 {
1479     int m1 = get_field(s, m1);
1480     DisasCompare c;
1481     bool is_imm;
1482     int imm;
1483 
1484     /* BCR with R2 = 0 causes no branching */
1485     if (have_field(s, r2) && get_field(s, r2) == 0) {
1486         if (m1 == 14) {
1487             /* Perform serialization */
1488             /* FIXME: check for fast-BCR-serialization facility */
1489             tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC);
1490         }
1491         if (m1 == 15) {
1492             /* Perform serialization */
1493             /* FIXME: perform checkpoint-synchronisation */
1494             tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC);
1495         }
1496         return DISAS_NEXT;
1497     }
1498 
1499     disas_jdest(s, i2, is_imm, imm, o->in2);
1500     disas_jcc(s, &c, m1);
1501     return help_branch(s, &c, is_imm, imm, o->in2);
1502 }
1503 
1504 static DisasJumpType op_bct32(DisasContext *s, DisasOps *o)
1505 {
1506     int r1 = get_field(s, r1);
1507     DisasCompare c;
1508     bool is_imm;
1509     TCGv_i64 t;
1510     int imm;
1511 
1512     c.cond = TCG_COND_NE;
1513     c.is_64 = false;
1514 
1515     t = tcg_temp_new_i64();
1516     tcg_gen_subi_i64(t, regs[r1], 1);
1517     store_reg32_i64(r1, t);
1518     c.u.s32.a = tcg_temp_new_i32();
1519     c.u.s32.b = tcg_constant_i32(0);
1520     tcg_gen_extrl_i64_i32(c.u.s32.a, t);
1521 
1522     disas_jdest(s, i2, is_imm, imm, o->in2);
1523     return help_branch(s, &c, is_imm, imm, o->in2);
1524 }
1525 
1526 static DisasJumpType op_bcth(DisasContext *s, DisasOps *o)
1527 {
1528     int r1 = get_field(s, r1);
1529     int imm = get_field(s, i2);
1530     DisasCompare c;
1531     TCGv_i64 t;
1532 
1533     c.cond = TCG_COND_NE;
1534     c.is_64 = false;
1535 
1536     t = tcg_temp_new_i64();
1537     tcg_gen_shri_i64(t, regs[r1], 32);
1538     tcg_gen_subi_i64(t, t, 1);
1539     store_reg32h_i64(r1, t);
1540     c.u.s32.a = tcg_temp_new_i32();
1541     c.u.s32.b = tcg_constant_i32(0);
1542     tcg_gen_extrl_i64_i32(c.u.s32.a, t);
1543 
1544     return help_branch(s, &c, 1, imm, o->in2);
1545 }
1546 
1547 static DisasJumpType op_bct64(DisasContext *s, DisasOps *o)
1548 {
1549     int r1 = get_field(s, r1);
1550     DisasCompare c;
1551     bool is_imm;
1552     int imm;
1553 
1554     c.cond = TCG_COND_NE;
1555     c.is_64 = true;
1556 
1557     tcg_gen_subi_i64(regs[r1], regs[r1], 1);
1558     c.u.s64.a = regs[r1];
1559     c.u.s64.b = tcg_constant_i64(0);
1560 
1561     disas_jdest(s, i2, is_imm, imm, o->in2);
1562     return help_branch(s, &c, is_imm, imm, o->in2);
1563 }
1564 
1565 static DisasJumpType op_bx32(DisasContext *s, DisasOps *o)
1566 {
1567     int r1 = get_field(s, r1);
1568     int r3 = get_field(s, r3);
1569     DisasCompare c;
1570     bool is_imm;
1571     TCGv_i64 t;
1572     int imm;
1573 
1574     c.cond = (s->insn->data ? TCG_COND_LE : TCG_COND_GT);
1575     c.is_64 = false;
1576 
1577     t = tcg_temp_new_i64();
1578     tcg_gen_add_i64(t, regs[r1], regs[r3]);
1579     c.u.s32.a = tcg_temp_new_i32();
1580     c.u.s32.b = tcg_temp_new_i32();
1581     tcg_gen_extrl_i64_i32(c.u.s32.a, t);
1582     tcg_gen_extrl_i64_i32(c.u.s32.b, regs[r3 | 1]);
1583     store_reg32_i64(r1, t);
1584 
1585     disas_jdest(s, i2, is_imm, imm, o->in2);
1586     return help_branch(s, &c, is_imm, imm, o->in2);
1587 }
1588 
1589 static DisasJumpType op_bx64(DisasContext *s, DisasOps *o)
1590 {
1591     int r1 = get_field(s, r1);
1592     int r3 = get_field(s, r3);
1593     DisasCompare c;
1594     bool is_imm;
1595     int imm;
1596 
1597     c.cond = (s->insn->data ? TCG_COND_LE : TCG_COND_GT);
1598     c.is_64 = true;
1599 
1600     if (r1 == (r3 | 1)) {
1601         c.u.s64.b = load_reg(r3 | 1);
1602     } else {
1603         c.u.s64.b = regs[r3 | 1];
1604     }
1605 
1606     tcg_gen_add_i64(regs[r1], regs[r1], regs[r3]);
1607     c.u.s64.a = regs[r1];
1608 
1609     disas_jdest(s, i2, is_imm, imm, o->in2);
1610     return help_branch(s, &c, is_imm, imm, o->in2);
1611 }
1612 
1613 static DisasJumpType op_cj(DisasContext *s, DisasOps *o)
1614 {
1615     int imm, m3 = get_field(s, m3);
1616     bool is_imm;
1617     DisasCompare c;
1618 
1619     c.cond = ltgt_cond[m3];
1620     if (s->insn->data) {
1621         c.cond = tcg_unsigned_cond(c.cond);
1622     }
1623     c.is_64 = true;
1624     c.u.s64.a = o->in1;
1625     c.u.s64.b = o->in2;
1626 
1627     o->out = NULL;
1628     disas_jdest(s, i4, is_imm, imm, o->out);
1629     if (!is_imm && !o->out) {
1630         imm = 0;
1631         o->out = get_address(s, 0, get_field(s, b4),
1632                              get_field(s, d4));
1633     }
1634 
1635     return help_branch(s, &c, is_imm, imm, o->out);
1636 }
1637 
1638 static DisasJumpType op_ceb(DisasContext *s, DisasOps *o)
1639 {
1640     gen_helper_ceb(cc_op, tcg_env, o->in1, o->in2);
1641     set_cc_static(s);
1642     return DISAS_NEXT;
1643 }
1644 
1645 static DisasJumpType op_cdb(DisasContext *s, DisasOps *o)
1646 {
1647     gen_helper_cdb(cc_op, tcg_env, o->in1, o->in2);
1648     set_cc_static(s);
1649     return DISAS_NEXT;
1650 }
1651 
1652 static DisasJumpType op_cxb(DisasContext *s, DisasOps *o)
1653 {
1654     gen_helper_cxb(cc_op, tcg_env, o->in1_128, o->in2_128);
1655     set_cc_static(s);
1656     return DISAS_NEXT;
1657 }
1658 
1659 static TCGv_i32 fpinst_extract_m34(DisasContext *s, bool m3_with_fpe,
1660                                    bool m4_with_fpe)
1661 {
1662     const bool fpe = s390_has_feat(S390_FEAT_FLOATING_POINT_EXT);
1663     uint8_t m3 = get_field(s, m3);
1664     uint8_t m4 = get_field(s, m4);
1665 
1666     /* m3 field was introduced with FPE */
1667     if (!fpe && m3_with_fpe) {
1668         m3 = 0;
1669     }
1670     /* m4 field was introduced with FPE */
1671     if (!fpe && m4_with_fpe) {
1672         m4 = 0;
1673     }
1674 
1675     /* Check for valid rounding modes. Mode 3 was introduced later. */
1676     if (m3 == 2 || m3 > 7 || (!fpe && m3 == 3)) {
1677         gen_program_exception(s, PGM_SPECIFICATION);
1678         return NULL;
1679     }
1680 
1681     return tcg_constant_i32(deposit32(m3, 4, 4, m4));
1682 }
1683 
1684 static DisasJumpType op_cfeb(DisasContext *s, DisasOps *o)
1685 {
1686     TCGv_i32 m34 = fpinst_extract_m34(s, false, true);
1687 
1688     if (!m34) {
1689         return DISAS_NORETURN;
1690     }
1691     gen_helper_cfeb(o->out, tcg_env, o->in2, m34);
1692     set_cc_static(s);
1693     return DISAS_NEXT;
1694 }
1695 
1696 static DisasJumpType op_cfdb(DisasContext *s, DisasOps *o)
1697 {
1698     TCGv_i32 m34 = fpinst_extract_m34(s, false, true);
1699 
1700     if (!m34) {
1701         return DISAS_NORETURN;
1702     }
1703     gen_helper_cfdb(o->out, tcg_env, o->in2, m34);
1704     set_cc_static(s);
1705     return DISAS_NEXT;
1706 }
1707 
1708 static DisasJumpType op_cfxb(DisasContext *s, DisasOps *o)
1709 {
1710     TCGv_i32 m34 = fpinst_extract_m34(s, false, true);
1711 
1712     if (!m34) {
1713         return DISAS_NORETURN;
1714     }
1715     gen_helper_cfxb(o->out, tcg_env, o->in2_128, m34);
1716     set_cc_static(s);
1717     return DISAS_NEXT;
1718 }
1719 
1720 static DisasJumpType op_cgeb(DisasContext *s, DisasOps *o)
1721 {
1722     TCGv_i32 m34 = fpinst_extract_m34(s, false, true);
1723 
1724     if (!m34) {
1725         return DISAS_NORETURN;
1726     }
1727     gen_helper_cgeb(o->out, tcg_env, o->in2, m34);
1728     set_cc_static(s);
1729     return DISAS_NEXT;
1730 }
1731 
1732 static DisasJumpType op_cgdb(DisasContext *s, DisasOps *o)
1733 {
1734     TCGv_i32 m34 = fpinst_extract_m34(s, false, true);
1735 
1736     if (!m34) {
1737         return DISAS_NORETURN;
1738     }
1739     gen_helper_cgdb(o->out, tcg_env, o->in2, m34);
1740     set_cc_static(s);
1741     return DISAS_NEXT;
1742 }
1743 
1744 static DisasJumpType op_cgxb(DisasContext *s, DisasOps *o)
1745 {
1746     TCGv_i32 m34 = fpinst_extract_m34(s, false, true);
1747 
1748     if (!m34) {
1749         return DISAS_NORETURN;
1750     }
1751     gen_helper_cgxb(o->out, tcg_env, o->in2_128, m34);
1752     set_cc_static(s);
1753     return DISAS_NEXT;
1754 }
1755 
1756 static DisasJumpType op_clfeb(DisasContext *s, DisasOps *o)
1757 {
1758     TCGv_i32 m34 = fpinst_extract_m34(s, false, false);
1759 
1760     if (!m34) {
1761         return DISAS_NORETURN;
1762     }
1763     gen_helper_clfeb(o->out, tcg_env, o->in2, m34);
1764     set_cc_static(s);
1765     return DISAS_NEXT;
1766 }
1767 
1768 static DisasJumpType op_clfdb(DisasContext *s, DisasOps *o)
1769 {
1770     TCGv_i32 m34 = fpinst_extract_m34(s, false, false);
1771 
1772     if (!m34) {
1773         return DISAS_NORETURN;
1774     }
1775     gen_helper_clfdb(o->out, tcg_env, o->in2, m34);
1776     set_cc_static(s);
1777     return DISAS_NEXT;
1778 }
1779 
1780 static DisasJumpType op_clfxb(DisasContext *s, DisasOps *o)
1781 {
1782     TCGv_i32 m34 = fpinst_extract_m34(s, false, false);
1783 
1784     if (!m34) {
1785         return DISAS_NORETURN;
1786     }
1787     gen_helper_clfxb(o->out, tcg_env, o->in2_128, m34);
1788     set_cc_static(s);
1789     return DISAS_NEXT;
1790 }
1791 
1792 static DisasJumpType op_clgeb(DisasContext *s, DisasOps *o)
1793 {
1794     TCGv_i32 m34 = fpinst_extract_m34(s, false, false);
1795 
1796     if (!m34) {
1797         return DISAS_NORETURN;
1798     }
1799     gen_helper_clgeb(o->out, tcg_env, o->in2, m34);
1800     set_cc_static(s);
1801     return DISAS_NEXT;
1802 }
1803 
1804 static DisasJumpType op_clgdb(DisasContext *s, DisasOps *o)
1805 {
1806     TCGv_i32 m34 = fpinst_extract_m34(s, false, false);
1807 
1808     if (!m34) {
1809         return DISAS_NORETURN;
1810     }
1811     gen_helper_clgdb(o->out, tcg_env, o->in2, m34);
1812     set_cc_static(s);
1813     return DISAS_NEXT;
1814 }
1815 
1816 static DisasJumpType op_clgxb(DisasContext *s, DisasOps *o)
1817 {
1818     TCGv_i32 m34 = fpinst_extract_m34(s, false, false);
1819 
1820     if (!m34) {
1821         return DISAS_NORETURN;
1822     }
1823     gen_helper_clgxb(o->out, tcg_env, o->in2_128, m34);
1824     set_cc_static(s);
1825     return DISAS_NEXT;
1826 }
1827 
1828 static DisasJumpType op_cegb(DisasContext *s, DisasOps *o)
1829 {
1830     TCGv_i32 m34 = fpinst_extract_m34(s, true, true);
1831 
1832     if (!m34) {
1833         return DISAS_NORETURN;
1834     }
1835     gen_helper_cegb(o->out, tcg_env, o->in2, m34);
1836     return DISAS_NEXT;
1837 }
1838 
1839 static DisasJumpType op_cdgb(DisasContext *s, DisasOps *o)
1840 {
1841     TCGv_i32 m34 = fpinst_extract_m34(s, true, true);
1842 
1843     if (!m34) {
1844         return DISAS_NORETURN;
1845     }
1846     gen_helper_cdgb(o->out, tcg_env, o->in2, m34);
1847     return DISAS_NEXT;
1848 }
1849 
1850 static DisasJumpType op_cxgb(DisasContext *s, DisasOps *o)
1851 {
1852     TCGv_i32 m34 = fpinst_extract_m34(s, true, true);
1853 
1854     if (!m34) {
1855         return DISAS_NORETURN;
1856     }
1857     gen_helper_cxgb(o->out_128, tcg_env, o->in2, m34);
1858     return DISAS_NEXT;
1859 }
1860 
1861 static DisasJumpType op_celgb(DisasContext *s, DisasOps *o)
1862 {
1863     TCGv_i32 m34 = fpinst_extract_m34(s, false, false);
1864 
1865     if (!m34) {
1866         return DISAS_NORETURN;
1867     }
1868     gen_helper_celgb(o->out, tcg_env, o->in2, m34);
1869     return DISAS_NEXT;
1870 }
1871 
1872 static DisasJumpType op_cdlgb(DisasContext *s, DisasOps *o)
1873 {
1874     TCGv_i32 m34 = fpinst_extract_m34(s, false, false);
1875 
1876     if (!m34) {
1877         return DISAS_NORETURN;
1878     }
1879     gen_helper_cdlgb(o->out, tcg_env, o->in2, m34);
1880     return DISAS_NEXT;
1881 }
1882 
1883 static DisasJumpType op_cxlgb(DisasContext *s, DisasOps *o)
1884 {
1885     TCGv_i32 m34 = fpinst_extract_m34(s, false, false);
1886 
1887     if (!m34) {
1888         return DISAS_NORETURN;
1889     }
1890     gen_helper_cxlgb(o->out_128, tcg_env, o->in2, m34);
1891     return DISAS_NEXT;
1892 }
1893 
1894 static DisasJumpType op_cksm(DisasContext *s, DisasOps *o)
1895 {
1896     int r2 = get_field(s, r2);
1897     TCGv_i128 pair = tcg_temp_new_i128();
1898     TCGv_i64 len = tcg_temp_new_i64();
1899 
1900     gen_helper_cksm(pair, tcg_env, o->in1, o->in2, regs[r2 + 1]);
1901     set_cc_static(s);
1902     tcg_gen_extr_i128_i64(o->out, len, pair);
1903 
1904     tcg_gen_add_i64(regs[r2], regs[r2], len);
1905     tcg_gen_sub_i64(regs[r2 + 1], regs[r2 + 1], len);
1906 
1907     return DISAS_NEXT;
1908 }
1909 
1910 static DisasJumpType op_clc(DisasContext *s, DisasOps *o)
1911 {
1912     int l = get_field(s, l1);
1913     TCGv_i64 src;
1914     TCGv_i32 vl;
1915     MemOp mop;
1916 
1917     switch (l + 1) {
1918     case 1:
1919     case 2:
1920     case 4:
1921     case 8:
1922         mop = ctz32(l + 1) | MO_TE;
1923         /* Do not update cc_src yet: loading cc_dst may cause an exception. */
1924         src = tcg_temp_new_i64();
1925         tcg_gen_qemu_ld_tl(src, o->addr1, get_mem_index(s), mop);
1926         tcg_gen_qemu_ld_tl(cc_dst, o->in2, get_mem_index(s), mop);
1927         gen_op_update2_cc_i64(s, CC_OP_LTUGTU_64, src, cc_dst);
1928         return DISAS_NEXT;
1929     default:
1930         vl = tcg_constant_i32(l);
1931         gen_helper_clc(cc_op, tcg_env, vl, o->addr1, o->in2);
1932         set_cc_static(s);
1933         return DISAS_NEXT;
1934     }
1935 }
1936 
1937 static DisasJumpType op_clcl(DisasContext *s, DisasOps *o)
1938 {
1939     int r1 = get_field(s, r1);
1940     int r2 = get_field(s, r2);
1941     TCGv_i32 t1, t2;
1942 
1943     /* r1 and r2 must be even.  */
1944     if (r1 & 1 || r2 & 1) {
1945         gen_program_exception(s, PGM_SPECIFICATION);
1946         return DISAS_NORETURN;
1947     }
1948 
1949     t1 = tcg_constant_i32(r1);
1950     t2 = tcg_constant_i32(r2);
1951     gen_helper_clcl(cc_op, tcg_env, t1, t2);
1952     set_cc_static(s);
1953     return DISAS_NEXT;
1954 }
1955 
1956 static DisasJumpType op_clcle(DisasContext *s, DisasOps *o)
1957 {
1958     int r1 = get_field(s, r1);
1959     int r3 = get_field(s, r3);
1960     TCGv_i32 t1, t3;
1961 
1962     /* r1 and r3 must be even.  */
1963     if (r1 & 1 || r3 & 1) {
1964         gen_program_exception(s, PGM_SPECIFICATION);
1965         return DISAS_NORETURN;
1966     }
1967 
1968     t1 = tcg_constant_i32(r1);
1969     t3 = tcg_constant_i32(r3);
1970     gen_helper_clcle(cc_op, tcg_env, t1, o->in2, t3);
1971     set_cc_static(s);
1972     return DISAS_NEXT;
1973 }
1974 
1975 static DisasJumpType op_clclu(DisasContext *s, DisasOps *o)
1976 {
1977     int r1 = get_field(s, r1);
1978     int r3 = get_field(s, r3);
1979     TCGv_i32 t1, t3;
1980 
1981     /* r1 and r3 must be even.  */
1982     if (r1 & 1 || r3 & 1) {
1983         gen_program_exception(s, PGM_SPECIFICATION);
1984         return DISAS_NORETURN;
1985     }
1986 
1987     t1 = tcg_constant_i32(r1);
1988     t3 = tcg_constant_i32(r3);
1989     gen_helper_clclu(cc_op, tcg_env, t1, o->in2, t3);
1990     set_cc_static(s);
1991     return DISAS_NEXT;
1992 }
1993 
1994 static DisasJumpType op_clm(DisasContext *s, DisasOps *o)
1995 {
1996     TCGv_i32 m3 = tcg_constant_i32(get_field(s, m3));
1997     TCGv_i32 t1 = tcg_temp_new_i32();
1998 
1999     tcg_gen_extrl_i64_i32(t1, o->in1);
2000     gen_helper_clm(cc_op, tcg_env, t1, m3, o->in2);
2001     set_cc_static(s);
2002     return DISAS_NEXT;
2003 }
2004 
2005 static DisasJumpType op_clst(DisasContext *s, DisasOps *o)
2006 {
2007     TCGv_i128 pair = tcg_temp_new_i128();
2008 
2009     gen_helper_clst(pair, tcg_env, regs[0], o->in1, o->in2);
2010     tcg_gen_extr_i128_i64(o->in2, o->in1, pair);
2011 
2012     set_cc_static(s);
2013     return DISAS_NEXT;
2014 }
2015 
2016 static DisasJumpType op_cps(DisasContext *s, DisasOps *o)
2017 {
2018     TCGv_i64 t = tcg_temp_new_i64();
2019     tcg_gen_andi_i64(t, o->in1, 0x8000000000000000ull);
2020     tcg_gen_andi_i64(o->out, o->in2, 0x7fffffffffffffffull);
2021     tcg_gen_or_i64(o->out, o->out, t);
2022     return DISAS_NEXT;
2023 }
2024 
2025 static DisasJumpType op_cs(DisasContext *s, DisasOps *o)
2026 {
2027     int d2 = get_field(s, d2);
2028     int b2 = get_field(s, b2);
2029     TCGv_i64 addr, cc;
2030 
2031     /* Note that in1 = R3 (new value) and
2032        in2 = (zero-extended) R1 (expected value).  */
2033 
2034     addr = get_address(s, 0, b2, d2);
2035     tcg_gen_atomic_cmpxchg_i64(o->out, addr, o->in2, o->in1,
2036                                get_mem_index(s), s->insn->data | MO_ALIGN);
2037 
2038     /* Are the memory and expected values (un)equal?  Note that this setcond
2039        produces the output CC value, thus the NE sense of the test.  */
2040     cc = tcg_temp_new_i64();
2041     tcg_gen_setcond_i64(TCG_COND_NE, cc, o->in2, o->out);
2042     tcg_gen_extrl_i64_i32(cc_op, cc);
2043     set_cc_static(s);
2044 
2045     return DISAS_NEXT;
2046 }
2047 
2048 static DisasJumpType op_cdsg(DisasContext *s, DisasOps *o)
2049 {
2050     int r1 = get_field(s, r1);
2051 
2052     o->out_128 = tcg_temp_new_i128();
2053     tcg_gen_concat_i64_i128(o->out_128, regs[r1 + 1], regs[r1]);
2054 
2055     /* Note out (R1:R1+1) = expected value and in2 (R3:R3+1) = new value.  */
2056     tcg_gen_atomic_cmpxchg_i128(o->out_128, o->addr1, o->out_128, o->in2_128,
2057                                 get_mem_index(s), MO_BE | MO_128 | MO_ALIGN);
2058 
2059     /*
2060      * Extract result into cc_dst:cc_src, compare vs the expected value
2061      * in the as yet unmodified input registers, then update CC_OP.
2062      */
2063     tcg_gen_extr_i128_i64(cc_src, cc_dst, o->out_128);
2064     tcg_gen_xor_i64(cc_dst, cc_dst, regs[r1]);
2065     tcg_gen_xor_i64(cc_src, cc_src, regs[r1 + 1]);
2066     tcg_gen_or_i64(cc_dst, cc_dst, cc_src);
2067     set_cc_nz_u64(s, cc_dst);
2068 
2069     return DISAS_NEXT;
2070 }
2071 
2072 static DisasJumpType op_csst(DisasContext *s, DisasOps *o)
2073 {
2074     int r3 = get_field(s, r3);
2075     TCGv_i32 t_r3 = tcg_constant_i32(r3);
2076 
2077     if (tb_cflags(s->base.tb) & CF_PARALLEL) {
2078         gen_helper_csst_parallel(cc_op, tcg_env, t_r3, o->addr1, o->in2);
2079     } else {
2080         gen_helper_csst(cc_op, tcg_env, t_r3, o->addr1, o->in2);
2081     }
2082 
2083     set_cc_static(s);
2084     return DISAS_NEXT;
2085 }
2086 
2087 #ifndef CONFIG_USER_ONLY
2088 static DisasJumpType op_csp(DisasContext *s, DisasOps *o)
2089 {
2090     MemOp mop = s->insn->data;
2091     TCGv_i64 addr, old, cc;
2092     TCGLabel *lab = gen_new_label();
2093 
2094     /* Note that in1 = R1 (zero-extended expected value),
2095        out = R1 (original reg), out2 = R1+1 (new value).  */
2096 
2097     addr = tcg_temp_new_i64();
2098     old = tcg_temp_new_i64();
2099     tcg_gen_andi_i64(addr, o->in2, -1ULL << (mop & MO_SIZE));
2100     tcg_gen_atomic_cmpxchg_i64(old, addr, o->in1, o->out2,
2101                                get_mem_index(s), mop | MO_ALIGN);
2102 
2103     /* Are the memory and expected values (un)equal?  */
2104     cc = tcg_temp_new_i64();
2105     tcg_gen_setcond_i64(TCG_COND_NE, cc, o->in1, old);
2106     tcg_gen_extrl_i64_i32(cc_op, cc);
2107 
2108     /* Write back the output now, so that it happens before the
2109        following branch, so that we don't need local temps.  */
2110     if ((mop & MO_SIZE) == MO_32) {
2111         tcg_gen_deposit_i64(o->out, o->out, old, 0, 32);
2112     } else {
2113         tcg_gen_mov_i64(o->out, old);
2114     }
2115 
2116     /* If the comparison was equal, and the LSB of R2 was set,
2117        then we need to flush the TLB (for all cpus).  */
2118     tcg_gen_xori_i64(cc, cc, 1);
2119     tcg_gen_and_i64(cc, cc, o->in2);
2120     tcg_gen_brcondi_i64(TCG_COND_EQ, cc, 0, lab);
2121 
2122     gen_helper_purge(tcg_env);
2123     gen_set_label(lab);
2124 
2125     return DISAS_NEXT;
2126 }
2127 #endif
2128 
2129 static DisasJumpType op_cvb(DisasContext *s, DisasOps *o)
2130 {
2131     TCGv_i64 t = tcg_temp_new_i64();
2132     tcg_gen_qemu_ld_i64(t, o->addr1, get_mem_index(s), MO_TEUQ);
2133     gen_helper_cvb(tcg_env, tcg_constant_i32(get_field(s, r1)), t);
2134     return DISAS_NEXT;
2135 }
2136 
2137 static DisasJumpType op_cvbg(DisasContext *s, DisasOps *o)
2138 {
2139     TCGv_i128 t = tcg_temp_new_i128();
2140     tcg_gen_qemu_ld_i128(t, o->addr1, get_mem_index(s), MO_TE | MO_128);
2141     gen_helper_cvbg(o->out, tcg_env, t);
2142     return DISAS_NEXT;
2143 }
2144 
2145 static DisasJumpType op_cvd(DisasContext *s, DisasOps *o)
2146 {
2147     TCGv_i64 t1 = tcg_temp_new_i64();
2148     TCGv_i32 t2 = tcg_temp_new_i32();
2149     tcg_gen_extrl_i64_i32(t2, o->in1);
2150     gen_helper_cvd(t1, t2);
2151     tcg_gen_qemu_st_i64(t1, o->in2, get_mem_index(s), MO_TEUQ);
2152     return DISAS_NEXT;
2153 }
2154 
2155 static DisasJumpType op_cvdg(DisasContext *s, DisasOps *o)
2156 {
2157     TCGv_i128 t = tcg_temp_new_i128();
2158     gen_helper_cvdg(t, o->in1);
2159     tcg_gen_qemu_st_i128(t, o->in2, get_mem_index(s), MO_TE | MO_128);
2160     return DISAS_NEXT;
2161 }
2162 
2163 static DisasJumpType op_ct(DisasContext *s, DisasOps *o)
2164 {
2165     int m3 = get_field(s, m3);
2166     TCGLabel *lab = gen_new_label();
2167     TCGCond c;
2168 
2169     c = tcg_invert_cond(ltgt_cond[m3]);
2170     if (s->insn->data) {
2171         c = tcg_unsigned_cond(c);
2172     }
2173     tcg_gen_brcond_i64(c, o->in1, o->in2, lab);
2174 
2175     /* Trap.  */
2176     gen_trap(s);
2177 
2178     gen_set_label(lab);
2179     return DISAS_NEXT;
2180 }
2181 
2182 static DisasJumpType op_cuXX(DisasContext *s, DisasOps *o)
2183 {
2184     int m3 = get_field(s, m3);
2185     int r1 = get_field(s, r1);
2186     int r2 = get_field(s, r2);
2187     TCGv_i32 tr1, tr2, chk;
2188 
2189     /* R1 and R2 must both be even.  */
2190     if ((r1 | r2) & 1) {
2191         gen_program_exception(s, PGM_SPECIFICATION);
2192         return DISAS_NORETURN;
2193     }
2194     if (!s390_has_feat(S390_FEAT_ETF3_ENH)) {
2195         m3 = 0;
2196     }
2197 
2198     tr1 = tcg_constant_i32(r1);
2199     tr2 = tcg_constant_i32(r2);
2200     chk = tcg_constant_i32(m3);
2201 
2202     switch (s->insn->data) {
2203     case 12:
2204         gen_helper_cu12(cc_op, tcg_env, tr1, tr2, chk);
2205         break;
2206     case 14:
2207         gen_helper_cu14(cc_op, tcg_env, tr1, tr2, chk);
2208         break;
2209     case 21:
2210         gen_helper_cu21(cc_op, tcg_env, tr1, tr2, chk);
2211         break;
2212     case 24:
2213         gen_helper_cu24(cc_op, tcg_env, tr1, tr2, chk);
2214         break;
2215     case 41:
2216         gen_helper_cu41(cc_op, tcg_env, tr1, tr2, chk);
2217         break;
2218     case 42:
2219         gen_helper_cu42(cc_op, tcg_env, tr1, tr2, chk);
2220         break;
2221     default:
2222         g_assert_not_reached();
2223     }
2224 
2225     set_cc_static(s);
2226     return DISAS_NEXT;
2227 }
2228 
2229 #ifndef CONFIG_USER_ONLY
2230 static DisasJumpType op_diag(DisasContext *s, DisasOps *o)
2231 {
2232     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
2233     TCGv_i32 r3 = tcg_constant_i32(get_field(s, r3));
2234     TCGv_i32 func_code = tcg_constant_i32(get_field(s, i2));
2235 
2236     gen_helper_diag(tcg_env, r1, r3, func_code);
2237     return DISAS_NEXT;
2238 }
2239 #endif
2240 
2241 static DisasJumpType op_divs32(DisasContext *s, DisasOps *o)
2242 {
2243     gen_helper_divs32(o->out, tcg_env, o->in1, o->in2);
2244     tcg_gen_extr32_i64(o->out2, o->out, o->out);
2245     return DISAS_NEXT;
2246 }
2247 
2248 static DisasJumpType op_divu32(DisasContext *s, DisasOps *o)
2249 {
2250     gen_helper_divu32(o->out, tcg_env, o->in1, o->in2);
2251     tcg_gen_extr32_i64(o->out2, o->out, o->out);
2252     return DISAS_NEXT;
2253 }
2254 
2255 static DisasJumpType op_divs64(DisasContext *s, DisasOps *o)
2256 {
2257     TCGv_i128 t = tcg_temp_new_i128();
2258 
2259     gen_helper_divs64(t, tcg_env, o->in1, o->in2);
2260     tcg_gen_extr_i128_i64(o->out2, o->out, t);
2261     return DISAS_NEXT;
2262 }
2263 
2264 static DisasJumpType op_divu64(DisasContext *s, DisasOps *o)
2265 {
2266     TCGv_i128 t = tcg_temp_new_i128();
2267 
2268     gen_helper_divu64(t, tcg_env, o->out, o->out2, o->in2);
2269     tcg_gen_extr_i128_i64(o->out2, o->out, t);
2270     return DISAS_NEXT;
2271 }
2272 
2273 static DisasJumpType op_deb(DisasContext *s, DisasOps *o)
2274 {
2275     gen_helper_deb(o->out, tcg_env, o->in1, o->in2);
2276     return DISAS_NEXT;
2277 }
2278 
2279 static DisasJumpType op_ddb(DisasContext *s, DisasOps *o)
2280 {
2281     gen_helper_ddb(o->out, tcg_env, o->in1, o->in2);
2282     return DISAS_NEXT;
2283 }
2284 
2285 static DisasJumpType op_dxb(DisasContext *s, DisasOps *o)
2286 {
2287     gen_helper_dxb(o->out_128, tcg_env, o->in1_128, o->in2_128);
2288     return DISAS_NEXT;
2289 }
2290 
2291 static DisasJumpType op_ear(DisasContext *s, DisasOps *o)
2292 {
2293     int r2 = get_field(s, r2);
2294     tcg_gen_ld32u_i64(o->out, tcg_env, offsetof(CPUS390XState, aregs[r2]));
2295     return DISAS_NEXT;
2296 }
2297 
2298 static DisasJumpType op_ecag(DisasContext *s, DisasOps *o)
2299 {
2300     /* No cache information provided.  */
2301     tcg_gen_movi_i64(o->out, -1);
2302     return DISAS_NEXT;
2303 }
2304 
2305 static DisasJumpType op_efpc(DisasContext *s, DisasOps *o)
2306 {
2307     tcg_gen_ld32u_i64(o->out, tcg_env, offsetof(CPUS390XState, fpc));
2308     return DISAS_NEXT;
2309 }
2310 
2311 static DisasJumpType op_epsw(DisasContext *s, DisasOps *o)
2312 {
2313     int r1 = get_field(s, r1);
2314     int r2 = get_field(s, r2);
2315     TCGv_i64 t = tcg_temp_new_i64();
2316     TCGv_i64 t_cc = tcg_temp_new_i64();
2317 
2318     /* Note the "subsequently" in the PoO, which implies a defined result
2319        if r1 == r2.  Thus we cannot defer these writes to an output hook.  */
2320     gen_op_calc_cc(s);
2321     tcg_gen_extu_i32_i64(t_cc, cc_op);
2322     tcg_gen_shri_i64(t, psw_mask, 32);
2323     tcg_gen_deposit_i64(t, t, t_cc, 12, 2);
2324     store_reg32_i64(r1, t);
2325     if (r2 != 0) {
2326         store_reg32_i64(r2, psw_mask);
2327     }
2328     return DISAS_NEXT;
2329 }
2330 
2331 static DisasJumpType op_ex(DisasContext *s, DisasOps *o)
2332 {
2333     int r1 = get_field(s, r1);
2334     TCGv_i32 ilen;
2335     TCGv_i64 v1;
2336 
2337     /* Nested EXECUTE is not allowed.  */
2338     if (unlikely(s->ex_value)) {
2339         gen_program_exception(s, PGM_EXECUTE);
2340         return DISAS_NORETURN;
2341     }
2342 
2343     update_psw_addr(s);
2344     update_cc_op(s);
2345 
2346     if (r1 == 0) {
2347         v1 = tcg_constant_i64(0);
2348     } else {
2349         v1 = regs[r1];
2350     }
2351 
2352     ilen = tcg_constant_i32(s->ilen);
2353     gen_helper_ex(tcg_env, ilen, v1, o->in2);
2354 
2355     return DISAS_PC_CC_UPDATED;
2356 }
2357 
2358 static DisasJumpType op_fieb(DisasContext *s, DisasOps *o)
2359 {
2360     TCGv_i32 m34 = fpinst_extract_m34(s, false, true);
2361 
2362     if (!m34) {
2363         return DISAS_NORETURN;
2364     }
2365     gen_helper_fieb(o->out, tcg_env, o->in2, m34);
2366     return DISAS_NEXT;
2367 }
2368 
2369 static DisasJumpType op_fidb(DisasContext *s, DisasOps *o)
2370 {
2371     TCGv_i32 m34 = fpinst_extract_m34(s, false, true);
2372 
2373     if (!m34) {
2374         return DISAS_NORETURN;
2375     }
2376     gen_helper_fidb(o->out, tcg_env, o->in2, m34);
2377     return DISAS_NEXT;
2378 }
2379 
2380 static DisasJumpType op_fixb(DisasContext *s, DisasOps *o)
2381 {
2382     TCGv_i32 m34 = fpinst_extract_m34(s, false, true);
2383 
2384     if (!m34) {
2385         return DISAS_NORETURN;
2386     }
2387     gen_helper_fixb(o->out_128, tcg_env, o->in2_128, m34);
2388     return DISAS_NEXT;
2389 }
2390 
2391 static DisasJumpType op_flogr(DisasContext *s, DisasOps *o)
2392 {
2393     /* We'll use the original input for cc computation, since we get to
2394        compare that against 0, which ought to be better than comparing
2395        the real output against 64.  It also lets cc_dst be a convenient
2396        temporary during our computation.  */
2397     gen_op_update1_cc_i64(s, CC_OP_FLOGR, o->in2);
2398 
2399     /* R1 = IN ? CLZ(IN) : 64.  */
2400     tcg_gen_clzi_i64(o->out, o->in2, 64);
2401 
2402     /* R1+1 = IN & ~(found bit).  Note that we may attempt to shift this
2403        value by 64, which is undefined.  But since the shift is 64 iff the
2404        input is zero, we still get the correct result after and'ing.  */
2405     tcg_gen_movi_i64(o->out2, 0x8000000000000000ull);
2406     tcg_gen_shr_i64(o->out2, o->out2, o->out);
2407     tcg_gen_andc_i64(o->out2, cc_dst, o->out2);
2408     return DISAS_NEXT;
2409 }
2410 
2411 static DisasJumpType op_icm(DisasContext *s, DisasOps *o)
2412 {
2413     int m3 = get_field(s, m3);
2414     int pos, len, base = s->insn->data;
2415     TCGv_i64 tmp = tcg_temp_new_i64();
2416     uint64_t ccm;
2417 
2418     switch (m3) {
2419     case 0xf:
2420         /* Effectively a 32-bit load.  */
2421         tcg_gen_qemu_ld_i64(tmp, o->in2, get_mem_index(s), MO_TEUL);
2422         len = 32;
2423         goto one_insert;
2424 
2425     case 0xc:
2426     case 0x6:
2427     case 0x3:
2428         /* Effectively a 16-bit load.  */
2429         tcg_gen_qemu_ld_i64(tmp, o->in2, get_mem_index(s), MO_TEUW);
2430         len = 16;
2431         goto one_insert;
2432 
2433     case 0x8:
2434     case 0x4:
2435     case 0x2:
2436     case 0x1:
2437         /* Effectively an 8-bit load.  */
2438         tcg_gen_qemu_ld_i64(tmp, o->in2, get_mem_index(s), MO_UB);
2439         len = 8;
2440         goto one_insert;
2441 
2442     one_insert:
2443         pos = base + ctz32(m3) * 8;
2444         tcg_gen_deposit_i64(o->out, o->out, tmp, pos, len);
2445         ccm = ((1ull << len) - 1) << pos;
2446         break;
2447 
2448     case 0:
2449         /* Recognize access exceptions for the first byte.  */
2450         tcg_gen_qemu_ld_i64(tmp, o->in2, get_mem_index(s), MO_UB);
2451         gen_op_movi_cc(s, 0);
2452         return DISAS_NEXT;
2453 
2454     default:
2455         /* This is going to be a sequence of loads and inserts.  */
2456         pos = base + 32 - 8;
2457         ccm = 0;
2458         while (m3) {
2459             if (m3 & 0x8) {
2460                 tcg_gen_qemu_ld_i64(tmp, o->in2, get_mem_index(s), MO_UB);
2461                 tcg_gen_addi_i64(o->in2, o->in2, 1);
2462                 tcg_gen_deposit_i64(o->out, o->out, tmp, pos, 8);
2463                 ccm |= 0xffull << pos;
2464             }
2465             m3 = (m3 << 1) & 0xf;
2466             pos -= 8;
2467         }
2468         break;
2469     }
2470 
2471     tcg_gen_movi_i64(tmp, ccm);
2472     gen_op_update2_cc_i64(s, CC_OP_ICM, tmp, o->out);
2473     return DISAS_NEXT;
2474 }
2475 
2476 static DisasJumpType op_insi(DisasContext *s, DisasOps *o)
2477 {
2478     int shift = s->insn->data & 0xff;
2479     int size = s->insn->data >> 8;
2480     tcg_gen_deposit_i64(o->out, o->in1, o->in2, shift, size);
2481     return DISAS_NEXT;
2482 }
2483 
2484 static DisasJumpType op_ipm(DisasContext *s, DisasOps *o)
2485 {
2486     TCGv_i64 t1, t2;
2487 
2488     gen_op_calc_cc(s);
2489     t1 = tcg_temp_new_i64();
2490     tcg_gen_extract_i64(t1, psw_mask, 40, 4);
2491     t2 = tcg_temp_new_i64();
2492     tcg_gen_extu_i32_i64(t2, cc_op);
2493     tcg_gen_deposit_i64(t1, t1, t2, 4, 60);
2494     tcg_gen_deposit_i64(o->out, o->out, t1, 24, 8);
2495     return DISAS_NEXT;
2496 }
2497 
2498 #ifndef CONFIG_USER_ONLY
2499 static DisasJumpType op_idte(DisasContext *s, DisasOps *o)
2500 {
2501     TCGv_i32 m4;
2502 
2503     if (s390_has_feat(S390_FEAT_LOCAL_TLB_CLEARING)) {
2504         m4 = tcg_constant_i32(get_field(s, m4));
2505     } else {
2506         m4 = tcg_constant_i32(0);
2507     }
2508     gen_helper_idte(tcg_env, o->in1, o->in2, m4);
2509     return DISAS_NEXT;
2510 }
2511 
2512 static DisasJumpType op_ipte(DisasContext *s, DisasOps *o)
2513 {
2514     TCGv_i32 m4;
2515 
2516     if (s390_has_feat(S390_FEAT_LOCAL_TLB_CLEARING)) {
2517         m4 = tcg_constant_i32(get_field(s, m4));
2518     } else {
2519         m4 = tcg_constant_i32(0);
2520     }
2521     gen_helper_ipte(tcg_env, o->in1, o->in2, m4);
2522     return DISAS_NEXT;
2523 }
2524 
2525 static DisasJumpType op_iske(DisasContext *s, DisasOps *o)
2526 {
2527     gen_helper_iske(o->out, tcg_env, o->in2);
2528     return DISAS_NEXT;
2529 }
2530 #endif
2531 
2532 static DisasJumpType op_msa(DisasContext *s, DisasOps *o)
2533 {
2534     int r1 = have_field(s, r1) ? get_field(s, r1) : 0;
2535     int r2 = have_field(s, r2) ? get_field(s, r2) : 0;
2536     int r3 = have_field(s, r3) ? get_field(s, r3) : 0;
2537     TCGv_i32 t_r1, t_r2, t_r3, type;
2538 
2539     switch (s->insn->data) {
2540     case S390_FEAT_TYPE_KMA:
2541         if (r3 == r1 || r3 == r2) {
2542             gen_program_exception(s, PGM_SPECIFICATION);
2543             return DISAS_NORETURN;
2544         }
2545         /* FALL THROUGH */
2546     case S390_FEAT_TYPE_KMCTR:
2547         if (r3 & 1 || !r3) {
2548             gen_program_exception(s, PGM_SPECIFICATION);
2549             return DISAS_NORETURN;
2550         }
2551         /* FALL THROUGH */
2552     case S390_FEAT_TYPE_PPNO:
2553     case S390_FEAT_TYPE_KMF:
2554     case S390_FEAT_TYPE_KMC:
2555     case S390_FEAT_TYPE_KMO:
2556     case S390_FEAT_TYPE_KM:
2557         if (r1 & 1 || !r1) {
2558             gen_program_exception(s, PGM_SPECIFICATION);
2559             return DISAS_NORETURN;
2560         }
2561         /* FALL THROUGH */
2562     case S390_FEAT_TYPE_KMAC:
2563     case S390_FEAT_TYPE_KIMD:
2564     case S390_FEAT_TYPE_KLMD:
2565         if (r2 & 1 || !r2) {
2566             gen_program_exception(s, PGM_SPECIFICATION);
2567             return DISAS_NORETURN;
2568         }
2569         /* FALL THROUGH */
2570     case S390_FEAT_TYPE_PCKMO:
2571     case S390_FEAT_TYPE_PCC:
2572         break;
2573     default:
2574         g_assert_not_reached();
2575     };
2576 
2577     t_r1 = tcg_constant_i32(r1);
2578     t_r2 = tcg_constant_i32(r2);
2579     t_r3 = tcg_constant_i32(r3);
2580     type = tcg_constant_i32(s->insn->data);
2581     gen_helper_msa(cc_op, tcg_env, t_r1, t_r2, t_r3, type);
2582     set_cc_static(s);
2583     return DISAS_NEXT;
2584 }
2585 
2586 static DisasJumpType op_keb(DisasContext *s, DisasOps *o)
2587 {
2588     gen_helper_keb(cc_op, tcg_env, o->in1, o->in2);
2589     set_cc_static(s);
2590     return DISAS_NEXT;
2591 }
2592 
2593 static DisasJumpType op_kdb(DisasContext *s, DisasOps *o)
2594 {
2595     gen_helper_kdb(cc_op, tcg_env, o->in1, o->in2);
2596     set_cc_static(s);
2597     return DISAS_NEXT;
2598 }
2599 
2600 static DisasJumpType op_kxb(DisasContext *s, DisasOps *o)
2601 {
2602     gen_helper_kxb(cc_op, tcg_env, o->in1_128, o->in2_128);
2603     set_cc_static(s);
2604     return DISAS_NEXT;
2605 }
2606 
2607 static DisasJumpType help_laa(DisasContext *s, DisasOps *o, bool addu64)
2608 {
2609     /* The real output is indeed the original value in memory;
2610        recompute the addition for the computation of CC.  */
2611     tcg_gen_atomic_fetch_add_i64(o->in2, o->in2, o->in1, get_mem_index(s),
2612                                  s->insn->data | MO_ALIGN);
2613     /* However, we need to recompute the addition for setting CC.  */
2614     if (addu64) {
2615         tcg_gen_movi_i64(cc_src, 0);
2616         tcg_gen_add2_i64(o->out, cc_src, o->in1, cc_src, o->in2, cc_src);
2617     } else {
2618         tcg_gen_add_i64(o->out, o->in1, o->in2);
2619     }
2620     return DISAS_NEXT;
2621 }
2622 
2623 static DisasJumpType op_laa(DisasContext *s, DisasOps *o)
2624 {
2625     return help_laa(s, o, false);
2626 }
2627 
2628 static DisasJumpType op_laa_addu64(DisasContext *s, DisasOps *o)
2629 {
2630     return help_laa(s, o, true);
2631 }
2632 
2633 static DisasJumpType op_lan(DisasContext *s, DisasOps *o)
2634 {
2635     /* The real output is indeed the original value in memory;
2636        recompute the addition for the computation of CC.  */
2637     tcg_gen_atomic_fetch_and_i64(o->in2, o->in2, o->in1, get_mem_index(s),
2638                                  s->insn->data | MO_ALIGN);
2639     /* However, we need to recompute the operation for setting CC.  */
2640     tcg_gen_and_i64(o->out, o->in1, o->in2);
2641     return DISAS_NEXT;
2642 }
2643 
2644 static DisasJumpType op_lao(DisasContext *s, DisasOps *o)
2645 {
2646     /* The real output is indeed the original value in memory;
2647        recompute the addition for the computation of CC.  */
2648     tcg_gen_atomic_fetch_or_i64(o->in2, o->in2, o->in1, get_mem_index(s),
2649                                 s->insn->data | MO_ALIGN);
2650     /* However, we need to recompute the operation for setting CC.  */
2651     tcg_gen_or_i64(o->out, o->in1, o->in2);
2652     return DISAS_NEXT;
2653 }
2654 
2655 static DisasJumpType op_lax(DisasContext *s, DisasOps *o)
2656 {
2657     /* The real output is indeed the original value in memory;
2658        recompute the addition for the computation of CC.  */
2659     tcg_gen_atomic_fetch_xor_i64(o->in2, o->in2, o->in1, get_mem_index(s),
2660                                  s->insn->data | MO_ALIGN);
2661     /* However, we need to recompute the operation for setting CC.  */
2662     tcg_gen_xor_i64(o->out, o->in1, o->in2);
2663     return DISAS_NEXT;
2664 }
2665 
2666 static DisasJumpType op_ldeb(DisasContext *s, DisasOps *o)
2667 {
2668     gen_helper_ldeb(o->out, tcg_env, o->in2);
2669     return DISAS_NEXT;
2670 }
2671 
2672 static DisasJumpType op_ledb(DisasContext *s, DisasOps *o)
2673 {
2674     TCGv_i32 m34 = fpinst_extract_m34(s, true, true);
2675 
2676     if (!m34) {
2677         return DISAS_NORETURN;
2678     }
2679     gen_helper_ledb(o->out, tcg_env, o->in2, m34);
2680     return DISAS_NEXT;
2681 }
2682 
2683 static DisasJumpType op_ldxb(DisasContext *s, DisasOps *o)
2684 {
2685     TCGv_i32 m34 = fpinst_extract_m34(s, true, true);
2686 
2687     if (!m34) {
2688         return DISAS_NORETURN;
2689     }
2690     gen_helper_ldxb(o->out, tcg_env, o->in2_128, m34);
2691     return DISAS_NEXT;
2692 }
2693 
2694 static DisasJumpType op_lexb(DisasContext *s, DisasOps *o)
2695 {
2696     TCGv_i32 m34 = fpinst_extract_m34(s, true, true);
2697 
2698     if (!m34) {
2699         return DISAS_NORETURN;
2700     }
2701     gen_helper_lexb(o->out, tcg_env, o->in2_128, m34);
2702     return DISAS_NEXT;
2703 }
2704 
2705 static DisasJumpType op_lxdb(DisasContext *s, DisasOps *o)
2706 {
2707     gen_helper_lxdb(o->out_128, tcg_env, o->in2);
2708     return DISAS_NEXT;
2709 }
2710 
2711 static DisasJumpType op_lxeb(DisasContext *s, DisasOps *o)
2712 {
2713     gen_helper_lxeb(o->out_128, tcg_env, o->in2);
2714     return DISAS_NEXT;
2715 }
2716 
2717 static DisasJumpType op_lde(DisasContext *s, DisasOps *o)
2718 {
2719     tcg_gen_shli_i64(o->out, o->in2, 32);
2720     return DISAS_NEXT;
2721 }
2722 
2723 static DisasJumpType op_llgt(DisasContext *s, DisasOps *o)
2724 {
2725     tcg_gen_andi_i64(o->out, o->in2, 0x7fffffff);
2726     return DISAS_NEXT;
2727 }
2728 
2729 static DisasJumpType op_ld8s(DisasContext *s, DisasOps *o)
2730 {
2731     tcg_gen_qemu_ld_i64(o->out, o->in2, get_mem_index(s), MO_SB);
2732     return DISAS_NEXT;
2733 }
2734 
2735 static DisasJumpType op_ld8u(DisasContext *s, DisasOps *o)
2736 {
2737     tcg_gen_qemu_ld_i64(o->out, o->in2, get_mem_index(s), MO_UB);
2738     return DISAS_NEXT;
2739 }
2740 
2741 static DisasJumpType op_ld16s(DisasContext *s, DisasOps *o)
2742 {
2743     tcg_gen_qemu_ld_i64(o->out, o->in2, get_mem_index(s), MO_TESW);
2744     return DISAS_NEXT;
2745 }
2746 
2747 static DisasJumpType op_ld16u(DisasContext *s, DisasOps *o)
2748 {
2749     tcg_gen_qemu_ld_i64(o->out, o->in2, get_mem_index(s), MO_TEUW);
2750     return DISAS_NEXT;
2751 }
2752 
2753 static DisasJumpType op_ld32s(DisasContext *s, DisasOps *o)
2754 {
2755     tcg_gen_qemu_ld_tl(o->out, o->in2, get_mem_index(s),
2756                        MO_TESL | s->insn->data);
2757     return DISAS_NEXT;
2758 }
2759 
2760 static DisasJumpType op_ld32u(DisasContext *s, DisasOps *o)
2761 {
2762     tcg_gen_qemu_ld_tl(o->out, o->in2, get_mem_index(s),
2763                        MO_TEUL | s->insn->data);
2764     return DISAS_NEXT;
2765 }
2766 
2767 static DisasJumpType op_ld64(DisasContext *s, DisasOps *o)
2768 {
2769     tcg_gen_qemu_ld_i64(o->out, o->in2, get_mem_index(s),
2770                         MO_TEUQ | s->insn->data);
2771     return DISAS_NEXT;
2772 }
2773 
2774 static DisasJumpType op_lat(DisasContext *s, DisasOps *o)
2775 {
2776     TCGLabel *lab = gen_new_label();
2777     store_reg32_i64(get_field(s, r1), o->in2);
2778     /* The value is stored even in case of trap. */
2779     tcg_gen_brcondi_i64(TCG_COND_NE, o->in2, 0, lab);
2780     gen_trap(s);
2781     gen_set_label(lab);
2782     return DISAS_NEXT;
2783 }
2784 
2785 static DisasJumpType op_lgat(DisasContext *s, DisasOps *o)
2786 {
2787     TCGLabel *lab = gen_new_label();
2788     tcg_gen_qemu_ld_i64(o->out, o->in2, get_mem_index(s), MO_TEUQ);
2789     /* The value is stored even in case of trap. */
2790     tcg_gen_brcondi_i64(TCG_COND_NE, o->out, 0, lab);
2791     gen_trap(s);
2792     gen_set_label(lab);
2793     return DISAS_NEXT;
2794 }
2795 
2796 static DisasJumpType op_lfhat(DisasContext *s, DisasOps *o)
2797 {
2798     TCGLabel *lab = gen_new_label();
2799     store_reg32h_i64(get_field(s, r1), o->in2);
2800     /* The value is stored even in case of trap. */
2801     tcg_gen_brcondi_i64(TCG_COND_NE, o->in2, 0, lab);
2802     gen_trap(s);
2803     gen_set_label(lab);
2804     return DISAS_NEXT;
2805 }
2806 
2807 static DisasJumpType op_llgfat(DisasContext *s, DisasOps *o)
2808 {
2809     TCGLabel *lab = gen_new_label();
2810 
2811     tcg_gen_qemu_ld_i64(o->out, o->in2, get_mem_index(s), MO_TEUL);
2812     /* The value is stored even in case of trap. */
2813     tcg_gen_brcondi_i64(TCG_COND_NE, o->out, 0, lab);
2814     gen_trap(s);
2815     gen_set_label(lab);
2816     return DISAS_NEXT;
2817 }
2818 
2819 static DisasJumpType op_llgtat(DisasContext *s, DisasOps *o)
2820 {
2821     TCGLabel *lab = gen_new_label();
2822     tcg_gen_andi_i64(o->out, o->in2, 0x7fffffff);
2823     /* The value is stored even in case of trap. */
2824     tcg_gen_brcondi_i64(TCG_COND_NE, o->out, 0, lab);
2825     gen_trap(s);
2826     gen_set_label(lab);
2827     return DISAS_NEXT;
2828 }
2829 
2830 static DisasJumpType op_loc(DisasContext *s, DisasOps *o)
2831 {
2832     DisasCompare c;
2833 
2834     if (have_field(s, m3)) {
2835         /* LOAD * ON CONDITION */
2836         disas_jcc(s, &c, get_field(s, m3));
2837     } else {
2838         /* SELECT */
2839         disas_jcc(s, &c, get_field(s, m4));
2840     }
2841 
2842     if (c.is_64) {
2843         tcg_gen_movcond_i64(c.cond, o->out, c.u.s64.a, c.u.s64.b,
2844                             o->in2, o->in1);
2845     } else {
2846         TCGv_i32 t32 = tcg_temp_new_i32();
2847         TCGv_i64 t, z;
2848 
2849         tcg_gen_setcond_i32(c.cond, t32, c.u.s32.a, c.u.s32.b);
2850 
2851         t = tcg_temp_new_i64();
2852         tcg_gen_extu_i32_i64(t, t32);
2853 
2854         z = tcg_constant_i64(0);
2855         tcg_gen_movcond_i64(TCG_COND_NE, o->out, t, z, o->in2, o->in1);
2856     }
2857 
2858     return DISAS_NEXT;
2859 }
2860 
2861 #ifndef CONFIG_USER_ONLY
2862 static DisasJumpType op_lctl(DisasContext *s, DisasOps *o)
2863 {
2864     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
2865     TCGv_i32 r3 = tcg_constant_i32(get_field(s, r3));
2866 
2867     gen_helper_lctl(tcg_env, r1, o->in2, r3);
2868     /* Exit to main loop to reevaluate s390_cpu_exec_interrupt.  */
2869     s->exit_to_mainloop = true;
2870     return DISAS_TOO_MANY;
2871 }
2872 
2873 static DisasJumpType op_lctlg(DisasContext *s, DisasOps *o)
2874 {
2875     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
2876     TCGv_i32 r3 = tcg_constant_i32(get_field(s, r3));
2877 
2878     gen_helper_lctlg(tcg_env, r1, o->in2, r3);
2879     /* Exit to main loop to reevaluate s390_cpu_exec_interrupt.  */
2880     s->exit_to_mainloop = true;
2881     return DISAS_TOO_MANY;
2882 }
2883 
2884 static DisasJumpType op_lra(DisasContext *s, DisasOps *o)
2885 {
2886     gen_helper_lra(o->out, tcg_env, o->out, o->in2);
2887     set_cc_static(s);
2888     return DISAS_NEXT;
2889 }
2890 
2891 static DisasJumpType op_lpp(DisasContext *s, DisasOps *o)
2892 {
2893     tcg_gen_st_i64(o->in2, tcg_env, offsetof(CPUS390XState, pp));
2894     return DISAS_NEXT;
2895 }
2896 
2897 static DisasJumpType op_lpsw(DisasContext *s, DisasOps *o)
2898 {
2899     TCGv_i64 mask, addr;
2900 
2901     per_breaking_event(s);
2902 
2903     /*
2904      * Convert the short PSW into the normal PSW, similar to what
2905      * s390_cpu_load_normal() does.
2906      */
2907     mask = tcg_temp_new_i64();
2908     addr = tcg_temp_new_i64();
2909     tcg_gen_qemu_ld_i64(mask, o->in2, get_mem_index(s), MO_TEUQ | MO_ALIGN_8);
2910     tcg_gen_andi_i64(addr, mask, PSW_MASK_SHORT_ADDR);
2911     tcg_gen_andi_i64(mask, mask, PSW_MASK_SHORT_CTRL);
2912     tcg_gen_xori_i64(mask, mask, PSW_MASK_SHORTPSW);
2913     gen_helper_load_psw(tcg_env, mask, addr);
2914     return DISAS_NORETURN;
2915 }
2916 
2917 static DisasJumpType op_lpswe(DisasContext *s, DisasOps *o)
2918 {
2919     TCGv_i64 t1, t2;
2920 
2921     per_breaking_event(s);
2922 
2923     t1 = tcg_temp_new_i64();
2924     t2 = tcg_temp_new_i64();
2925     tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s),
2926                         MO_TEUQ | MO_ALIGN_8);
2927     tcg_gen_addi_i64(o->in2, o->in2, 8);
2928     tcg_gen_qemu_ld_i64(t2, o->in2, get_mem_index(s), MO_TEUQ);
2929     gen_helper_load_psw(tcg_env, t1, t2);
2930     return DISAS_NORETURN;
2931 }
2932 #endif
2933 
2934 static DisasJumpType op_lam(DisasContext *s, DisasOps *o)
2935 {
2936     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
2937     TCGv_i32 r3 = tcg_constant_i32(get_field(s, r3));
2938 
2939     gen_helper_lam(tcg_env, r1, o->in2, r3);
2940     return DISAS_NEXT;
2941 }
2942 
2943 static DisasJumpType op_lm32(DisasContext *s, DisasOps *o)
2944 {
2945     int r1 = get_field(s, r1);
2946     int r3 = get_field(s, r3);
2947     TCGv_i64 t1, t2;
2948 
2949     /* Only one register to read. */
2950     t1 = tcg_temp_new_i64();
2951     if (unlikely(r1 == r3)) {
2952         tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUL);
2953         store_reg32_i64(r1, t1);
2954         return DISAS_NEXT;
2955     }
2956 
2957     /* First load the values of the first and last registers to trigger
2958        possible page faults. */
2959     t2 = tcg_temp_new_i64();
2960     tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUL);
2961     tcg_gen_addi_i64(t2, o->in2, 4 * ((r3 - r1) & 15));
2962     tcg_gen_qemu_ld_i64(t2, t2, get_mem_index(s), MO_TEUL);
2963     store_reg32_i64(r1, t1);
2964     store_reg32_i64(r3, t2);
2965 
2966     /* Only two registers to read. */
2967     if (((r1 + 1) & 15) == r3) {
2968         return DISAS_NEXT;
2969     }
2970 
2971     /* Then load the remaining registers. Page fault can't occur. */
2972     r3 = (r3 - 1) & 15;
2973     tcg_gen_movi_i64(t2, 4);
2974     while (r1 != r3) {
2975         r1 = (r1 + 1) & 15;
2976         tcg_gen_add_i64(o->in2, o->in2, t2);
2977         tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUL);
2978         store_reg32_i64(r1, t1);
2979     }
2980     return DISAS_NEXT;
2981 }
2982 
2983 static DisasJumpType op_lmh(DisasContext *s, DisasOps *o)
2984 {
2985     int r1 = get_field(s, r1);
2986     int r3 = get_field(s, r3);
2987     TCGv_i64 t1, t2;
2988 
2989     /* Only one register to read. */
2990     t1 = tcg_temp_new_i64();
2991     if (unlikely(r1 == r3)) {
2992         tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUL);
2993         store_reg32h_i64(r1, t1);
2994         return DISAS_NEXT;
2995     }
2996 
2997     /* First load the values of the first and last registers to trigger
2998        possible page faults. */
2999     t2 = tcg_temp_new_i64();
3000     tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUL);
3001     tcg_gen_addi_i64(t2, o->in2, 4 * ((r3 - r1) & 15));
3002     tcg_gen_qemu_ld_i64(t2, t2, get_mem_index(s), MO_TEUL);
3003     store_reg32h_i64(r1, t1);
3004     store_reg32h_i64(r3, t2);
3005 
3006     /* Only two registers to read. */
3007     if (((r1 + 1) & 15) == r3) {
3008         return DISAS_NEXT;
3009     }
3010 
3011     /* Then load the remaining registers. Page fault can't occur. */
3012     r3 = (r3 - 1) & 15;
3013     tcg_gen_movi_i64(t2, 4);
3014     while (r1 != r3) {
3015         r1 = (r1 + 1) & 15;
3016         tcg_gen_add_i64(o->in2, o->in2, t2);
3017         tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUL);
3018         store_reg32h_i64(r1, t1);
3019     }
3020     return DISAS_NEXT;
3021 }
3022 
3023 static DisasJumpType op_lm64(DisasContext *s, DisasOps *o)
3024 {
3025     int r1 = get_field(s, r1);
3026     int r3 = get_field(s, r3);
3027     TCGv_i64 t1, t2;
3028 
3029     /* Only one register to read. */
3030     if (unlikely(r1 == r3)) {
3031         tcg_gen_qemu_ld_i64(regs[r1], o->in2, get_mem_index(s), MO_TEUQ);
3032         return DISAS_NEXT;
3033     }
3034 
3035     /* First load the values of the first and last registers to trigger
3036        possible page faults. */
3037     t1 = tcg_temp_new_i64();
3038     t2 = tcg_temp_new_i64();
3039     tcg_gen_qemu_ld_i64(t1, o->in2, get_mem_index(s), MO_TEUQ);
3040     tcg_gen_addi_i64(t2, o->in2, 8 * ((r3 - r1) & 15));
3041     tcg_gen_qemu_ld_i64(regs[r3], t2, get_mem_index(s), MO_TEUQ);
3042     tcg_gen_mov_i64(regs[r1], t1);
3043 
3044     /* Only two registers to read. */
3045     if (((r1 + 1) & 15) == r3) {
3046         return DISAS_NEXT;
3047     }
3048 
3049     /* Then load the remaining registers. Page fault can't occur. */
3050     r3 = (r3 - 1) & 15;
3051     tcg_gen_movi_i64(t1, 8);
3052     while (r1 != r3) {
3053         r1 = (r1 + 1) & 15;
3054         tcg_gen_add_i64(o->in2, o->in2, t1);
3055         tcg_gen_qemu_ld_i64(regs[r1], o->in2, get_mem_index(s), MO_TEUQ);
3056     }
3057     return DISAS_NEXT;
3058 }
3059 
3060 static DisasJumpType op_lpd(DisasContext *s, DisasOps *o)
3061 {
3062     TCGv_i64 a1, a2;
3063     MemOp mop = s->insn->data;
3064 
3065     /* In a parallel context, stop the world and single step.  */
3066     if (tb_cflags(s->base.tb) & CF_PARALLEL) {
3067         update_psw_addr(s);
3068         update_cc_op(s);
3069         gen_exception(EXCP_ATOMIC);
3070         return DISAS_NORETURN;
3071     }
3072 
3073     /* In a serial context, perform the two loads ... */
3074     a1 = get_address(s, 0, get_field(s, b1), get_field(s, d1));
3075     a2 = get_address(s, 0, get_field(s, b2), get_field(s, d2));
3076     tcg_gen_qemu_ld_i64(o->out, a1, get_mem_index(s), mop | MO_ALIGN);
3077     tcg_gen_qemu_ld_i64(o->out2, a2, get_mem_index(s), mop | MO_ALIGN);
3078 
3079     /* ... and indicate that we performed them while interlocked.  */
3080     gen_op_movi_cc(s, 0);
3081     return DISAS_NEXT;
3082 }
3083 
3084 static DisasJumpType op_lpq(DisasContext *s, DisasOps *o)
3085 {
3086     o->out_128 = tcg_temp_new_i128();
3087     tcg_gen_qemu_ld_i128(o->out_128, o->in2, get_mem_index(s),
3088                          MO_TE | MO_128 | MO_ALIGN);
3089     return DISAS_NEXT;
3090 }
3091 
3092 #ifndef CONFIG_USER_ONLY
3093 static DisasJumpType op_lura(DisasContext *s, DisasOps *o)
3094 {
3095     tcg_gen_qemu_ld_tl(o->out, o->in2, MMU_REAL_IDX, s->insn->data);
3096     return DISAS_NEXT;
3097 }
3098 #endif
3099 
3100 static DisasJumpType op_lzrb(DisasContext *s, DisasOps *o)
3101 {
3102     tcg_gen_andi_i64(o->out, o->in2, -256);
3103     return DISAS_NEXT;
3104 }
3105 
3106 static DisasJumpType op_lcbb(DisasContext *s, DisasOps *o)
3107 {
3108     const int64_t block_size = (1ull << (get_field(s, m3) + 6));
3109 
3110     if (get_field(s, m3) > 6) {
3111         gen_program_exception(s, PGM_SPECIFICATION);
3112         return DISAS_NORETURN;
3113     }
3114 
3115     tcg_gen_ori_i64(o->addr1, o->addr1, -block_size);
3116     tcg_gen_neg_i64(o->addr1, o->addr1);
3117     tcg_gen_movi_i64(o->out, 16);
3118     tcg_gen_umin_i64(o->out, o->out, o->addr1);
3119     gen_op_update1_cc_i64(s, CC_OP_LCBB, o->out);
3120     return DISAS_NEXT;
3121 }
3122 
3123 static DisasJumpType op_mc(DisasContext *s, DisasOps *o)
3124 {
3125     const uint8_t monitor_class = get_field(s, i2);
3126 
3127     if (monitor_class & 0xf0) {
3128         gen_program_exception(s, PGM_SPECIFICATION);
3129         return DISAS_NORETURN;
3130     }
3131 
3132 #if !defined(CONFIG_USER_ONLY)
3133     gen_helper_monitor_call(tcg_env, o->addr1,
3134                             tcg_constant_i32(monitor_class));
3135 #endif
3136     /* Defaults to a NOP. */
3137     return DISAS_NEXT;
3138 }
3139 
3140 static DisasJumpType op_mov2(DisasContext *s, DisasOps *o)
3141 {
3142     o->out = o->in2;
3143     o->in2 = NULL;
3144     return DISAS_NEXT;
3145 }
3146 
3147 static DisasJumpType op_mov2e(DisasContext *s, DisasOps *o)
3148 {
3149     int b2 = get_field(s, b2);
3150     TCGv ar1 = tcg_temp_new_i64();
3151     int r1 = get_field(s, r1);
3152 
3153     o->out = o->in2;
3154     o->in2 = NULL;
3155 
3156     switch (s->base.tb->flags & FLAG_MASK_ASC) {
3157     case PSW_ASC_PRIMARY >> FLAG_MASK_PSW_SHIFT:
3158         tcg_gen_movi_i64(ar1, 0);
3159         break;
3160     case PSW_ASC_ACCREG >> FLAG_MASK_PSW_SHIFT:
3161         tcg_gen_movi_i64(ar1, 1);
3162         break;
3163     case PSW_ASC_SECONDARY >> FLAG_MASK_PSW_SHIFT:
3164         if (b2) {
3165             tcg_gen_ld32u_i64(ar1, tcg_env, offsetof(CPUS390XState, aregs[b2]));
3166         } else {
3167             tcg_gen_movi_i64(ar1, 0);
3168         }
3169         break;
3170     case PSW_ASC_HOME >> FLAG_MASK_PSW_SHIFT:
3171         tcg_gen_movi_i64(ar1, 2);
3172         break;
3173     }
3174 
3175     tcg_gen_st32_i64(ar1, tcg_env, offsetof(CPUS390XState, aregs[r1]));
3176     return DISAS_NEXT;
3177 }
3178 
3179 static DisasJumpType op_movx(DisasContext *s, DisasOps *o)
3180 {
3181     o->out = o->in1;
3182     o->out2 = o->in2;
3183     o->in1 = NULL;
3184     o->in2 = NULL;
3185     return DISAS_NEXT;
3186 }
3187 
3188 static DisasJumpType op_mvc(DisasContext *s, DisasOps *o)
3189 {
3190     TCGv_i32 l = tcg_constant_i32(get_field(s, l1));
3191 
3192     gen_helper_mvc(tcg_env, l, o->addr1, o->in2);
3193     return DISAS_NEXT;
3194 }
3195 
3196 static DisasJumpType op_mvcrl(DisasContext *s, DisasOps *o)
3197 {
3198     gen_helper_mvcrl(tcg_env, regs[0], o->addr1, o->in2);
3199     return DISAS_NEXT;
3200 }
3201 
3202 static DisasJumpType op_mvcin(DisasContext *s, DisasOps *o)
3203 {
3204     TCGv_i32 l = tcg_constant_i32(get_field(s, l1));
3205 
3206     gen_helper_mvcin(tcg_env, l, o->addr1, o->in2);
3207     return DISAS_NEXT;
3208 }
3209 
3210 static DisasJumpType op_mvcl(DisasContext *s, DisasOps *o)
3211 {
3212     int r1 = get_field(s, r1);
3213     int r2 = get_field(s, r2);
3214     TCGv_i32 t1, t2;
3215 
3216     /* r1 and r2 must be even.  */
3217     if (r1 & 1 || r2 & 1) {
3218         gen_program_exception(s, PGM_SPECIFICATION);
3219         return DISAS_NORETURN;
3220     }
3221 
3222     t1 = tcg_constant_i32(r1);
3223     t2 = tcg_constant_i32(r2);
3224     gen_helper_mvcl(cc_op, tcg_env, t1, t2);
3225     set_cc_static(s);
3226     return DISAS_NEXT;
3227 }
3228 
3229 static DisasJumpType op_mvcle(DisasContext *s, DisasOps *o)
3230 {
3231     int r1 = get_field(s, r1);
3232     int r3 = get_field(s, r3);
3233     TCGv_i32 t1, t3;
3234 
3235     /* r1 and r3 must be even.  */
3236     if (r1 & 1 || r3 & 1) {
3237         gen_program_exception(s, PGM_SPECIFICATION);
3238         return DISAS_NORETURN;
3239     }
3240 
3241     t1 = tcg_constant_i32(r1);
3242     t3 = tcg_constant_i32(r3);
3243     gen_helper_mvcle(cc_op, tcg_env, t1, o->in2, t3);
3244     set_cc_static(s);
3245     return DISAS_NEXT;
3246 }
3247 
3248 static DisasJumpType op_mvclu(DisasContext *s, DisasOps *o)
3249 {
3250     int r1 = get_field(s, r1);
3251     int r3 = get_field(s, r3);
3252     TCGv_i32 t1, t3;
3253 
3254     /* r1 and r3 must be even.  */
3255     if (r1 & 1 || r3 & 1) {
3256         gen_program_exception(s, PGM_SPECIFICATION);
3257         return DISAS_NORETURN;
3258     }
3259 
3260     t1 = tcg_constant_i32(r1);
3261     t3 = tcg_constant_i32(r3);
3262     gen_helper_mvclu(cc_op, tcg_env, t1, o->in2, t3);
3263     set_cc_static(s);
3264     return DISAS_NEXT;
3265 }
3266 
3267 static DisasJumpType op_mvcos(DisasContext *s, DisasOps *o)
3268 {
3269     int r3 = get_field(s, r3);
3270     gen_helper_mvcos(cc_op, tcg_env, o->addr1, o->in2, regs[r3]);
3271     set_cc_static(s);
3272     return DISAS_NEXT;
3273 }
3274 
3275 #ifndef CONFIG_USER_ONLY
3276 static DisasJumpType op_mvcp(DisasContext *s, DisasOps *o)
3277 {
3278     int r1 = get_field(s, l1);
3279     int r3 = get_field(s, r3);
3280     gen_helper_mvcp(cc_op, tcg_env, regs[r1], o->addr1, o->in2, regs[r3]);
3281     set_cc_static(s);
3282     return DISAS_NEXT;
3283 }
3284 
3285 static DisasJumpType op_mvcs(DisasContext *s, DisasOps *o)
3286 {
3287     int r1 = get_field(s, l1);
3288     int r3 = get_field(s, r3);
3289     gen_helper_mvcs(cc_op, tcg_env, regs[r1], o->addr1, o->in2, regs[r3]);
3290     set_cc_static(s);
3291     return DISAS_NEXT;
3292 }
3293 #endif
3294 
3295 static DisasJumpType op_mvn(DisasContext *s, DisasOps *o)
3296 {
3297     TCGv_i32 l = tcg_constant_i32(get_field(s, l1));
3298 
3299     gen_helper_mvn(tcg_env, l, o->addr1, o->in2);
3300     return DISAS_NEXT;
3301 }
3302 
3303 static DisasJumpType op_mvo(DisasContext *s, DisasOps *o)
3304 {
3305     TCGv_i32 l = tcg_constant_i32(get_field(s, l1));
3306 
3307     gen_helper_mvo(tcg_env, l, o->addr1, o->in2);
3308     return DISAS_NEXT;
3309 }
3310 
3311 static DisasJumpType op_mvpg(DisasContext *s, DisasOps *o)
3312 {
3313     TCGv_i32 t1 = tcg_constant_i32(get_field(s, r1));
3314     TCGv_i32 t2 = tcg_constant_i32(get_field(s, r2));
3315 
3316     gen_helper_mvpg(cc_op, tcg_env, regs[0], t1, t2);
3317     set_cc_static(s);
3318     return DISAS_NEXT;
3319 }
3320 
3321 static DisasJumpType op_mvst(DisasContext *s, DisasOps *o)
3322 {
3323     TCGv_i32 t1 = tcg_constant_i32(get_field(s, r1));
3324     TCGv_i32 t2 = tcg_constant_i32(get_field(s, r2));
3325 
3326     gen_helper_mvst(cc_op, tcg_env, t1, t2);
3327     set_cc_static(s);
3328     return DISAS_NEXT;
3329 }
3330 
3331 static DisasJumpType op_mvz(DisasContext *s, DisasOps *o)
3332 {
3333     TCGv_i32 l = tcg_constant_i32(get_field(s, l1));
3334 
3335     gen_helper_mvz(tcg_env, l, o->addr1, o->in2);
3336     return DISAS_NEXT;
3337 }
3338 
3339 static DisasJumpType op_mul(DisasContext *s, DisasOps *o)
3340 {
3341     tcg_gen_mul_i64(o->out, o->in1, o->in2);
3342     return DISAS_NEXT;
3343 }
3344 
3345 static DisasJumpType op_mul128(DisasContext *s, DisasOps *o)
3346 {
3347     tcg_gen_mulu2_i64(o->out2, o->out, o->in1, o->in2);
3348     return DISAS_NEXT;
3349 }
3350 
3351 static DisasJumpType op_muls128(DisasContext *s, DisasOps *o)
3352 {
3353     tcg_gen_muls2_i64(o->out2, o->out, o->in1, o->in2);
3354     return DISAS_NEXT;
3355 }
3356 
3357 static DisasJumpType op_meeb(DisasContext *s, DisasOps *o)
3358 {
3359     gen_helper_meeb(o->out, tcg_env, o->in1, o->in2);
3360     return DISAS_NEXT;
3361 }
3362 
3363 static DisasJumpType op_mdeb(DisasContext *s, DisasOps *o)
3364 {
3365     gen_helper_mdeb(o->out, tcg_env, o->in1, o->in2);
3366     return DISAS_NEXT;
3367 }
3368 
3369 static DisasJumpType op_mdb(DisasContext *s, DisasOps *o)
3370 {
3371     gen_helper_mdb(o->out, tcg_env, o->in1, o->in2);
3372     return DISAS_NEXT;
3373 }
3374 
3375 static DisasJumpType op_mxb(DisasContext *s, DisasOps *o)
3376 {
3377     gen_helper_mxb(o->out_128, tcg_env, o->in1_128, o->in2_128);
3378     return DISAS_NEXT;
3379 }
3380 
3381 static DisasJumpType op_mxdb(DisasContext *s, DisasOps *o)
3382 {
3383     gen_helper_mxdb(o->out_128, tcg_env, o->in1, o->in2);
3384     return DISAS_NEXT;
3385 }
3386 
3387 static DisasJumpType op_maeb(DisasContext *s, DisasOps *o)
3388 {
3389     TCGv_i64 r3 = load_freg32_i64(get_field(s, r3));
3390     gen_helper_maeb(o->out, tcg_env, o->in1, o->in2, r3);
3391     return DISAS_NEXT;
3392 }
3393 
3394 static DisasJumpType op_madb(DisasContext *s, DisasOps *o)
3395 {
3396     TCGv_i64 r3 = load_freg(get_field(s, r3));
3397     gen_helper_madb(o->out, tcg_env, o->in1, o->in2, r3);
3398     return DISAS_NEXT;
3399 }
3400 
3401 static DisasJumpType op_mseb(DisasContext *s, DisasOps *o)
3402 {
3403     TCGv_i64 r3 = load_freg32_i64(get_field(s, r3));
3404     gen_helper_mseb(o->out, tcg_env, o->in1, o->in2, r3);
3405     return DISAS_NEXT;
3406 }
3407 
3408 static DisasJumpType op_msdb(DisasContext *s, DisasOps *o)
3409 {
3410     TCGv_i64 r3 = load_freg(get_field(s, r3));
3411     gen_helper_msdb(o->out, tcg_env, o->in1, o->in2, r3);
3412     return DISAS_NEXT;
3413 }
3414 
3415 static DisasJumpType op_nabs(DisasContext *s, DisasOps *o)
3416 {
3417     TCGv_i64 z = tcg_constant_i64(0);
3418     TCGv_i64 n = tcg_temp_new_i64();
3419 
3420     tcg_gen_neg_i64(n, o->in2);
3421     tcg_gen_movcond_i64(TCG_COND_GE, o->out, o->in2, z, n, o->in2);
3422     return DISAS_NEXT;
3423 }
3424 
3425 static DisasJumpType op_nabsf32(DisasContext *s, DisasOps *o)
3426 {
3427     tcg_gen_ori_i64(o->out, o->in2, 0x80000000ull);
3428     return DISAS_NEXT;
3429 }
3430 
3431 static DisasJumpType op_nabsf64(DisasContext *s, DisasOps *o)
3432 {
3433     tcg_gen_ori_i64(o->out, o->in2, 0x8000000000000000ull);
3434     return DISAS_NEXT;
3435 }
3436 
3437 static DisasJumpType op_nabsf128(DisasContext *s, DisasOps *o)
3438 {
3439     tcg_gen_ori_i64(o->out, o->in1, 0x8000000000000000ull);
3440     tcg_gen_mov_i64(o->out2, o->in2);
3441     return DISAS_NEXT;
3442 }
3443 
3444 static DisasJumpType op_nc(DisasContext *s, DisasOps *o)
3445 {
3446     TCGv_i32 l = tcg_constant_i32(get_field(s, l1));
3447 
3448     gen_helper_nc(cc_op, tcg_env, l, o->addr1, o->in2);
3449     set_cc_static(s);
3450     return DISAS_NEXT;
3451 }
3452 
3453 static DisasJumpType op_neg(DisasContext *s, DisasOps *o)
3454 {
3455     tcg_gen_neg_i64(o->out, o->in2);
3456     return DISAS_NEXT;
3457 }
3458 
3459 static DisasJumpType op_negf32(DisasContext *s, DisasOps *o)
3460 {
3461     tcg_gen_xori_i64(o->out, o->in2, 0x80000000ull);
3462     return DISAS_NEXT;
3463 }
3464 
3465 static DisasJumpType op_negf64(DisasContext *s, DisasOps *o)
3466 {
3467     tcg_gen_xori_i64(o->out, o->in2, 0x8000000000000000ull);
3468     return DISAS_NEXT;
3469 }
3470 
3471 static DisasJumpType op_negf128(DisasContext *s, DisasOps *o)
3472 {
3473     tcg_gen_xori_i64(o->out, o->in1, 0x8000000000000000ull);
3474     tcg_gen_mov_i64(o->out2, o->in2);
3475     return DISAS_NEXT;
3476 }
3477 
3478 static DisasJumpType op_oc(DisasContext *s, DisasOps *o)
3479 {
3480     TCGv_i32 l = tcg_constant_i32(get_field(s, l1));
3481 
3482     gen_helper_oc(cc_op, tcg_env, l, o->addr1, o->in2);
3483     set_cc_static(s);
3484     return DISAS_NEXT;
3485 }
3486 
3487 static DisasJumpType op_or(DisasContext *s, DisasOps *o)
3488 {
3489     tcg_gen_or_i64(o->out, o->in1, o->in2);
3490     return DISAS_NEXT;
3491 }
3492 
3493 static DisasJumpType op_ori(DisasContext *s, DisasOps *o)
3494 {
3495     int shift = s->insn->data & 0xff;
3496     int size = s->insn->data >> 8;
3497     uint64_t mask = ((1ull << size) - 1) << shift;
3498     TCGv_i64 t = tcg_temp_new_i64();
3499 
3500     tcg_gen_shli_i64(t, o->in2, shift);
3501     tcg_gen_or_i64(o->out, o->in1, t);
3502 
3503     /* Produce the CC from only the bits manipulated.  */
3504     tcg_gen_andi_i64(cc_dst, o->out, mask);
3505     set_cc_nz_u64(s, cc_dst);
3506     return DISAS_NEXT;
3507 }
3508 
3509 static DisasJumpType op_oi(DisasContext *s, DisasOps *o)
3510 {
3511     o->in1 = tcg_temp_new_i64();
3512 
3513     if (!s390_has_feat(S390_FEAT_INTERLOCKED_ACCESS_2)) {
3514         tcg_gen_qemu_ld_tl(o->in1, o->addr1, get_mem_index(s), s->insn->data);
3515     } else {
3516         /* Perform the atomic operation in memory. */
3517         tcg_gen_atomic_fetch_or_i64(o->in1, o->addr1, o->in2, get_mem_index(s),
3518                                     s->insn->data);
3519     }
3520 
3521     /* Recompute also for atomic case: needed for setting CC. */
3522     tcg_gen_or_i64(o->out, o->in1, o->in2);
3523 
3524     if (!s390_has_feat(S390_FEAT_INTERLOCKED_ACCESS_2)) {
3525         tcg_gen_qemu_st_tl(o->out, o->addr1, get_mem_index(s), s->insn->data);
3526     }
3527     return DISAS_NEXT;
3528 }
3529 
3530 static DisasJumpType op_pack(DisasContext *s, DisasOps *o)
3531 {
3532     TCGv_i32 l = tcg_constant_i32(get_field(s, l1));
3533 
3534     gen_helper_pack(tcg_env, l, o->addr1, o->in2);
3535     return DISAS_NEXT;
3536 }
3537 
3538 static DisasJumpType op_pka(DisasContext *s, DisasOps *o)
3539 {
3540     int l2 = get_field(s, l2) + 1;
3541     TCGv_i32 l;
3542 
3543     /* The length must not exceed 32 bytes.  */
3544     if (l2 > 32) {
3545         gen_program_exception(s, PGM_SPECIFICATION);
3546         return DISAS_NORETURN;
3547     }
3548     l = tcg_constant_i32(l2);
3549     gen_helper_pka(tcg_env, o->addr1, o->in2, l);
3550     return DISAS_NEXT;
3551 }
3552 
3553 static DisasJumpType op_pku(DisasContext *s, DisasOps *o)
3554 {
3555     int l2 = get_field(s, l2) + 1;
3556     TCGv_i32 l;
3557 
3558     /* The length must be even and should not exceed 64 bytes.  */
3559     if ((l2 & 1) || (l2 > 64)) {
3560         gen_program_exception(s, PGM_SPECIFICATION);
3561         return DISAS_NORETURN;
3562     }
3563     l = tcg_constant_i32(l2);
3564     gen_helper_pku(tcg_env, o->addr1, o->in2, l);
3565     return DISAS_NEXT;
3566 }
3567 
3568 static DisasJumpType op_popcnt(DisasContext *s, DisasOps *o)
3569 {
3570     const uint8_t m3 = get_field(s, m3);
3571 
3572     if ((m3 & 8) && s390_has_feat(S390_FEAT_MISC_INSTRUCTION_EXT3)) {
3573         tcg_gen_ctpop_i64(o->out, o->in2);
3574     } else {
3575         gen_helper_popcnt(o->out, o->in2);
3576     }
3577     return DISAS_NEXT;
3578 }
3579 
3580 #ifndef CONFIG_USER_ONLY
3581 static DisasJumpType op_ptlb(DisasContext *s, DisasOps *o)
3582 {
3583     gen_helper_ptlb(tcg_env);
3584     return DISAS_NEXT;
3585 }
3586 #endif
3587 
3588 static DisasJumpType op_risbg(DisasContext *s, DisasOps *o)
3589 {
3590     int i3 = get_field(s, i3);
3591     int i4 = get_field(s, i4);
3592     int i5 = get_field(s, i5);
3593     int do_zero = i4 & 0x80;
3594     uint64_t mask, imask, pmask;
3595     int pos, len, rot;
3596 
3597     /* Adjust the arguments for the specific insn.  */
3598     switch (s->fields.op2) {
3599     case 0x55: /* risbg */
3600     case 0x59: /* risbgn */
3601         i3 &= 63;
3602         i4 &= 63;
3603         pmask = ~0;
3604         break;
3605     case 0x5d: /* risbhg */
3606         i3 &= 31;
3607         i4 &= 31;
3608         pmask = 0xffffffff00000000ull;
3609         break;
3610     case 0x51: /* risblg */
3611         i3 = (i3 & 31) + 32;
3612         i4 = (i4 & 31) + 32;
3613         pmask = 0x00000000ffffffffull;
3614         break;
3615     default:
3616         g_assert_not_reached();
3617     }
3618 
3619     /* MASK is the set of bits to be inserted from R2. */
3620     if (i3 <= i4) {
3621         /* [0...i3---i4...63] */
3622         mask = (-1ull >> i3) & (-1ull << (63 - i4));
3623     } else {
3624         /* [0---i4...i3---63] */
3625         mask = (-1ull >> i3) | (-1ull << (63 - i4));
3626     }
3627     /* For RISBLG/RISBHG, the wrapping is limited to the high/low doubleword. */
3628     mask &= pmask;
3629 
3630     /* IMASK is the set of bits to be kept from R1.  In the case of the high/low
3631        insns, we need to keep the other half of the register.  */
3632     imask = ~mask | ~pmask;
3633     if (do_zero) {
3634         imask = ~pmask;
3635     }
3636 
3637     len = i4 - i3 + 1;
3638     pos = 63 - i4;
3639     rot = i5 & 63;
3640 
3641     /* In some cases we can implement this with extract.  */
3642     if (imask == 0 && pos == 0 && len > 0 && len <= rot) {
3643         tcg_gen_extract_i64(o->out, o->in2, 64 - rot, len);
3644         return DISAS_NEXT;
3645     }
3646 
3647     /* In some cases we can implement this with deposit.  */
3648     if (len > 0 && (imask == 0 || ~mask == imask)) {
3649         /* Note that we rotate the bits to be inserted to the lsb, not to
3650            the position as described in the PoO.  */
3651         rot = (rot - pos) & 63;
3652     } else {
3653         pos = -1;
3654     }
3655 
3656     /* Rotate the input as necessary.  */
3657     tcg_gen_rotli_i64(o->in2, o->in2, rot);
3658 
3659     /* Insert the selected bits into the output.  */
3660     if (pos >= 0) {
3661         if (imask == 0) {
3662             tcg_gen_deposit_z_i64(o->out, o->in2, pos, len);
3663         } else {
3664             tcg_gen_deposit_i64(o->out, o->out, o->in2, pos, len);
3665         }
3666     } else if (imask == 0) {
3667         tcg_gen_andi_i64(o->out, o->in2, mask);
3668     } else {
3669         tcg_gen_andi_i64(o->in2, o->in2, mask);
3670         tcg_gen_andi_i64(o->out, o->out, imask);
3671         tcg_gen_or_i64(o->out, o->out, o->in2);
3672     }
3673     return DISAS_NEXT;
3674 }
3675 
3676 static DisasJumpType op_rosbg(DisasContext *s, DisasOps *o)
3677 {
3678     int i3 = get_field(s, i3);
3679     int i4 = get_field(s, i4);
3680     int i5 = get_field(s, i5);
3681     TCGv_i64 orig_out;
3682     uint64_t mask;
3683 
3684     /* If this is a test-only form, arrange to discard the result.  */
3685     if (i3 & 0x80) {
3686         tcg_debug_assert(o->out != NULL);
3687         orig_out = o->out;
3688         o->out = tcg_temp_new_i64();
3689         tcg_gen_mov_i64(o->out, orig_out);
3690     }
3691 
3692     i3 &= 63;
3693     i4 &= 63;
3694     i5 &= 63;
3695 
3696     /* MASK is the set of bits to be operated on from R2.
3697        Take care for I3/I4 wraparound.  */
3698     mask = ~0ull >> i3;
3699     if (i3 <= i4) {
3700         mask ^= ~0ull >> i4 >> 1;
3701     } else {
3702         mask |= ~(~0ull >> i4 >> 1);
3703     }
3704 
3705     /* Rotate the input as necessary.  */
3706     tcg_gen_rotli_i64(o->in2, o->in2, i5);
3707 
3708     /* Operate.  */
3709     switch (s->fields.op2) {
3710     case 0x54: /* AND */
3711         tcg_gen_ori_i64(o->in2, o->in2, ~mask);
3712         tcg_gen_and_i64(o->out, o->out, o->in2);
3713         break;
3714     case 0x56: /* OR */
3715         tcg_gen_andi_i64(o->in2, o->in2, mask);
3716         tcg_gen_or_i64(o->out, o->out, o->in2);
3717         break;
3718     case 0x57: /* XOR */
3719         tcg_gen_andi_i64(o->in2, o->in2, mask);
3720         tcg_gen_xor_i64(o->out, o->out, o->in2);
3721         break;
3722     default:
3723         abort();
3724     }
3725 
3726     /* Set the CC.  */
3727     tcg_gen_andi_i64(cc_dst, o->out, mask);
3728     set_cc_nz_u64(s, cc_dst);
3729     return DISAS_NEXT;
3730 }
3731 
3732 static DisasJumpType op_rev16(DisasContext *s, DisasOps *o)
3733 {
3734     tcg_gen_bswap16_i64(o->out, o->in2, TCG_BSWAP_IZ | TCG_BSWAP_OZ);
3735     return DISAS_NEXT;
3736 }
3737 
3738 static DisasJumpType op_rev32(DisasContext *s, DisasOps *o)
3739 {
3740     tcg_gen_bswap32_i64(o->out, o->in2, TCG_BSWAP_IZ | TCG_BSWAP_OZ);
3741     return DISAS_NEXT;
3742 }
3743 
3744 static DisasJumpType op_rev64(DisasContext *s, DisasOps *o)
3745 {
3746     tcg_gen_bswap64_i64(o->out, o->in2);
3747     return DISAS_NEXT;
3748 }
3749 
3750 static DisasJumpType op_rll32(DisasContext *s, DisasOps *o)
3751 {
3752     TCGv_i32 t1 = tcg_temp_new_i32();
3753     TCGv_i32 t2 = tcg_temp_new_i32();
3754     TCGv_i32 to = tcg_temp_new_i32();
3755     tcg_gen_extrl_i64_i32(t1, o->in1);
3756     tcg_gen_extrl_i64_i32(t2, o->in2);
3757     tcg_gen_rotl_i32(to, t1, t2);
3758     tcg_gen_extu_i32_i64(o->out, to);
3759     return DISAS_NEXT;
3760 }
3761 
3762 static DisasJumpType op_rll64(DisasContext *s, DisasOps *o)
3763 {
3764     tcg_gen_rotl_i64(o->out, o->in1, o->in2);
3765     return DISAS_NEXT;
3766 }
3767 
3768 #ifndef CONFIG_USER_ONLY
3769 static DisasJumpType op_rrbe(DisasContext *s, DisasOps *o)
3770 {
3771     gen_helper_rrbe(cc_op, tcg_env, o->in2);
3772     set_cc_static(s);
3773     return DISAS_NEXT;
3774 }
3775 
3776 static DisasJumpType op_sacf(DisasContext *s, DisasOps *o)
3777 {
3778     gen_helper_sacf(tcg_env, o->in2);
3779     /* Addressing mode has changed, so end the block.  */
3780     return DISAS_TOO_MANY;
3781 }
3782 #endif
3783 
3784 static DisasJumpType op_sam(DisasContext *s, DisasOps *o)
3785 {
3786     int sam = s->insn->data;
3787     TCGv_i64 tsam;
3788     uint64_t mask;
3789 
3790     switch (sam) {
3791     case 0:
3792         mask = 0xffffff;
3793         break;
3794     case 1:
3795         mask = 0x7fffffff;
3796         break;
3797     default:
3798         mask = -1;
3799         break;
3800     }
3801 
3802     /* Bizarre but true, we check the address of the current insn for the
3803        specification exception, not the next to be executed.  Thus the PoO
3804        documents that Bad Things Happen two bytes before the end.  */
3805     if (s->base.pc_next & ~mask) {
3806         gen_program_exception(s, PGM_SPECIFICATION);
3807         return DISAS_NORETURN;
3808     }
3809     s->pc_tmp &= mask;
3810 
3811     tsam = tcg_constant_i64(sam);
3812     tcg_gen_deposit_i64(psw_mask, psw_mask, tsam, 31, 2);
3813 
3814     /* Always exit the TB, since we (may have) changed execution mode.  */
3815     return DISAS_TOO_MANY;
3816 }
3817 
3818 static DisasJumpType op_sar(DisasContext *s, DisasOps *o)
3819 {
3820     int r1 = get_field(s, r1);
3821     tcg_gen_st32_i64(o->in2, tcg_env, offsetof(CPUS390XState, aregs[r1]));
3822     return DISAS_NEXT;
3823 }
3824 
3825 static DisasJumpType op_seb(DisasContext *s, DisasOps *o)
3826 {
3827     gen_helper_seb(o->out, tcg_env, o->in1, o->in2);
3828     return DISAS_NEXT;
3829 }
3830 
3831 static DisasJumpType op_sdb(DisasContext *s, DisasOps *o)
3832 {
3833     gen_helper_sdb(o->out, tcg_env, o->in1, o->in2);
3834     return DISAS_NEXT;
3835 }
3836 
3837 static DisasJumpType op_sxb(DisasContext *s, DisasOps *o)
3838 {
3839     gen_helper_sxb(o->out_128, tcg_env, o->in1_128, o->in2_128);
3840     return DISAS_NEXT;
3841 }
3842 
3843 static DisasJumpType op_sqeb(DisasContext *s, DisasOps *o)
3844 {
3845     gen_helper_sqeb(o->out, tcg_env, o->in2);
3846     return DISAS_NEXT;
3847 }
3848 
3849 static DisasJumpType op_sqdb(DisasContext *s, DisasOps *o)
3850 {
3851     gen_helper_sqdb(o->out, tcg_env, o->in2);
3852     return DISAS_NEXT;
3853 }
3854 
3855 static DisasJumpType op_sqxb(DisasContext *s, DisasOps *o)
3856 {
3857     gen_helper_sqxb(o->out_128, tcg_env, o->in2_128);
3858     return DISAS_NEXT;
3859 }
3860 
3861 #ifndef CONFIG_USER_ONLY
3862 static DisasJumpType op_servc(DisasContext *s, DisasOps *o)
3863 {
3864     gen_helper_servc(cc_op, tcg_env, o->in2, o->in1);
3865     set_cc_static(s);
3866     return DISAS_NEXT;
3867 }
3868 
3869 static DisasJumpType op_sigp(DisasContext *s, DisasOps *o)
3870 {
3871     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
3872     TCGv_i32 r3 = tcg_constant_i32(get_field(s, r3));
3873 
3874     gen_helper_sigp(cc_op, tcg_env, o->in2, r1, r3);
3875     set_cc_static(s);
3876     return DISAS_NEXT;
3877 }
3878 #endif
3879 
3880 static DisasJumpType op_soc(DisasContext *s, DisasOps *o)
3881 {
3882     DisasCompare c;
3883     TCGv_i64 a, h;
3884     TCGLabel *lab;
3885     int r1;
3886 
3887     disas_jcc(s, &c, get_field(s, m3));
3888 
3889     /* We want to store when the condition is fulfilled, so branch
3890        out when it's not */
3891     c.cond = tcg_invert_cond(c.cond);
3892 
3893     lab = gen_new_label();
3894     if (c.is_64) {
3895         tcg_gen_brcond_i64(c.cond, c.u.s64.a, c.u.s64.b, lab);
3896     } else {
3897         tcg_gen_brcond_i32(c.cond, c.u.s32.a, c.u.s32.b, lab);
3898     }
3899 
3900     r1 = get_field(s, r1);
3901     a = get_address(s, 0, get_field(s, b2), get_field(s, d2));
3902     switch (s->insn->data) {
3903     case 1: /* STOCG */
3904         tcg_gen_qemu_st_i64(regs[r1], a, get_mem_index(s), MO_TEUQ);
3905         break;
3906     case 0: /* STOC */
3907         tcg_gen_qemu_st_i64(regs[r1], a, get_mem_index(s), MO_TEUL);
3908         break;
3909     case 2: /* STOCFH */
3910         h = tcg_temp_new_i64();
3911         tcg_gen_shri_i64(h, regs[r1], 32);
3912         tcg_gen_qemu_st_i64(h, a, get_mem_index(s), MO_TEUL);
3913         break;
3914     default:
3915         g_assert_not_reached();
3916     }
3917 
3918     gen_set_label(lab);
3919     return DISAS_NEXT;
3920 }
3921 
3922 static DisasJumpType op_sla(DisasContext *s, DisasOps *o)
3923 {
3924     TCGv_i64 t;
3925     uint64_t sign = 1ull << s->insn->data;
3926     if (s->insn->data == 31) {
3927         t = tcg_temp_new_i64();
3928         tcg_gen_shli_i64(t, o->in1, 32);
3929     } else {
3930         t = o->in1;
3931     }
3932     gen_op_update2_cc_i64(s, CC_OP_SLA, t, o->in2);
3933     tcg_gen_shl_i64(o->out, o->in1, o->in2);
3934     /* The arithmetic left shift is curious in that it does not affect
3935        the sign bit.  Copy that over from the source unchanged.  */
3936     tcg_gen_andi_i64(o->out, o->out, ~sign);
3937     tcg_gen_andi_i64(o->in1, o->in1, sign);
3938     tcg_gen_or_i64(o->out, o->out, o->in1);
3939     return DISAS_NEXT;
3940 }
3941 
3942 static DisasJumpType op_sll(DisasContext *s, DisasOps *o)
3943 {
3944     tcg_gen_shl_i64(o->out, o->in1, o->in2);
3945     return DISAS_NEXT;
3946 }
3947 
3948 static DisasJumpType op_sra(DisasContext *s, DisasOps *o)
3949 {
3950     tcg_gen_sar_i64(o->out, o->in1, o->in2);
3951     return DISAS_NEXT;
3952 }
3953 
3954 static DisasJumpType op_srl(DisasContext *s, DisasOps *o)
3955 {
3956     tcg_gen_shr_i64(o->out, o->in1, o->in2);
3957     return DISAS_NEXT;
3958 }
3959 
3960 static DisasJumpType op_sfpc(DisasContext *s, DisasOps *o)
3961 {
3962     gen_helper_sfpc(tcg_env, o->in2);
3963     return DISAS_NEXT;
3964 }
3965 
3966 static DisasJumpType op_sfas(DisasContext *s, DisasOps *o)
3967 {
3968     gen_helper_sfas(tcg_env, o->in2);
3969     return DISAS_NEXT;
3970 }
3971 
3972 static DisasJumpType op_srnm(DisasContext *s, DisasOps *o)
3973 {
3974     /* Bits other than 62 and 63 are ignored. Bit 29 is set to zero. */
3975     tcg_gen_andi_i64(o->addr1, o->addr1, 0x3ull);
3976     gen_helper_srnm(tcg_env, o->addr1);
3977     return DISAS_NEXT;
3978 }
3979 
3980 static DisasJumpType op_srnmb(DisasContext *s, DisasOps *o)
3981 {
3982     /* Bits 0-55 are are ignored. */
3983     tcg_gen_andi_i64(o->addr1, o->addr1, 0xffull);
3984     gen_helper_srnm(tcg_env, o->addr1);
3985     return DISAS_NEXT;
3986 }
3987 
3988 static DisasJumpType op_srnmt(DisasContext *s, DisasOps *o)
3989 {
3990     TCGv_i64 tmp = tcg_temp_new_i64();
3991 
3992     /* Bits other than 61-63 are ignored. */
3993     tcg_gen_andi_i64(o->addr1, o->addr1, 0x7ull);
3994 
3995     /* No need to call a helper, we don't implement dfp */
3996     tcg_gen_ld32u_i64(tmp, tcg_env, offsetof(CPUS390XState, fpc));
3997     tcg_gen_deposit_i64(tmp, tmp, o->addr1, 4, 3);
3998     tcg_gen_st32_i64(tmp, tcg_env, offsetof(CPUS390XState, fpc));
3999     return DISAS_NEXT;
4000 }
4001 
4002 static DisasJumpType op_spm(DisasContext *s, DisasOps *o)
4003 {
4004     tcg_gen_extrl_i64_i32(cc_op, o->in1);
4005     tcg_gen_extract_i32(cc_op, cc_op, 28, 2);
4006     set_cc_static(s);
4007 
4008     tcg_gen_shri_i64(o->in1, o->in1, 24);
4009     tcg_gen_deposit_i64(psw_mask, psw_mask, o->in1, PSW_SHIFT_MASK_PM, 4);
4010     return DISAS_NEXT;
4011 }
4012 
4013 static DisasJumpType op_ectg(DisasContext *s, DisasOps *o)
4014 {
4015     int b1 = get_field(s, b1);
4016     int d1 = get_field(s, d1);
4017     int b2 = get_field(s, b2);
4018     int d2 = get_field(s, d2);
4019     int r3 = get_field(s, r3);
4020     TCGv_i64 tmp = tcg_temp_new_i64();
4021 
4022     /* fetch all operands first */
4023     o->in1 = tcg_temp_new_i64();
4024     tcg_gen_addi_i64(o->in1, regs[b1], d1);
4025     o->in2 = tcg_temp_new_i64();
4026     tcg_gen_addi_i64(o->in2, regs[b2], d2);
4027     o->addr1 = tcg_temp_new_i64();
4028     gen_addi_and_wrap_i64(s, o->addr1, regs[r3], 0);
4029 
4030     /* load the third operand into r3 before modifying anything */
4031     tcg_gen_qemu_ld_i64(regs[r3], o->addr1, get_mem_index(s), MO_TEUQ);
4032 
4033     /* subtract CPU timer from first operand and store in GR0 */
4034     gen_helper_stpt(tmp, tcg_env);
4035     tcg_gen_sub_i64(regs[0], o->in1, tmp);
4036 
4037     /* store second operand in GR1 */
4038     tcg_gen_mov_i64(regs[1], o->in2);
4039     return DISAS_NEXT;
4040 }
4041 
4042 #ifndef CONFIG_USER_ONLY
4043 static DisasJumpType op_spka(DisasContext *s, DisasOps *o)
4044 {
4045     tcg_gen_shri_i64(o->in2, o->in2, 4);
4046     tcg_gen_deposit_i64(psw_mask, psw_mask, o->in2, PSW_SHIFT_KEY, 4);
4047     return DISAS_NEXT;
4048 }
4049 
4050 static DisasJumpType op_sske(DisasContext *s, DisasOps *o)
4051 {
4052     gen_helper_sske(tcg_env, o->in1, o->in2);
4053     return DISAS_NEXT;
4054 }
4055 
4056 static void gen_check_psw_mask(DisasContext *s)
4057 {
4058     TCGv_i64 reserved = tcg_temp_new_i64();
4059     TCGLabel *ok = gen_new_label();
4060 
4061     tcg_gen_andi_i64(reserved, psw_mask, PSW_MASK_RESERVED);
4062     tcg_gen_brcondi_i64(TCG_COND_EQ, reserved, 0, ok);
4063     gen_program_exception(s, PGM_SPECIFICATION);
4064     gen_set_label(ok);
4065 }
4066 
4067 static DisasJumpType op_ssm(DisasContext *s, DisasOps *o)
4068 {
4069     tcg_gen_deposit_i64(psw_mask, psw_mask, o->in2, 56, 8);
4070 
4071     gen_check_psw_mask(s);
4072 
4073     /* Exit to main loop to reevaluate s390_cpu_exec_interrupt.  */
4074     s->exit_to_mainloop = true;
4075     return DISAS_TOO_MANY;
4076 }
4077 
4078 static DisasJumpType op_stap(DisasContext *s, DisasOps *o)
4079 {
4080     tcg_gen_ld32u_i64(o->out, tcg_env, offsetof(CPUS390XState, core_id));
4081     return DISAS_NEXT;
4082 }
4083 #endif
4084 
4085 static DisasJumpType op_stck(DisasContext *s, DisasOps *o)
4086 {
4087     gen_helper_stck(o->out, tcg_env);
4088     /* ??? We don't implement clock states.  */
4089     gen_op_movi_cc(s, 0);
4090     return DISAS_NEXT;
4091 }
4092 
4093 static DisasJumpType op_stcke(DisasContext *s, DisasOps *o)
4094 {
4095     TCGv_i64 c1 = tcg_temp_new_i64();
4096     TCGv_i64 c2 = tcg_temp_new_i64();
4097     TCGv_i64 todpr = tcg_temp_new_i64();
4098     gen_helper_stck(c1, tcg_env);
4099     /* 16 bit value store in an uint32_t (only valid bits set) */
4100     tcg_gen_ld32u_i64(todpr, tcg_env, offsetof(CPUS390XState, todpr));
4101     /* Shift the 64-bit value into its place as a zero-extended
4102        104-bit value.  Note that "bit positions 64-103 are always
4103        non-zero so that they compare differently to STCK"; we set
4104        the least significant bit to 1.  */
4105     tcg_gen_shli_i64(c2, c1, 56);
4106     tcg_gen_shri_i64(c1, c1, 8);
4107     tcg_gen_ori_i64(c2, c2, 0x10000);
4108     tcg_gen_or_i64(c2, c2, todpr);
4109     tcg_gen_qemu_st_i64(c1, o->in2, get_mem_index(s), MO_TEUQ);
4110     tcg_gen_addi_i64(o->in2, o->in2, 8);
4111     tcg_gen_qemu_st_i64(c2, o->in2, get_mem_index(s), MO_TEUQ);
4112     /* ??? We don't implement clock states.  */
4113     gen_op_movi_cc(s, 0);
4114     return DISAS_NEXT;
4115 }
4116 
4117 #ifndef CONFIG_USER_ONLY
4118 static DisasJumpType op_sck(DisasContext *s, DisasOps *o)
4119 {
4120     gen_helper_sck(cc_op, tcg_env, o->in2);
4121     set_cc_static(s);
4122     return DISAS_NEXT;
4123 }
4124 
4125 static DisasJumpType op_sckc(DisasContext *s, DisasOps *o)
4126 {
4127     gen_helper_sckc(tcg_env, o->in2);
4128     return DISAS_NEXT;
4129 }
4130 
4131 static DisasJumpType op_sckpf(DisasContext *s, DisasOps *o)
4132 {
4133     gen_helper_sckpf(tcg_env, regs[0]);
4134     return DISAS_NEXT;
4135 }
4136 
4137 static DisasJumpType op_stckc(DisasContext *s, DisasOps *o)
4138 {
4139     gen_helper_stckc(o->out, tcg_env);
4140     return DISAS_NEXT;
4141 }
4142 
4143 static DisasJumpType op_stctg(DisasContext *s, DisasOps *o)
4144 {
4145     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
4146     TCGv_i32 r3 = tcg_constant_i32(get_field(s, r3));
4147 
4148     gen_helper_stctg(tcg_env, r1, o->in2, r3);
4149     return DISAS_NEXT;
4150 }
4151 
4152 static DisasJumpType op_stctl(DisasContext *s, DisasOps *o)
4153 {
4154     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
4155     TCGv_i32 r3 = tcg_constant_i32(get_field(s, r3));
4156 
4157     gen_helper_stctl(tcg_env, r1, o->in2, r3);
4158     return DISAS_NEXT;
4159 }
4160 
4161 static DisasJumpType op_stidp(DisasContext *s, DisasOps *o)
4162 {
4163     tcg_gen_ld_i64(o->out, tcg_env, offsetof(CPUS390XState, cpuid));
4164     return DISAS_NEXT;
4165 }
4166 
4167 static DisasJumpType op_spt(DisasContext *s, DisasOps *o)
4168 {
4169     gen_helper_spt(tcg_env, o->in2);
4170     return DISAS_NEXT;
4171 }
4172 
4173 static DisasJumpType op_stfl(DisasContext *s, DisasOps *o)
4174 {
4175     gen_helper_stfl(tcg_env);
4176     return DISAS_NEXT;
4177 }
4178 
4179 static DisasJumpType op_stpt(DisasContext *s, DisasOps *o)
4180 {
4181     gen_helper_stpt(o->out, tcg_env);
4182     return DISAS_NEXT;
4183 }
4184 
4185 static DisasJumpType op_stsi(DisasContext *s, DisasOps *o)
4186 {
4187     gen_helper_stsi(cc_op, tcg_env, o->in2, regs[0], regs[1]);
4188     set_cc_static(s);
4189     return DISAS_NEXT;
4190 }
4191 
4192 static DisasJumpType op_spx(DisasContext *s, DisasOps *o)
4193 {
4194     gen_helper_spx(tcg_env, o->in2);
4195     return DISAS_NEXT;
4196 }
4197 
4198 static DisasJumpType op_xsch(DisasContext *s, DisasOps *o)
4199 {
4200     gen_helper_xsch(tcg_env, regs[1]);
4201     set_cc_static(s);
4202     return DISAS_NEXT;
4203 }
4204 
4205 static DisasJumpType op_csch(DisasContext *s, DisasOps *o)
4206 {
4207     gen_helper_csch(tcg_env, regs[1]);
4208     set_cc_static(s);
4209     return DISAS_NEXT;
4210 }
4211 
4212 static DisasJumpType op_hsch(DisasContext *s, DisasOps *o)
4213 {
4214     gen_helper_hsch(tcg_env, regs[1]);
4215     set_cc_static(s);
4216     return DISAS_NEXT;
4217 }
4218 
4219 static DisasJumpType op_msch(DisasContext *s, DisasOps *o)
4220 {
4221     gen_helper_msch(tcg_env, regs[1], o->in2);
4222     set_cc_static(s);
4223     return DISAS_NEXT;
4224 }
4225 
4226 static DisasJumpType op_rchp(DisasContext *s, DisasOps *o)
4227 {
4228     gen_helper_rchp(tcg_env, regs[1]);
4229     set_cc_static(s);
4230     return DISAS_NEXT;
4231 }
4232 
4233 static DisasJumpType op_rsch(DisasContext *s, DisasOps *o)
4234 {
4235     gen_helper_rsch(tcg_env, regs[1]);
4236     set_cc_static(s);
4237     return DISAS_NEXT;
4238 }
4239 
4240 static DisasJumpType op_sal(DisasContext *s, DisasOps *o)
4241 {
4242     gen_helper_sal(tcg_env, regs[1]);
4243     return DISAS_NEXT;
4244 }
4245 
4246 static DisasJumpType op_schm(DisasContext *s, DisasOps *o)
4247 {
4248     gen_helper_schm(tcg_env, regs[1], regs[2], o->in2);
4249     return DISAS_NEXT;
4250 }
4251 
4252 static DisasJumpType op_siga(DisasContext *s, DisasOps *o)
4253 {
4254     /* From KVM code: Not provided, set CC = 3 for subchannel not operational */
4255     gen_op_movi_cc(s, 3);
4256     return DISAS_NEXT;
4257 }
4258 
4259 static DisasJumpType op_stcps(DisasContext *s, DisasOps *o)
4260 {
4261     /* The instruction is suppressed if not provided. */
4262     return DISAS_NEXT;
4263 }
4264 
4265 static DisasJumpType op_ssch(DisasContext *s, DisasOps *o)
4266 {
4267     gen_helper_ssch(tcg_env, regs[1], o->in2);
4268     set_cc_static(s);
4269     return DISAS_NEXT;
4270 }
4271 
4272 static DisasJumpType op_stsch(DisasContext *s, DisasOps *o)
4273 {
4274     gen_helper_stsch(tcg_env, regs[1], o->in2);
4275     set_cc_static(s);
4276     return DISAS_NEXT;
4277 }
4278 
4279 static DisasJumpType op_stcrw(DisasContext *s, DisasOps *o)
4280 {
4281     gen_helper_stcrw(tcg_env, o->in2);
4282     set_cc_static(s);
4283     return DISAS_NEXT;
4284 }
4285 
4286 static DisasJumpType op_tpi(DisasContext *s, DisasOps *o)
4287 {
4288     gen_helper_tpi(cc_op, tcg_env, o->addr1);
4289     set_cc_static(s);
4290     return DISAS_NEXT;
4291 }
4292 
4293 static DisasJumpType op_tsch(DisasContext *s, DisasOps *o)
4294 {
4295     gen_helper_tsch(tcg_env, regs[1], o->in2);
4296     set_cc_static(s);
4297     return DISAS_NEXT;
4298 }
4299 
4300 static DisasJumpType op_chsc(DisasContext *s, DisasOps *o)
4301 {
4302     gen_helper_chsc(tcg_env, o->in2);
4303     set_cc_static(s);
4304     return DISAS_NEXT;
4305 }
4306 
4307 static DisasJumpType op_stpx(DisasContext *s, DisasOps *o)
4308 {
4309     tcg_gen_ld_i64(o->out, tcg_env, offsetof(CPUS390XState, psa));
4310     tcg_gen_andi_i64(o->out, o->out, 0x7fffe000);
4311     return DISAS_NEXT;
4312 }
4313 
4314 static DisasJumpType op_stnosm(DisasContext *s, DisasOps *o)
4315 {
4316     uint64_t i2 = get_field(s, i2);
4317     TCGv_i64 t;
4318 
4319     /* It is important to do what the instruction name says: STORE THEN.
4320        If we let the output hook perform the store then if we fault and
4321        restart, we'll have the wrong SYSTEM MASK in place.  */
4322     t = tcg_temp_new_i64();
4323     tcg_gen_shri_i64(t, psw_mask, 56);
4324     tcg_gen_qemu_st_i64(t, o->addr1, get_mem_index(s), MO_UB);
4325 
4326     if (s->fields.op == 0xac) {
4327         tcg_gen_andi_i64(psw_mask, psw_mask,
4328                          (i2 << 56) | 0x00ffffffffffffffull);
4329     } else {
4330         tcg_gen_ori_i64(psw_mask, psw_mask, i2 << 56);
4331     }
4332 
4333     gen_check_psw_mask(s);
4334 
4335     /* Exit to main loop to reevaluate s390_cpu_exec_interrupt.  */
4336     s->exit_to_mainloop = true;
4337     return DISAS_TOO_MANY;
4338 }
4339 
4340 static DisasJumpType op_stura(DisasContext *s, DisasOps *o)
4341 {
4342     tcg_gen_qemu_st_tl(o->in1, o->in2, MMU_REAL_IDX, s->insn->data);
4343 
4344     if (s->base.tb->flags & FLAG_MASK_PER_STORE_REAL) {
4345         update_cc_op(s);
4346         update_psw_addr(s);
4347         gen_helper_per_store_real(tcg_env, tcg_constant_i32(s->ilen));
4348         return DISAS_NORETURN;
4349     }
4350     return DISAS_NEXT;
4351 }
4352 #endif
4353 
4354 static DisasJumpType op_stfle(DisasContext *s, DisasOps *o)
4355 {
4356     gen_helper_stfle(cc_op, tcg_env, o->in2);
4357     set_cc_static(s);
4358     return DISAS_NEXT;
4359 }
4360 
4361 static DisasJumpType op_st8(DisasContext *s, DisasOps *o)
4362 {
4363     tcg_gen_qemu_st_i64(o->in1, o->in2, get_mem_index(s), MO_UB);
4364     return DISAS_NEXT;
4365 }
4366 
4367 static DisasJumpType op_st16(DisasContext *s, DisasOps *o)
4368 {
4369     tcg_gen_qemu_st_i64(o->in1, o->in2, get_mem_index(s), MO_TEUW);
4370     return DISAS_NEXT;
4371 }
4372 
4373 static DisasJumpType op_st32(DisasContext *s, DisasOps *o)
4374 {
4375     tcg_gen_qemu_st_tl(o->in1, o->in2, get_mem_index(s),
4376                        MO_TEUL | s->insn->data);
4377     return DISAS_NEXT;
4378 }
4379 
4380 static DisasJumpType op_st64(DisasContext *s, DisasOps *o)
4381 {
4382     tcg_gen_qemu_st_i64(o->in1, o->in2, get_mem_index(s),
4383                         MO_TEUQ | s->insn->data);
4384     return DISAS_NEXT;
4385 }
4386 
4387 static DisasJumpType op_stam(DisasContext *s, DisasOps *o)
4388 {
4389     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
4390     TCGv_i32 r3 = tcg_constant_i32(get_field(s, r3));
4391 
4392     gen_helper_stam(tcg_env, r1, o->in2, r3);
4393     return DISAS_NEXT;
4394 }
4395 
4396 static DisasJumpType op_stcm(DisasContext *s, DisasOps *o)
4397 {
4398     int m3 = get_field(s, m3);
4399     int pos, base = s->insn->data;
4400     TCGv_i64 tmp = tcg_temp_new_i64();
4401 
4402     pos = base + ctz32(m3) * 8;
4403     switch (m3) {
4404     case 0xf:
4405         /* Effectively a 32-bit store.  */
4406         tcg_gen_shri_i64(tmp, o->in1, pos);
4407         tcg_gen_qemu_st_i64(tmp, o->in2, get_mem_index(s), MO_TEUL);
4408         break;
4409 
4410     case 0xc:
4411     case 0x6:
4412     case 0x3:
4413         /* Effectively a 16-bit store.  */
4414         tcg_gen_shri_i64(tmp, o->in1, pos);
4415         tcg_gen_qemu_st_i64(tmp, o->in2, get_mem_index(s), MO_TEUW);
4416         break;
4417 
4418     case 0x8:
4419     case 0x4:
4420     case 0x2:
4421     case 0x1:
4422         /* Effectively an 8-bit store.  */
4423         tcg_gen_shri_i64(tmp, o->in1, pos);
4424         tcg_gen_qemu_st_i64(tmp, o->in2, get_mem_index(s), MO_UB);
4425         break;
4426 
4427     default:
4428         /* This is going to be a sequence of shifts and stores.  */
4429         pos = base + 32 - 8;
4430         while (m3) {
4431             if (m3 & 0x8) {
4432                 tcg_gen_shri_i64(tmp, o->in1, pos);
4433                 tcg_gen_qemu_st_i64(tmp, o->in2, get_mem_index(s), MO_UB);
4434                 tcg_gen_addi_i64(o->in2, o->in2, 1);
4435             }
4436             m3 = (m3 << 1) & 0xf;
4437             pos -= 8;
4438         }
4439         break;
4440     }
4441     return DISAS_NEXT;
4442 }
4443 
4444 static DisasJumpType op_stm(DisasContext *s, DisasOps *o)
4445 {
4446     int r1 = get_field(s, r1);
4447     int r3 = get_field(s, r3);
4448     int size = s->insn->data;
4449     TCGv_i64 tsize = tcg_constant_i64(size);
4450 
4451     while (1) {
4452         tcg_gen_qemu_st_i64(regs[r1], o->in2, get_mem_index(s),
4453                             size == 8 ? MO_TEUQ : MO_TEUL);
4454         if (r1 == r3) {
4455             break;
4456         }
4457         tcg_gen_add_i64(o->in2, o->in2, tsize);
4458         r1 = (r1 + 1) & 15;
4459     }
4460 
4461     return DISAS_NEXT;
4462 }
4463 
4464 static DisasJumpType op_stmh(DisasContext *s, DisasOps *o)
4465 {
4466     int r1 = get_field(s, r1);
4467     int r3 = get_field(s, r3);
4468     TCGv_i64 t = tcg_temp_new_i64();
4469     TCGv_i64 t4 = tcg_constant_i64(4);
4470     TCGv_i64 t32 = tcg_constant_i64(32);
4471 
4472     while (1) {
4473         tcg_gen_shl_i64(t, regs[r1], t32);
4474         tcg_gen_qemu_st_i64(t, o->in2, get_mem_index(s), MO_TEUL);
4475         if (r1 == r3) {
4476             break;
4477         }
4478         tcg_gen_add_i64(o->in2, o->in2, t4);
4479         r1 = (r1 + 1) & 15;
4480     }
4481     return DISAS_NEXT;
4482 }
4483 
4484 static DisasJumpType op_stpq(DisasContext *s, DisasOps *o)
4485 {
4486     TCGv_i128 t16 = tcg_temp_new_i128();
4487 
4488     tcg_gen_concat_i64_i128(t16, o->out2, o->out);
4489     tcg_gen_qemu_st_i128(t16, o->in2, get_mem_index(s),
4490                          MO_TE | MO_128 | MO_ALIGN);
4491     return DISAS_NEXT;
4492 }
4493 
4494 static DisasJumpType op_srst(DisasContext *s, DisasOps *o)
4495 {
4496     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
4497     TCGv_i32 r2 = tcg_constant_i32(get_field(s, r2));
4498 
4499     gen_helper_srst(tcg_env, r1, r2);
4500     set_cc_static(s);
4501     return DISAS_NEXT;
4502 }
4503 
4504 static DisasJumpType op_srstu(DisasContext *s, DisasOps *o)
4505 {
4506     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
4507     TCGv_i32 r2 = tcg_constant_i32(get_field(s, r2));
4508 
4509     gen_helper_srstu(tcg_env, r1, r2);
4510     set_cc_static(s);
4511     return DISAS_NEXT;
4512 }
4513 
4514 static DisasJumpType op_sub(DisasContext *s, DisasOps *o)
4515 {
4516     tcg_gen_sub_i64(o->out, o->in1, o->in2);
4517     return DISAS_NEXT;
4518 }
4519 
4520 static DisasJumpType op_subu64(DisasContext *s, DisasOps *o)
4521 {
4522     tcg_gen_movi_i64(cc_src, 0);
4523     tcg_gen_sub2_i64(o->out, cc_src, o->in1, cc_src, o->in2, cc_src);
4524     return DISAS_NEXT;
4525 }
4526 
4527 /* Compute borrow (0, -1) into cc_src. */
4528 static void compute_borrow(DisasContext *s)
4529 {
4530     switch (s->cc_op) {
4531     case CC_OP_SUBU:
4532         /* The borrow value is already in cc_src (0,-1). */
4533         break;
4534     default:
4535         gen_op_calc_cc(s);
4536         /* fall through */
4537     case CC_OP_STATIC:
4538         /* The carry flag is the msb of CC; compute into cc_src. */
4539         tcg_gen_extu_i32_i64(cc_src, cc_op);
4540         tcg_gen_shri_i64(cc_src, cc_src, 1);
4541         /* fall through */
4542     case CC_OP_ADDU:
4543         /* Convert carry (1,0) to borrow (0,-1). */
4544         tcg_gen_subi_i64(cc_src, cc_src, 1);
4545         break;
4546     }
4547 }
4548 
4549 static DisasJumpType op_subb32(DisasContext *s, DisasOps *o)
4550 {
4551     compute_borrow(s);
4552 
4553     /* Borrow is {0, -1}, so add to subtract. */
4554     tcg_gen_add_i64(o->out, o->in1, cc_src);
4555     tcg_gen_sub_i64(o->out, o->out, o->in2);
4556     return DISAS_NEXT;
4557 }
4558 
4559 static DisasJumpType op_subb64(DisasContext *s, DisasOps *o)
4560 {
4561     compute_borrow(s);
4562 
4563     /*
4564      * Borrow is {0, -1}, so add to subtract; replicate the
4565      * borrow input to produce 128-bit -1 for the addition.
4566      */
4567     TCGv_i64 zero = tcg_constant_i64(0);
4568     tcg_gen_add2_i64(o->out, cc_src, o->in1, zero, cc_src, cc_src);
4569     tcg_gen_sub2_i64(o->out, cc_src, o->out, cc_src, o->in2, zero);
4570 
4571     return DISAS_NEXT;
4572 }
4573 
4574 static DisasJumpType op_svc(DisasContext *s, DisasOps *o)
4575 {
4576     TCGv_i32 t;
4577 
4578     update_psw_addr(s);
4579     update_cc_op(s);
4580 
4581     t = tcg_constant_i32(get_field(s, i1) & 0xff);
4582     tcg_gen_st_i32(t, tcg_env, offsetof(CPUS390XState, int_svc_code));
4583 
4584     t = tcg_constant_i32(s->ilen);
4585     tcg_gen_st_i32(t, tcg_env, offsetof(CPUS390XState, int_svc_ilen));
4586 
4587     gen_exception(EXCP_SVC);
4588     return DISAS_NORETURN;
4589 }
4590 
4591 static DisasJumpType op_tam(DisasContext *s, DisasOps *o)
4592 {
4593     int cc = 0;
4594 
4595     cc |= (s->base.tb->flags & FLAG_MASK_64) ? 2 : 0;
4596     cc |= (s->base.tb->flags & FLAG_MASK_32) ? 1 : 0;
4597     gen_op_movi_cc(s, cc);
4598     return DISAS_NEXT;
4599 }
4600 
4601 static DisasJumpType op_tceb(DisasContext *s, DisasOps *o)
4602 {
4603     gen_helper_tceb(cc_op, tcg_env, o->in1, o->in2);
4604     set_cc_static(s);
4605     return DISAS_NEXT;
4606 }
4607 
4608 static DisasJumpType op_tcdb(DisasContext *s, DisasOps *o)
4609 {
4610     gen_helper_tcdb(cc_op, tcg_env, o->in1, o->in2);
4611     set_cc_static(s);
4612     return DISAS_NEXT;
4613 }
4614 
4615 static DisasJumpType op_tcxb(DisasContext *s, DisasOps *o)
4616 {
4617     gen_helper_tcxb(cc_op, tcg_env, o->in1_128, o->in2);
4618     set_cc_static(s);
4619     return DISAS_NEXT;
4620 }
4621 
4622 #ifndef CONFIG_USER_ONLY
4623 
4624 static DisasJumpType op_testblock(DisasContext *s, DisasOps *o)
4625 {
4626     gen_helper_testblock(cc_op, tcg_env, o->in2);
4627     set_cc_static(s);
4628     return DISAS_NEXT;
4629 }
4630 
4631 static DisasJumpType op_tprot(DisasContext *s, DisasOps *o)
4632 {
4633     gen_helper_tprot(cc_op, tcg_env, o->addr1, o->in2);
4634     set_cc_static(s);
4635     return DISAS_NEXT;
4636 }
4637 
4638 #endif
4639 
4640 static DisasJumpType op_tp(DisasContext *s, DisasOps *o)
4641 {
4642     TCGv_i32 l1 = tcg_constant_i32(get_field(s, l1) + 1);
4643 
4644     gen_helper_tp(cc_op, tcg_env, o->addr1, l1);
4645     set_cc_static(s);
4646     return DISAS_NEXT;
4647 }
4648 
4649 static DisasJumpType op_tr(DisasContext *s, DisasOps *o)
4650 {
4651     TCGv_i32 l = tcg_constant_i32(get_field(s, l1));
4652 
4653     gen_helper_tr(tcg_env, l, o->addr1, o->in2);
4654     set_cc_static(s);
4655     return DISAS_NEXT;
4656 }
4657 
4658 static DisasJumpType op_tre(DisasContext *s, DisasOps *o)
4659 {
4660     TCGv_i128 pair = tcg_temp_new_i128();
4661 
4662     gen_helper_tre(pair, tcg_env, o->out, o->out2, o->in2);
4663     tcg_gen_extr_i128_i64(o->out2, o->out, pair);
4664     set_cc_static(s);
4665     return DISAS_NEXT;
4666 }
4667 
4668 static DisasJumpType op_trt(DisasContext *s, DisasOps *o)
4669 {
4670     TCGv_i32 l = tcg_constant_i32(get_field(s, l1));
4671 
4672     gen_helper_trt(cc_op, tcg_env, l, o->addr1, o->in2);
4673     set_cc_static(s);
4674     return DISAS_NEXT;
4675 }
4676 
4677 static DisasJumpType op_trtr(DisasContext *s, DisasOps *o)
4678 {
4679     TCGv_i32 l = tcg_constant_i32(get_field(s, l1));
4680 
4681     gen_helper_trtr(cc_op, tcg_env, l, o->addr1, o->in2);
4682     set_cc_static(s);
4683     return DISAS_NEXT;
4684 }
4685 
4686 static DisasJumpType op_trXX(DisasContext *s, DisasOps *o)
4687 {
4688     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
4689     TCGv_i32 r2 = tcg_constant_i32(get_field(s, r2));
4690     TCGv_i32 sizes = tcg_constant_i32(s->insn->opc & 3);
4691     TCGv_i32 tst = tcg_temp_new_i32();
4692     int m3 = get_field(s, m3);
4693 
4694     if (!s390_has_feat(S390_FEAT_ETF2_ENH)) {
4695         m3 = 0;
4696     }
4697     if (m3 & 1) {
4698         tcg_gen_movi_i32(tst, -1);
4699     } else {
4700         tcg_gen_extrl_i64_i32(tst, regs[0]);
4701         if (s->insn->opc & 3) {
4702             tcg_gen_ext8u_i32(tst, tst);
4703         } else {
4704             tcg_gen_ext16u_i32(tst, tst);
4705         }
4706     }
4707     gen_helper_trXX(cc_op, tcg_env, r1, r2, tst, sizes);
4708 
4709     set_cc_static(s);
4710     return DISAS_NEXT;
4711 }
4712 
4713 static DisasJumpType op_ts(DisasContext *s, DisasOps *o)
4714 {
4715     TCGv_i32 ff = tcg_constant_i32(0xff);
4716     TCGv_i32 t1 = tcg_temp_new_i32();
4717 
4718     tcg_gen_atomic_xchg_i32(t1, o->in2, ff, get_mem_index(s), MO_UB);
4719     tcg_gen_extract_i32(cc_op, t1, 7, 1);
4720     set_cc_static(s);
4721     return DISAS_NEXT;
4722 }
4723 
4724 static DisasJumpType op_unpk(DisasContext *s, DisasOps *o)
4725 {
4726     TCGv_i32 l = tcg_constant_i32(get_field(s, l1));
4727 
4728     gen_helper_unpk(tcg_env, l, o->addr1, o->in2);
4729     return DISAS_NEXT;
4730 }
4731 
4732 static DisasJumpType op_unpka(DisasContext *s, DisasOps *o)
4733 {
4734     int l1 = get_field(s, l1) + 1;
4735     TCGv_i32 l;
4736 
4737     /* The length must not exceed 32 bytes.  */
4738     if (l1 > 32) {
4739         gen_program_exception(s, PGM_SPECIFICATION);
4740         return DISAS_NORETURN;
4741     }
4742     l = tcg_constant_i32(l1);
4743     gen_helper_unpka(cc_op, tcg_env, o->addr1, l, o->in2);
4744     set_cc_static(s);
4745     return DISAS_NEXT;
4746 }
4747 
4748 static DisasJumpType op_unpku(DisasContext *s, DisasOps *o)
4749 {
4750     int l1 = get_field(s, l1) + 1;
4751     TCGv_i32 l;
4752 
4753     /* The length must be even and should not exceed 64 bytes.  */
4754     if ((l1 & 1) || (l1 > 64)) {
4755         gen_program_exception(s, PGM_SPECIFICATION);
4756         return DISAS_NORETURN;
4757     }
4758     l = tcg_constant_i32(l1);
4759     gen_helper_unpku(cc_op, tcg_env, o->addr1, l, o->in2);
4760     set_cc_static(s);
4761     return DISAS_NEXT;
4762 }
4763 
4764 
4765 static DisasJumpType op_xc(DisasContext *s, DisasOps *o)
4766 {
4767     int d1 = get_field(s, d1);
4768     int d2 = get_field(s, d2);
4769     int b1 = get_field(s, b1);
4770     int b2 = get_field(s, b2);
4771     int l = get_field(s, l1);
4772     TCGv_i32 t32;
4773 
4774     o->addr1 = get_address(s, 0, b1, d1);
4775 
4776     /* If the addresses are identical, this is a store/memset of zero.  */
4777     if (b1 == b2 && d1 == d2 && (l + 1) <= 32) {
4778         o->in2 = tcg_constant_i64(0);
4779 
4780         l++;
4781         while (l >= 8) {
4782             tcg_gen_qemu_st_i64(o->in2, o->addr1, get_mem_index(s), MO_UQ);
4783             l -= 8;
4784             if (l > 0) {
4785                 tcg_gen_addi_i64(o->addr1, o->addr1, 8);
4786             }
4787         }
4788         if (l >= 4) {
4789             tcg_gen_qemu_st_i64(o->in2, o->addr1, get_mem_index(s), MO_UL);
4790             l -= 4;
4791             if (l > 0) {
4792                 tcg_gen_addi_i64(o->addr1, o->addr1, 4);
4793             }
4794         }
4795         if (l >= 2) {
4796             tcg_gen_qemu_st_i64(o->in2, o->addr1, get_mem_index(s), MO_UW);
4797             l -= 2;
4798             if (l > 0) {
4799                 tcg_gen_addi_i64(o->addr1, o->addr1, 2);
4800             }
4801         }
4802         if (l) {
4803             tcg_gen_qemu_st_i64(o->in2, o->addr1, get_mem_index(s), MO_UB);
4804         }
4805         gen_op_movi_cc(s, 0);
4806         return DISAS_NEXT;
4807     }
4808 
4809     /* But in general we'll defer to a helper.  */
4810     o->in2 = get_address(s, 0, b2, d2);
4811     t32 = tcg_constant_i32(l);
4812     gen_helper_xc(cc_op, tcg_env, t32, o->addr1, o->in2);
4813     set_cc_static(s);
4814     return DISAS_NEXT;
4815 }
4816 
4817 static DisasJumpType op_xor(DisasContext *s, DisasOps *o)
4818 {
4819     tcg_gen_xor_i64(o->out, o->in1, o->in2);
4820     return DISAS_NEXT;
4821 }
4822 
4823 static DisasJumpType op_xori(DisasContext *s, DisasOps *o)
4824 {
4825     int shift = s->insn->data & 0xff;
4826     int size = s->insn->data >> 8;
4827     uint64_t mask = ((1ull << size) - 1) << shift;
4828     TCGv_i64 t = tcg_temp_new_i64();
4829 
4830     tcg_gen_shli_i64(t, o->in2, shift);
4831     tcg_gen_xor_i64(o->out, o->in1, t);
4832 
4833     /* Produce the CC from only the bits manipulated.  */
4834     tcg_gen_andi_i64(cc_dst, o->out, mask);
4835     set_cc_nz_u64(s, cc_dst);
4836     return DISAS_NEXT;
4837 }
4838 
4839 static DisasJumpType op_xi(DisasContext *s, DisasOps *o)
4840 {
4841     o->in1 = tcg_temp_new_i64();
4842 
4843     if (!s390_has_feat(S390_FEAT_INTERLOCKED_ACCESS_2)) {
4844         tcg_gen_qemu_ld_tl(o->in1, o->addr1, get_mem_index(s), s->insn->data);
4845     } else {
4846         /* Perform the atomic operation in memory. */
4847         tcg_gen_atomic_fetch_xor_i64(o->in1, o->addr1, o->in2, get_mem_index(s),
4848                                      s->insn->data);
4849     }
4850 
4851     /* Recompute also for atomic case: needed for setting CC. */
4852     tcg_gen_xor_i64(o->out, o->in1, o->in2);
4853 
4854     if (!s390_has_feat(S390_FEAT_INTERLOCKED_ACCESS_2)) {
4855         tcg_gen_qemu_st_tl(o->out, o->addr1, get_mem_index(s), s->insn->data);
4856     }
4857     return DISAS_NEXT;
4858 }
4859 
4860 static DisasJumpType op_zero(DisasContext *s, DisasOps *o)
4861 {
4862     o->out = tcg_constant_i64(0);
4863     return DISAS_NEXT;
4864 }
4865 
4866 static DisasJumpType op_zero2(DisasContext *s, DisasOps *o)
4867 {
4868     o->out = tcg_constant_i64(0);
4869     o->out2 = o->out;
4870     return DISAS_NEXT;
4871 }
4872 
4873 #ifndef CONFIG_USER_ONLY
4874 static DisasJumpType op_clp(DisasContext *s, DisasOps *o)
4875 {
4876     TCGv_i32 r2 = tcg_constant_i32(get_field(s, r2));
4877 
4878     gen_helper_clp(tcg_env, r2);
4879     set_cc_static(s);
4880     return DISAS_NEXT;
4881 }
4882 
4883 static DisasJumpType op_pcilg(DisasContext *s, DisasOps *o)
4884 {
4885     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
4886     TCGv_i32 r2 = tcg_constant_i32(get_field(s, r2));
4887 
4888     gen_helper_pcilg(tcg_env, r1, r2);
4889     set_cc_static(s);
4890     return DISAS_NEXT;
4891 }
4892 
4893 static DisasJumpType op_pcistg(DisasContext *s, DisasOps *o)
4894 {
4895     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
4896     TCGv_i32 r2 = tcg_constant_i32(get_field(s, r2));
4897 
4898     gen_helper_pcistg(tcg_env, r1, r2);
4899     set_cc_static(s);
4900     return DISAS_NEXT;
4901 }
4902 
4903 static DisasJumpType op_stpcifc(DisasContext *s, DisasOps *o)
4904 {
4905     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
4906     TCGv_i32 ar = tcg_constant_i32(get_field(s, b2));
4907 
4908     gen_helper_stpcifc(tcg_env, r1, o->addr1, ar);
4909     set_cc_static(s);
4910     return DISAS_NEXT;
4911 }
4912 
4913 static DisasJumpType op_sic(DisasContext *s, DisasOps *o)
4914 {
4915     gen_helper_sic(tcg_env, o->in1, o->in2);
4916     return DISAS_NEXT;
4917 }
4918 
4919 static DisasJumpType op_rpcit(DisasContext *s, DisasOps *o)
4920 {
4921     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
4922     TCGv_i32 r2 = tcg_constant_i32(get_field(s, r2));
4923 
4924     gen_helper_rpcit(tcg_env, r1, r2);
4925     set_cc_static(s);
4926     return DISAS_NEXT;
4927 }
4928 
4929 static DisasJumpType op_pcistb(DisasContext *s, DisasOps *o)
4930 {
4931     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
4932     TCGv_i32 r3 = tcg_constant_i32(get_field(s, r3));
4933     TCGv_i32 ar = tcg_constant_i32(get_field(s, b2));
4934 
4935     gen_helper_pcistb(tcg_env, r1, r3, o->addr1, ar);
4936     set_cc_static(s);
4937     return DISAS_NEXT;
4938 }
4939 
4940 static DisasJumpType op_mpcifc(DisasContext *s, DisasOps *o)
4941 {
4942     TCGv_i32 r1 = tcg_constant_i32(get_field(s, r1));
4943     TCGv_i32 ar = tcg_constant_i32(get_field(s, b2));
4944 
4945     gen_helper_mpcifc(tcg_env, r1, o->addr1, ar);
4946     set_cc_static(s);
4947     return DISAS_NEXT;
4948 }
4949 #endif
4950 
4951 #include "translate_vx.c.inc"
4952 
4953 /* ====================================================================== */
4954 /* The "Cc OUTput" generators.  Given the generated output (and in some cases
4955    the original inputs), update the various cc data structures in order to
4956    be able to compute the new condition code.  */
4957 
4958 static void cout_abs32(DisasContext *s, DisasOps *o)
4959 {
4960     gen_op_update1_cc_i64(s, CC_OP_ABS_32, o->out);
4961 }
4962 
4963 static void cout_abs64(DisasContext *s, DisasOps *o)
4964 {
4965     gen_op_update1_cc_i64(s, CC_OP_ABS_64, o->out);
4966 }
4967 
4968 static void cout_adds32(DisasContext *s, DisasOps *o)
4969 {
4970     gen_op_update3_cc_i64(s, CC_OP_ADD_32, o->in1, o->in2, o->out);
4971 }
4972 
4973 static void cout_adds64(DisasContext *s, DisasOps *o)
4974 {
4975     gen_op_update3_cc_i64(s, CC_OP_ADD_64, o->in1, o->in2, o->out);
4976 }
4977 
4978 static void cout_addu32(DisasContext *s, DisasOps *o)
4979 {
4980     tcg_gen_shri_i64(cc_src, o->out, 32);
4981     tcg_gen_ext32u_i64(cc_dst, o->out);
4982     gen_op_update2_cc_i64(s, CC_OP_ADDU, cc_src, cc_dst);
4983 }
4984 
4985 static void cout_addu64(DisasContext *s, DisasOps *o)
4986 {
4987     gen_op_update2_cc_i64(s, CC_OP_ADDU, cc_src, o->out);
4988 }
4989 
4990 static void cout_cmps32(DisasContext *s, DisasOps *o)
4991 {
4992     gen_op_update2_cc_i64(s, CC_OP_LTGT_32, o->in1, o->in2);
4993 }
4994 
4995 static void cout_cmps64(DisasContext *s, DisasOps *o)
4996 {
4997     gen_op_update2_cc_i64(s, CC_OP_LTGT_64, o->in1, o->in2);
4998 }
4999 
5000 static void cout_cmpu32(DisasContext *s, DisasOps *o)
5001 {
5002     gen_op_update2_cc_i64(s, CC_OP_LTUGTU_32, o->in1, o->in2);
5003 }
5004 
5005 static void cout_cmpu64(DisasContext *s, DisasOps *o)
5006 {
5007     gen_op_update2_cc_i64(s, CC_OP_LTUGTU_64, o->in1, o->in2);
5008 }
5009 
5010 static void cout_f32(DisasContext *s, DisasOps *o)
5011 {
5012     gen_op_update1_cc_i64(s, CC_OP_NZ_F32, o->out);
5013 }
5014 
5015 static void cout_f64(DisasContext *s, DisasOps *o)
5016 {
5017     gen_op_update1_cc_i64(s, CC_OP_NZ_F64, o->out);
5018 }
5019 
5020 static void cout_f128(DisasContext *s, DisasOps *o)
5021 {
5022     gen_op_update2_cc_i64(s, CC_OP_NZ_F128, o->out, o->out2);
5023 }
5024 
5025 static void cout_nabs32(DisasContext *s, DisasOps *o)
5026 {
5027     gen_op_update1_cc_i64(s, CC_OP_NABS_32, o->out);
5028 }
5029 
5030 static void cout_nabs64(DisasContext *s, DisasOps *o)
5031 {
5032     gen_op_update1_cc_i64(s, CC_OP_NABS_64, o->out);
5033 }
5034 
5035 static void cout_neg32(DisasContext *s, DisasOps *o)
5036 {
5037     gen_op_update1_cc_i64(s, CC_OP_COMP_32, o->out);
5038 }
5039 
5040 static void cout_neg64(DisasContext *s, DisasOps *o)
5041 {
5042     gen_op_update1_cc_i64(s, CC_OP_COMP_64, o->out);
5043 }
5044 
5045 static void cout_nz32(DisasContext *s, DisasOps *o)
5046 {
5047     tcg_gen_ext32u_i64(cc_dst, o->out);
5048     gen_op_update1_cc_i64(s, CC_OP_NZ, cc_dst);
5049 }
5050 
5051 static void cout_nz64(DisasContext *s, DisasOps *o)
5052 {
5053     gen_op_update1_cc_i64(s, CC_OP_NZ, o->out);
5054 }
5055 
5056 static void cout_s32(DisasContext *s, DisasOps *o)
5057 {
5058     gen_op_update1_cc_i64(s, CC_OP_LTGT0_32, o->out);
5059 }
5060 
5061 static void cout_s64(DisasContext *s, DisasOps *o)
5062 {
5063     gen_op_update1_cc_i64(s, CC_OP_LTGT0_64, o->out);
5064 }
5065 
5066 static void cout_subs32(DisasContext *s, DisasOps *o)
5067 {
5068     gen_op_update3_cc_i64(s, CC_OP_SUB_32, o->in1, o->in2, o->out);
5069 }
5070 
5071 static void cout_subs64(DisasContext *s, DisasOps *o)
5072 {
5073     gen_op_update3_cc_i64(s, CC_OP_SUB_64, o->in1, o->in2, o->out);
5074 }
5075 
5076 static void cout_subu32(DisasContext *s, DisasOps *o)
5077 {
5078     tcg_gen_sari_i64(cc_src, o->out, 32);
5079     tcg_gen_ext32u_i64(cc_dst, o->out);
5080     gen_op_update2_cc_i64(s, CC_OP_SUBU, cc_src, cc_dst);
5081 }
5082 
5083 static void cout_subu64(DisasContext *s, DisasOps *o)
5084 {
5085     gen_op_update2_cc_i64(s, CC_OP_SUBU, cc_src, o->out);
5086 }
5087 
5088 static void cout_tm32(DisasContext *s, DisasOps *o)
5089 {
5090     gen_op_update2_cc_i64(s, CC_OP_TM_32, o->in1, o->in2);
5091 }
5092 
5093 static void cout_tm64(DisasContext *s, DisasOps *o)
5094 {
5095     gen_op_update2_cc_i64(s, CC_OP_TM_64, o->in1, o->in2);
5096 }
5097 
5098 static void cout_muls32(DisasContext *s, DisasOps *o)
5099 {
5100     gen_op_update1_cc_i64(s, CC_OP_MULS_32, o->out);
5101 }
5102 
5103 static void cout_muls64(DisasContext *s, DisasOps *o)
5104 {
5105     /* out contains "high" part, out2 contains "low" part of 128 bit result */
5106     gen_op_update2_cc_i64(s, CC_OP_MULS_64, o->out, o->out2);
5107 }
5108 
5109 /* ====================================================================== */
5110 /* The "PREParation" generators.  These initialize the DisasOps.OUT fields
5111    with the TCG register to which we will write.  Used in combination with
5112    the "wout" generators, in some cases we need a new temporary, and in
5113    some cases we can write to a TCG global.  */
5114 
5115 static void prep_new(DisasContext *s, DisasOps *o)
5116 {
5117     o->out = tcg_temp_new_i64();
5118 }
5119 #define SPEC_prep_new 0
5120 
5121 static void prep_new_P(DisasContext *s, DisasOps *o)
5122 {
5123     o->out = tcg_temp_new_i64();
5124     o->out2 = tcg_temp_new_i64();
5125 }
5126 #define SPEC_prep_new_P 0
5127 
5128 static void prep_new_x(DisasContext *s, DisasOps *o)
5129 {
5130     o->out_128 = tcg_temp_new_i128();
5131 }
5132 #define SPEC_prep_new_x 0
5133 
5134 static void prep_r1(DisasContext *s, DisasOps *o)
5135 {
5136     o->out = regs[get_field(s, r1)];
5137 }
5138 #define SPEC_prep_r1 0
5139 
5140 static void prep_r1_P(DisasContext *s, DisasOps *o)
5141 {
5142     int r1 = get_field(s, r1);
5143     o->out = regs[r1];
5144     o->out2 = regs[r1 + 1];
5145 }
5146 #define SPEC_prep_r1_P SPEC_r1_even
5147 
5148 /* ====================================================================== */
5149 /* The "Write OUTput" generators.  These generally perform some non-trivial
5150    copy of data to TCG globals, or to main memory.  The trivial cases are
5151    generally handled by having a "prep" generator install the TCG global
5152    as the destination of the operation.  */
5153 
5154 static void wout_r1(DisasContext *s, DisasOps *o)
5155 {
5156     store_reg(get_field(s, r1), o->out);
5157 }
5158 #define SPEC_wout_r1 0
5159 
5160 static void wout_out2_r1(DisasContext *s, DisasOps *o)
5161 {
5162     store_reg(get_field(s, r1), o->out2);
5163 }
5164 #define SPEC_wout_out2_r1 0
5165 
5166 static void wout_r1_8(DisasContext *s, DisasOps *o)
5167 {
5168     int r1 = get_field(s, r1);
5169     tcg_gen_deposit_i64(regs[r1], regs[r1], o->out, 0, 8);
5170 }
5171 #define SPEC_wout_r1_8 0
5172 
5173 static void wout_r1_16(DisasContext *s, DisasOps *o)
5174 {
5175     int r1 = get_field(s, r1);
5176     tcg_gen_deposit_i64(regs[r1], regs[r1], o->out, 0, 16);
5177 }
5178 #define SPEC_wout_r1_16 0
5179 
5180 static void wout_r1_32(DisasContext *s, DisasOps *o)
5181 {
5182     store_reg32_i64(get_field(s, r1), o->out);
5183 }
5184 #define SPEC_wout_r1_32 0
5185 
5186 static void wout_r1_32h(DisasContext *s, DisasOps *o)
5187 {
5188     store_reg32h_i64(get_field(s, r1), o->out);
5189 }
5190 #define SPEC_wout_r1_32h 0
5191 
5192 static void wout_r1_P32(DisasContext *s, DisasOps *o)
5193 {
5194     int r1 = get_field(s, r1);
5195     store_reg32_i64(r1, o->out);
5196     store_reg32_i64(r1 + 1, o->out2);
5197 }
5198 #define SPEC_wout_r1_P32 SPEC_r1_even
5199 
5200 static void wout_r1_D32(DisasContext *s, DisasOps *o)
5201 {
5202     int r1 = get_field(s, r1);
5203     TCGv_i64 t = tcg_temp_new_i64();
5204     store_reg32_i64(r1 + 1, o->out);
5205     tcg_gen_shri_i64(t, o->out, 32);
5206     store_reg32_i64(r1, t);
5207 }
5208 #define SPEC_wout_r1_D32 SPEC_r1_even
5209 
5210 static void wout_r1_D64(DisasContext *s, DisasOps *o)
5211 {
5212     int r1 = get_field(s, r1);
5213     tcg_gen_extr_i128_i64(regs[r1 + 1], regs[r1], o->out_128);
5214 }
5215 #define SPEC_wout_r1_D64 SPEC_r1_even
5216 
5217 static void wout_r3_P32(DisasContext *s, DisasOps *o)
5218 {
5219     int r3 = get_field(s, r3);
5220     store_reg32_i64(r3, o->out);
5221     store_reg32_i64(r3 + 1, o->out2);
5222 }
5223 #define SPEC_wout_r3_P32 SPEC_r3_even
5224 
5225 static void wout_r3_P64(DisasContext *s, DisasOps *o)
5226 {
5227     int r3 = get_field(s, r3);
5228     store_reg(r3, o->out);
5229     store_reg(r3 + 1, o->out2);
5230 }
5231 #define SPEC_wout_r3_P64 SPEC_r3_even
5232 
5233 static void wout_e1(DisasContext *s, DisasOps *o)
5234 {
5235     store_freg32_i64(get_field(s, r1), o->out);
5236 }
5237 #define SPEC_wout_e1 0
5238 
5239 static void wout_f1(DisasContext *s, DisasOps *o)
5240 {
5241     store_freg(get_field(s, r1), o->out);
5242 }
5243 #define SPEC_wout_f1 0
5244 
5245 static void wout_x1(DisasContext *s, DisasOps *o)
5246 {
5247     int f1 = get_field(s, r1);
5248 
5249     /* Split out_128 into out+out2 for cout_f128. */
5250     tcg_debug_assert(o->out == NULL);
5251     o->out = tcg_temp_new_i64();
5252     o->out2 = tcg_temp_new_i64();
5253 
5254     tcg_gen_extr_i128_i64(o->out2, o->out, o->out_128);
5255     store_freg(f1, o->out);
5256     store_freg(f1 + 2, o->out2);
5257 }
5258 #define SPEC_wout_x1 SPEC_r1_f128
5259 
5260 static void wout_x1_P(DisasContext *s, DisasOps *o)
5261 {
5262     int f1 = get_field(s, r1);
5263     store_freg(f1, o->out);
5264     store_freg(f1 + 2, o->out2);
5265 }
5266 #define SPEC_wout_x1_P SPEC_r1_f128
5267 
5268 static void wout_cond_r1r2_32(DisasContext *s, DisasOps *o)
5269 {
5270     if (get_field(s, r1) != get_field(s, r2)) {
5271         store_reg32_i64(get_field(s, r1), o->out);
5272     }
5273 }
5274 #define SPEC_wout_cond_r1r2_32 0
5275 
5276 static void wout_cond_e1e2(DisasContext *s, DisasOps *o)
5277 {
5278     if (get_field(s, r1) != get_field(s, r2)) {
5279         store_freg32_i64(get_field(s, r1), o->out);
5280     }
5281 }
5282 #define SPEC_wout_cond_e1e2 0
5283 
5284 static void wout_m1_8(DisasContext *s, DisasOps *o)
5285 {
5286     tcg_gen_qemu_st_i64(o->out, o->addr1, get_mem_index(s), MO_UB);
5287 }
5288 #define SPEC_wout_m1_8 0
5289 
5290 static void wout_m1_16(DisasContext *s, DisasOps *o)
5291 {
5292     tcg_gen_qemu_st_i64(o->out, o->addr1, get_mem_index(s), MO_TEUW);
5293 }
5294 #define SPEC_wout_m1_16 0
5295 
5296 #ifndef CONFIG_USER_ONLY
5297 static void wout_m1_16a(DisasContext *s, DisasOps *o)
5298 {
5299     tcg_gen_qemu_st_tl(o->out, o->addr1, get_mem_index(s), MO_TEUW | MO_ALIGN);
5300 }
5301 #define SPEC_wout_m1_16a 0
5302 #endif
5303 
5304 static void wout_m1_32(DisasContext *s, DisasOps *o)
5305 {
5306     tcg_gen_qemu_st_i64(o->out, o->addr1, get_mem_index(s), MO_TEUL);
5307 }
5308 #define SPEC_wout_m1_32 0
5309 
5310 #ifndef CONFIG_USER_ONLY
5311 static void wout_m1_32a(DisasContext *s, DisasOps *o)
5312 {
5313     tcg_gen_qemu_st_tl(o->out, o->addr1, get_mem_index(s), MO_TEUL | MO_ALIGN);
5314 }
5315 #define SPEC_wout_m1_32a 0
5316 #endif
5317 
5318 static void wout_m1_64(DisasContext *s, DisasOps *o)
5319 {
5320     tcg_gen_qemu_st_i64(o->out, o->addr1, get_mem_index(s), MO_TEUQ);
5321 }
5322 #define SPEC_wout_m1_64 0
5323 
5324 #ifndef CONFIG_USER_ONLY
5325 static void wout_m1_64a(DisasContext *s, DisasOps *o)
5326 {
5327     tcg_gen_qemu_st_i64(o->out, o->addr1, get_mem_index(s), MO_TEUQ | MO_ALIGN);
5328 }
5329 #define SPEC_wout_m1_64a 0
5330 #endif
5331 
5332 static void wout_m2_32(DisasContext *s, DisasOps *o)
5333 {
5334     tcg_gen_qemu_st_i64(o->out, o->in2, get_mem_index(s), MO_TEUL);
5335 }
5336 #define SPEC_wout_m2_32 0
5337 
5338 static void wout_in2_r1(DisasContext *s, DisasOps *o)
5339 {
5340     store_reg(get_field(s, r1), o->in2);
5341 }
5342 #define SPEC_wout_in2_r1 0
5343 
5344 static void wout_in2_r1_32(DisasContext *s, DisasOps *o)
5345 {
5346     store_reg32_i64(get_field(s, r1), o->in2);
5347 }
5348 #define SPEC_wout_in2_r1_32 0
5349 
5350 /* ====================================================================== */
5351 /* The "INput 1" generators.  These load the first operand to an insn.  */
5352 
5353 static void in1_r1(DisasContext *s, DisasOps *o)
5354 {
5355     o->in1 = load_reg(get_field(s, r1));
5356 }
5357 #define SPEC_in1_r1 0
5358 
5359 static void in1_r1_o(DisasContext *s, DisasOps *o)
5360 {
5361     o->in1 = regs[get_field(s, r1)];
5362 }
5363 #define SPEC_in1_r1_o 0
5364 
5365 static void in1_r1_32s(DisasContext *s, DisasOps *o)
5366 {
5367     o->in1 = tcg_temp_new_i64();
5368     tcg_gen_ext32s_i64(o->in1, regs[get_field(s, r1)]);
5369 }
5370 #define SPEC_in1_r1_32s 0
5371 
5372 static void in1_r1_32u(DisasContext *s, DisasOps *o)
5373 {
5374     o->in1 = tcg_temp_new_i64();
5375     tcg_gen_ext32u_i64(o->in1, regs[get_field(s, r1)]);
5376 }
5377 #define SPEC_in1_r1_32u 0
5378 
5379 static void in1_r1_sr32(DisasContext *s, DisasOps *o)
5380 {
5381     o->in1 = tcg_temp_new_i64();
5382     tcg_gen_shri_i64(o->in1, regs[get_field(s, r1)], 32);
5383 }
5384 #define SPEC_in1_r1_sr32 0
5385 
5386 static void in1_r1p1(DisasContext *s, DisasOps *o)
5387 {
5388     o->in1 = load_reg(get_field(s, r1) + 1);
5389 }
5390 #define SPEC_in1_r1p1 SPEC_r1_even
5391 
5392 static void in1_r1p1_o(DisasContext *s, DisasOps *o)
5393 {
5394     o->in1 = regs[get_field(s, r1) + 1];
5395 }
5396 #define SPEC_in1_r1p1_o SPEC_r1_even
5397 
5398 static void in1_r1p1_32s(DisasContext *s, DisasOps *o)
5399 {
5400     o->in1 = tcg_temp_new_i64();
5401     tcg_gen_ext32s_i64(o->in1, regs[get_field(s, r1) + 1]);
5402 }
5403 #define SPEC_in1_r1p1_32s SPEC_r1_even
5404 
5405 static void in1_r1p1_32u(DisasContext *s, DisasOps *o)
5406 {
5407     o->in1 = tcg_temp_new_i64();
5408     tcg_gen_ext32u_i64(o->in1, regs[get_field(s, r1) + 1]);
5409 }
5410 #define SPEC_in1_r1p1_32u SPEC_r1_even
5411 
5412 static void in1_r1_D32(DisasContext *s, DisasOps *o)
5413 {
5414     int r1 = get_field(s, r1);
5415     o->in1 = tcg_temp_new_i64();
5416     tcg_gen_concat32_i64(o->in1, regs[r1 + 1], regs[r1]);
5417 }
5418 #define SPEC_in1_r1_D32 SPEC_r1_even
5419 
5420 static void in1_r2(DisasContext *s, DisasOps *o)
5421 {
5422     o->in1 = load_reg(get_field(s, r2));
5423 }
5424 #define SPEC_in1_r2 0
5425 
5426 static void in1_r2_sr32(DisasContext *s, DisasOps *o)
5427 {
5428     o->in1 = tcg_temp_new_i64();
5429     tcg_gen_shri_i64(o->in1, regs[get_field(s, r2)], 32);
5430 }
5431 #define SPEC_in1_r2_sr32 0
5432 
5433 static void in1_r2_32u(DisasContext *s, DisasOps *o)
5434 {
5435     o->in1 = tcg_temp_new_i64();
5436     tcg_gen_ext32u_i64(o->in1, regs[get_field(s, r2)]);
5437 }
5438 #define SPEC_in1_r2_32u 0
5439 
5440 static void in1_r3(DisasContext *s, DisasOps *o)
5441 {
5442     o->in1 = load_reg(get_field(s, r3));
5443 }
5444 #define SPEC_in1_r3 0
5445 
5446 static void in1_r3_o(DisasContext *s, DisasOps *o)
5447 {
5448     o->in1 = regs[get_field(s, r3)];
5449 }
5450 #define SPEC_in1_r3_o 0
5451 
5452 static void in1_r3_32s(DisasContext *s, DisasOps *o)
5453 {
5454     o->in1 = tcg_temp_new_i64();
5455     tcg_gen_ext32s_i64(o->in1, regs[get_field(s, r3)]);
5456 }
5457 #define SPEC_in1_r3_32s 0
5458 
5459 static void in1_r3_32u(DisasContext *s, DisasOps *o)
5460 {
5461     o->in1 = tcg_temp_new_i64();
5462     tcg_gen_ext32u_i64(o->in1, regs[get_field(s, r3)]);
5463 }
5464 #define SPEC_in1_r3_32u 0
5465 
5466 static void in1_r3_D32(DisasContext *s, DisasOps *o)
5467 {
5468     int r3 = get_field(s, r3);
5469     o->in1 = tcg_temp_new_i64();
5470     tcg_gen_concat32_i64(o->in1, regs[r3 + 1], regs[r3]);
5471 }
5472 #define SPEC_in1_r3_D32 SPEC_r3_even
5473 
5474 static void in1_r3_sr32(DisasContext *s, DisasOps *o)
5475 {
5476     o->in1 = tcg_temp_new_i64();
5477     tcg_gen_shri_i64(o->in1, regs[get_field(s, r3)], 32);
5478 }
5479 #define SPEC_in1_r3_sr32 0
5480 
5481 static void in1_e1(DisasContext *s, DisasOps *o)
5482 {
5483     o->in1 = load_freg32_i64(get_field(s, r1));
5484 }
5485 #define SPEC_in1_e1 0
5486 
5487 static void in1_f1(DisasContext *s, DisasOps *o)
5488 {
5489     o->in1 = load_freg(get_field(s, r1));
5490 }
5491 #define SPEC_in1_f1 0
5492 
5493 static void in1_x1(DisasContext *s, DisasOps *o)
5494 {
5495     o->in1_128 = load_freg_128(get_field(s, r1));
5496 }
5497 #define SPEC_in1_x1 SPEC_r1_f128
5498 
5499 /* Load the high double word of an extended (128-bit) format FP number */
5500 static void in1_x2h(DisasContext *s, DisasOps *o)
5501 {
5502     o->in1 = load_freg(get_field(s, r2));
5503 }
5504 #define SPEC_in1_x2h SPEC_r2_f128
5505 
5506 static void in1_f3(DisasContext *s, DisasOps *o)
5507 {
5508     o->in1 = load_freg(get_field(s, r3));
5509 }
5510 #define SPEC_in1_f3 0
5511 
5512 static void in1_la1(DisasContext *s, DisasOps *o)
5513 {
5514     o->addr1 = get_address(s, 0, get_field(s, b1), get_field(s, d1));
5515 }
5516 #define SPEC_in1_la1 0
5517 
5518 static void in1_la2(DisasContext *s, DisasOps *o)
5519 {
5520     int x2 = have_field(s, x2) ? get_field(s, x2) : 0;
5521     o->addr1 = get_address(s, x2, get_field(s, b2), get_field(s, d2));
5522 }
5523 #define SPEC_in1_la2 0
5524 
5525 static void in1_m1_8u(DisasContext *s, DisasOps *o)
5526 {
5527     in1_la1(s, o);
5528     o->in1 = tcg_temp_new_i64();
5529     tcg_gen_qemu_ld_i64(o->in1, o->addr1, get_mem_index(s), MO_UB);
5530 }
5531 #define SPEC_in1_m1_8u 0
5532 
5533 static void in1_m1_16s(DisasContext *s, DisasOps *o)
5534 {
5535     in1_la1(s, o);
5536     o->in1 = tcg_temp_new_i64();
5537     tcg_gen_qemu_ld_i64(o->in1, o->addr1, get_mem_index(s), MO_TESW);
5538 }
5539 #define SPEC_in1_m1_16s 0
5540 
5541 static void in1_m1_16u(DisasContext *s, DisasOps *o)
5542 {
5543     in1_la1(s, o);
5544     o->in1 = tcg_temp_new_i64();
5545     tcg_gen_qemu_ld_i64(o->in1, o->addr1, get_mem_index(s), MO_TEUW);
5546 }
5547 #define SPEC_in1_m1_16u 0
5548 
5549 static void in1_m1_32s(DisasContext *s, DisasOps *o)
5550 {
5551     in1_la1(s, o);
5552     o->in1 = tcg_temp_new_i64();
5553     tcg_gen_qemu_ld_i64(o->in1, o->addr1, get_mem_index(s), MO_TESL);
5554 }
5555 #define SPEC_in1_m1_32s 0
5556 
5557 static void in1_m1_32u(DisasContext *s, DisasOps *o)
5558 {
5559     in1_la1(s, o);
5560     o->in1 = tcg_temp_new_i64();
5561     tcg_gen_qemu_ld_i64(o->in1, o->addr1, get_mem_index(s), MO_TEUL);
5562 }
5563 #define SPEC_in1_m1_32u 0
5564 
5565 static void in1_m1_64(DisasContext *s, DisasOps *o)
5566 {
5567     in1_la1(s, o);
5568     o->in1 = tcg_temp_new_i64();
5569     tcg_gen_qemu_ld_i64(o->in1, o->addr1, get_mem_index(s), MO_TEUQ);
5570 }
5571 #define SPEC_in1_m1_64 0
5572 
5573 /* ====================================================================== */
5574 /* The "INput 2" generators.  These load the second operand to an insn.  */
5575 
5576 static void in2_r1_o(DisasContext *s, DisasOps *o)
5577 {
5578     o->in2 = regs[get_field(s, r1)];
5579 }
5580 #define SPEC_in2_r1_o 0
5581 
5582 static void in2_r1_16u(DisasContext *s, DisasOps *o)
5583 {
5584     o->in2 = tcg_temp_new_i64();
5585     tcg_gen_ext16u_i64(o->in2, regs[get_field(s, r1)]);
5586 }
5587 #define SPEC_in2_r1_16u 0
5588 
5589 static void in2_r1_32u(DisasContext *s, DisasOps *o)
5590 {
5591     o->in2 = tcg_temp_new_i64();
5592     tcg_gen_ext32u_i64(o->in2, regs[get_field(s, r1)]);
5593 }
5594 #define SPEC_in2_r1_32u 0
5595 
5596 static void in2_r1_D32(DisasContext *s, DisasOps *o)
5597 {
5598     int r1 = get_field(s, r1);
5599     o->in2 = tcg_temp_new_i64();
5600     tcg_gen_concat32_i64(o->in2, regs[r1 + 1], regs[r1]);
5601 }
5602 #define SPEC_in2_r1_D32 SPEC_r1_even
5603 
5604 static void in2_r2(DisasContext *s, DisasOps *o)
5605 {
5606     o->in2 = load_reg(get_field(s, r2));
5607 }
5608 #define SPEC_in2_r2 0
5609 
5610 static void in2_r2_o(DisasContext *s, DisasOps *o)
5611 {
5612     o->in2 = regs[get_field(s, r2)];
5613 }
5614 #define SPEC_in2_r2_o 0
5615 
5616 static void in2_r2_nz(DisasContext *s, DisasOps *o)
5617 {
5618     int r2 = get_field(s, r2);
5619     if (r2 != 0) {
5620         o->in2 = load_reg(r2);
5621     }
5622 }
5623 #define SPEC_in2_r2_nz 0
5624 
5625 static void in2_r2_8s(DisasContext *s, DisasOps *o)
5626 {
5627     o->in2 = tcg_temp_new_i64();
5628     tcg_gen_ext8s_i64(o->in2, regs[get_field(s, r2)]);
5629 }
5630 #define SPEC_in2_r2_8s 0
5631 
5632 static void in2_r2_8u(DisasContext *s, DisasOps *o)
5633 {
5634     o->in2 = tcg_temp_new_i64();
5635     tcg_gen_ext8u_i64(o->in2, regs[get_field(s, r2)]);
5636 }
5637 #define SPEC_in2_r2_8u 0
5638 
5639 static void in2_r2_16s(DisasContext *s, DisasOps *o)
5640 {
5641     o->in2 = tcg_temp_new_i64();
5642     tcg_gen_ext16s_i64(o->in2, regs[get_field(s, r2)]);
5643 }
5644 #define SPEC_in2_r2_16s 0
5645 
5646 static void in2_r2_16u(DisasContext *s, DisasOps *o)
5647 {
5648     o->in2 = tcg_temp_new_i64();
5649     tcg_gen_ext16u_i64(o->in2, regs[get_field(s, r2)]);
5650 }
5651 #define SPEC_in2_r2_16u 0
5652 
5653 static void in2_r3(DisasContext *s, DisasOps *o)
5654 {
5655     o->in2 = load_reg(get_field(s, r3));
5656 }
5657 #define SPEC_in2_r3 0
5658 
5659 static void in2_r3_D64(DisasContext *s, DisasOps *o)
5660 {
5661     int r3 = get_field(s, r3);
5662     o->in2_128 = tcg_temp_new_i128();
5663     tcg_gen_concat_i64_i128(o->in2_128, regs[r3 + 1], regs[r3]);
5664 }
5665 #define SPEC_in2_r3_D64 SPEC_r3_even
5666 
5667 static void in2_r3_sr32(DisasContext *s, DisasOps *o)
5668 {
5669     o->in2 = tcg_temp_new_i64();
5670     tcg_gen_shri_i64(o->in2, regs[get_field(s, r3)], 32);
5671 }
5672 #define SPEC_in2_r3_sr32 0
5673 
5674 static void in2_r3_32u(DisasContext *s, DisasOps *o)
5675 {
5676     o->in2 = tcg_temp_new_i64();
5677     tcg_gen_ext32u_i64(o->in2, regs[get_field(s, r3)]);
5678 }
5679 #define SPEC_in2_r3_32u 0
5680 
5681 static void in2_r2_32s(DisasContext *s, DisasOps *o)
5682 {
5683     o->in2 = tcg_temp_new_i64();
5684     tcg_gen_ext32s_i64(o->in2, regs[get_field(s, r2)]);
5685 }
5686 #define SPEC_in2_r2_32s 0
5687 
5688 static void in2_r2_32u(DisasContext *s, DisasOps *o)
5689 {
5690     o->in2 = tcg_temp_new_i64();
5691     tcg_gen_ext32u_i64(o->in2, regs[get_field(s, r2)]);
5692 }
5693 #define SPEC_in2_r2_32u 0
5694 
5695 static void in2_r2_sr32(DisasContext *s, DisasOps *o)
5696 {
5697     o->in2 = tcg_temp_new_i64();
5698     tcg_gen_shri_i64(o->in2, regs[get_field(s, r2)], 32);
5699 }
5700 #define SPEC_in2_r2_sr32 0
5701 
5702 static void in2_e2(DisasContext *s, DisasOps *o)
5703 {
5704     o->in2 = load_freg32_i64(get_field(s, r2));
5705 }
5706 #define SPEC_in2_e2 0
5707 
5708 static void in2_f2(DisasContext *s, DisasOps *o)
5709 {
5710     o->in2 = load_freg(get_field(s, r2));
5711 }
5712 #define SPEC_in2_f2 0
5713 
5714 static void in2_x2(DisasContext *s, DisasOps *o)
5715 {
5716     o->in2_128 = load_freg_128(get_field(s, r2));
5717 }
5718 #define SPEC_in2_x2 SPEC_r2_f128
5719 
5720 /* Load the low double word of an extended (128-bit) format FP number */
5721 static void in2_x2l(DisasContext *s, DisasOps *o)
5722 {
5723     o->in2 = load_freg(get_field(s, r2) + 2);
5724 }
5725 #define SPEC_in2_x2l SPEC_r2_f128
5726 
5727 static void in2_ra2(DisasContext *s, DisasOps *o)
5728 {
5729     int r2 = get_field(s, r2);
5730 
5731     /* Note: *don't* treat !r2 as 0, use the reg value. */
5732     o->in2 = tcg_temp_new_i64();
5733     gen_addi_and_wrap_i64(s, o->in2, regs[r2], 0);
5734 }
5735 #define SPEC_in2_ra2 0
5736 
5737 static void in2_ra2_E(DisasContext *s, DisasOps *o)
5738 {
5739     return in2_ra2(s, o);
5740 }
5741 #define SPEC_in2_ra2_E SPEC_r2_even
5742 
5743 static void in2_a2(DisasContext *s, DisasOps *o)
5744 {
5745     int x2 = have_field(s, x2) ? get_field(s, x2) : 0;
5746     o->in2 = get_address(s, x2, get_field(s, b2), get_field(s, d2));
5747 }
5748 #define SPEC_in2_a2 0
5749 
5750 static TCGv gen_ri2(DisasContext *s)
5751 {
5752     TCGv ri2 = NULL;
5753     bool is_imm;
5754     int imm;
5755 
5756     disas_jdest(s, i2, is_imm, imm, ri2);
5757     if (is_imm) {
5758         ri2 = tcg_constant_i64(s->base.pc_next + (int64_t)imm * 2);
5759     }
5760 
5761     return ri2;
5762 }
5763 
5764 static void in2_ri2(DisasContext *s, DisasOps *o)
5765 {
5766     o->in2 = gen_ri2(s);
5767 }
5768 #define SPEC_in2_ri2 0
5769 
5770 static void in2_sh(DisasContext *s, DisasOps *o)
5771 {
5772     int b2 = get_field(s, b2);
5773     int d2 = get_field(s, d2);
5774 
5775     if (b2 == 0) {
5776         o->in2 = tcg_constant_i64(d2 & 0x3f);
5777     } else {
5778         o->in2 = get_address(s, 0, b2, d2);
5779         tcg_gen_andi_i64(o->in2, o->in2, 0x3f);
5780     }
5781 }
5782 #define SPEC_in2_sh 0
5783 
5784 static void in2_m2_8u(DisasContext *s, DisasOps *o)
5785 {
5786     in2_a2(s, o);
5787     tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_UB);
5788 }
5789 #define SPEC_in2_m2_8u 0
5790 
5791 static void in2_m2_16s(DisasContext *s, DisasOps *o)
5792 {
5793     in2_a2(s, o);
5794     tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_TESW);
5795 }
5796 #define SPEC_in2_m2_16s 0
5797 
5798 static void in2_m2_16u(DisasContext *s, DisasOps *o)
5799 {
5800     in2_a2(s, o);
5801     tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_TEUW);
5802 }
5803 #define SPEC_in2_m2_16u 0
5804 
5805 static void in2_m2_32s(DisasContext *s, DisasOps *o)
5806 {
5807     in2_a2(s, o);
5808     tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_TESL);
5809 }
5810 #define SPEC_in2_m2_32s 0
5811 
5812 static void in2_m2_32u(DisasContext *s, DisasOps *o)
5813 {
5814     in2_a2(s, o);
5815     tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_TEUL);
5816 }
5817 #define SPEC_in2_m2_32u 0
5818 
5819 #ifndef CONFIG_USER_ONLY
5820 static void in2_m2_32ua(DisasContext *s, DisasOps *o)
5821 {
5822     in2_a2(s, o);
5823     tcg_gen_qemu_ld_tl(o->in2, o->in2, get_mem_index(s), MO_TEUL | MO_ALIGN);
5824 }
5825 #define SPEC_in2_m2_32ua 0
5826 #endif
5827 
5828 static void in2_m2_64(DisasContext *s, DisasOps *o)
5829 {
5830     in2_a2(s, o);
5831     tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_TEUQ);
5832 }
5833 #define SPEC_in2_m2_64 0
5834 
5835 static void in2_m2_64w(DisasContext *s, DisasOps *o)
5836 {
5837     in2_a2(s, o);
5838     tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_TEUQ);
5839     gen_addi_and_wrap_i64(s, o->in2, o->in2, 0);
5840 }
5841 #define SPEC_in2_m2_64w 0
5842 
5843 #ifndef CONFIG_USER_ONLY
5844 static void in2_m2_64a(DisasContext *s, DisasOps *o)
5845 {
5846     in2_a2(s, o);
5847     tcg_gen_qemu_ld_i64(o->in2, o->in2, get_mem_index(s), MO_TEUQ | MO_ALIGN);
5848 }
5849 #define SPEC_in2_m2_64a 0
5850 #endif
5851 
5852 static void in2_mri2_16s(DisasContext *s, DisasOps *o)
5853 {
5854     o->in2 = tcg_temp_new_i64();
5855     tcg_gen_qemu_ld_i64(o->in2, gen_ri2(s), get_mem_index(s), MO_TESW);
5856 }
5857 #define SPEC_in2_mri2_16s 0
5858 
5859 static void in2_mri2_16u(DisasContext *s, DisasOps *o)
5860 {
5861     o->in2 = tcg_temp_new_i64();
5862     tcg_gen_qemu_ld_i64(o->in2, gen_ri2(s), get_mem_index(s), MO_TEUW);
5863 }
5864 #define SPEC_in2_mri2_16u 0
5865 
5866 static void in2_mri2_32s(DisasContext *s, DisasOps *o)
5867 {
5868     o->in2 = tcg_temp_new_i64();
5869     tcg_gen_qemu_ld_tl(o->in2, gen_ri2(s), get_mem_index(s),
5870                        MO_TESL | MO_ALIGN);
5871 }
5872 #define SPEC_in2_mri2_32s 0
5873 
5874 static void in2_mri2_32u(DisasContext *s, DisasOps *o)
5875 {
5876     o->in2 = tcg_temp_new_i64();
5877     tcg_gen_qemu_ld_tl(o->in2, gen_ri2(s), get_mem_index(s),
5878                        MO_TEUL | MO_ALIGN);
5879 }
5880 #define SPEC_in2_mri2_32u 0
5881 
5882 static void in2_mri2_64(DisasContext *s, DisasOps *o)
5883 {
5884     o->in2 = tcg_temp_new_i64();
5885     tcg_gen_qemu_ld_i64(o->in2, gen_ri2(s), get_mem_index(s),
5886                         MO_TEUQ | MO_ALIGN);
5887 }
5888 #define SPEC_in2_mri2_64 0
5889 
5890 static void in2_i2(DisasContext *s, DisasOps *o)
5891 {
5892     o->in2 = tcg_constant_i64(get_field(s, i2));
5893 }
5894 #define SPEC_in2_i2 0
5895 
5896 static void in2_i2_8u(DisasContext *s, DisasOps *o)
5897 {
5898     o->in2 = tcg_constant_i64((uint8_t)get_field(s, i2));
5899 }
5900 #define SPEC_in2_i2_8u 0
5901 
5902 static void in2_i2_16u(DisasContext *s, DisasOps *o)
5903 {
5904     o->in2 = tcg_constant_i64((uint16_t)get_field(s, i2));
5905 }
5906 #define SPEC_in2_i2_16u 0
5907 
5908 static void in2_i2_32u(DisasContext *s, DisasOps *o)
5909 {
5910     o->in2 = tcg_constant_i64((uint32_t)get_field(s, i2));
5911 }
5912 #define SPEC_in2_i2_32u 0
5913 
5914 static void in2_i2_16u_shl(DisasContext *s, DisasOps *o)
5915 {
5916     uint64_t i2 = (uint16_t)get_field(s, i2);
5917     o->in2 = tcg_constant_i64(i2 << s->insn->data);
5918 }
5919 #define SPEC_in2_i2_16u_shl 0
5920 
5921 static void in2_i2_32u_shl(DisasContext *s, DisasOps *o)
5922 {
5923     uint64_t i2 = (uint32_t)get_field(s, i2);
5924     o->in2 = tcg_constant_i64(i2 << s->insn->data);
5925 }
5926 #define SPEC_in2_i2_32u_shl 0
5927 
5928 #ifndef CONFIG_USER_ONLY
5929 static void in2_insn(DisasContext *s, DisasOps *o)
5930 {
5931     o->in2 = tcg_constant_i64(s->fields.raw_insn);
5932 }
5933 #define SPEC_in2_insn 0
5934 #endif
5935 
5936 /* ====================================================================== */
5937 
5938 /* Find opc within the table of insns.  This is formulated as a switch
5939    statement so that (1) we get compile-time notice of cut-paste errors
5940    for duplicated opcodes, and (2) the compiler generates the binary
5941    search tree, rather than us having to post-process the table.  */
5942 
5943 #define C(OPC, NM, FT, FC, I1, I2, P, W, OP, CC) \
5944     E(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, 0, 0)
5945 
5946 #define D(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, D) \
5947     E(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, D, 0)
5948 
5949 #define F(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, FL) \
5950     E(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, 0, FL)
5951 
5952 #define E(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, D, FL) insn_ ## NM,
5953 
5954 enum DisasInsnEnum {
5955 #include "insn-data.h.inc"
5956 };
5957 
5958 #undef E
5959 #define E(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, D, FL) {                   \
5960     .opc = OPC,                                                             \
5961     .flags = FL,                                                            \
5962     .fmt = FMT_##FT,                                                        \
5963     .fac = FAC_##FC,                                                        \
5964     .spec = SPEC_in1_##I1 | SPEC_in2_##I2 | SPEC_prep_##P | SPEC_wout_##W,  \
5965     .name = #NM,                                                            \
5966     .help_in1 = in1_##I1,                                                   \
5967     .help_in2 = in2_##I2,                                                   \
5968     .help_prep = prep_##P,                                                  \
5969     .help_wout = wout_##W,                                                  \
5970     .help_cout = cout_##CC,                                                 \
5971     .help_op = op_##OP,                                                     \
5972     .data = D                                                               \
5973  },
5974 
5975 /* Allow 0 to be used for NULL in the table below.  */
5976 #define in1_0  NULL
5977 #define in2_0  NULL
5978 #define prep_0  NULL
5979 #define wout_0  NULL
5980 #define cout_0  NULL
5981 #define op_0  NULL
5982 
5983 #define SPEC_in1_0 0
5984 #define SPEC_in2_0 0
5985 #define SPEC_prep_0 0
5986 #define SPEC_wout_0 0
5987 
5988 /* Give smaller names to the various facilities.  */
5989 #define FAC_Z           S390_FEAT_ZARCH
5990 #define FAC_CASS        S390_FEAT_COMPARE_AND_SWAP_AND_STORE
5991 #define FAC_DFP         S390_FEAT_DFP
5992 #define FAC_DFPR        S390_FEAT_FLOATING_POINT_SUPPORT_ENH /* DFP-rounding */
5993 #define FAC_DO          S390_FEAT_STFLE_45 /* distinct-operands */
5994 #define FAC_EE          S390_FEAT_EXECUTE_EXT
5995 #define FAC_EI          S390_FEAT_EXTENDED_IMMEDIATE
5996 #define FAC_FPE         S390_FEAT_FLOATING_POINT_EXT
5997 #define FAC_FPSSH       S390_FEAT_FLOATING_POINT_SUPPORT_ENH /* FPS-sign-handling */
5998 #define FAC_FPRGR       S390_FEAT_FLOATING_POINT_SUPPORT_ENH /* FPR-GR-transfer */
5999 #define FAC_GIE         S390_FEAT_GENERAL_INSTRUCTIONS_EXT
6000 #define FAC_HFP_MA      S390_FEAT_HFP_MADDSUB
6001 #define FAC_HW          S390_FEAT_STFLE_45 /* high-word */
6002 #define FAC_IEEEE_SIM   S390_FEAT_FLOATING_POINT_SUPPORT_ENH /* IEEE-exception-simulation */
6003 #define FAC_MIE         S390_FEAT_STFLE_49 /* misc-instruction-extensions */
6004 #define FAC_LAT         S390_FEAT_STFLE_49 /* load-and-trap */
6005 #define FAC_LOC         S390_FEAT_STFLE_45 /* load/store on condition 1 */
6006 #define FAC_LOC2        S390_FEAT_STFLE_53 /* load/store on condition 2 */
6007 #define FAC_LD          S390_FEAT_LONG_DISPLACEMENT
6008 #define FAC_PC          S390_FEAT_STFLE_45 /* population count */
6009 #define FAC_SCF         S390_FEAT_STORE_CLOCK_FAST
6010 #define FAC_SFLE        S390_FEAT_STFLE
6011 #define FAC_ILA         S390_FEAT_STFLE_45 /* interlocked-access-facility 1 */
6012 #define FAC_MVCOS       S390_FEAT_MOVE_WITH_OPTIONAL_SPEC
6013 #define FAC_LPP         S390_FEAT_SET_PROGRAM_PARAMETERS /* load-program-parameter */
6014 #define FAC_DAT_ENH     S390_FEAT_DAT_ENH
6015 #define FAC_E2          S390_FEAT_EXTENDED_TRANSLATION_2
6016 #define FAC_EH          S390_FEAT_STFLE_49 /* execution-hint */
6017 #define FAC_PPA         S390_FEAT_STFLE_49 /* processor-assist */
6018 #define FAC_LZRB        S390_FEAT_STFLE_53 /* load-and-zero-rightmost-byte */
6019 #define FAC_ETF3        S390_FEAT_EXTENDED_TRANSLATION_3
6020 #define FAC_MSA         S390_FEAT_MSA /* message-security-assist facility */
6021 #define FAC_MSA3        S390_FEAT_MSA_EXT_3 /* msa-extension-3 facility */
6022 #define FAC_MSA4        S390_FEAT_MSA_EXT_4 /* msa-extension-4 facility */
6023 #define FAC_MSA5        S390_FEAT_MSA_EXT_5 /* msa-extension-5 facility */
6024 #define FAC_MSA8        S390_FEAT_MSA_EXT_8 /* msa-extension-8 facility */
6025 #define FAC_ECT         S390_FEAT_EXTRACT_CPU_TIME
6026 #define FAC_PCI         S390_FEAT_ZPCI /* z/PCI facility */
6027 #define FAC_AIS         S390_FEAT_ADAPTER_INT_SUPPRESSION
6028 #define FAC_V           S390_FEAT_VECTOR /* vector facility */
6029 #define FAC_VE          S390_FEAT_VECTOR_ENH  /* vector enhancements facility 1 */
6030 #define FAC_VE2         S390_FEAT_VECTOR_ENH2 /* vector enhancements facility 2 */
6031 #define FAC_MIE2        S390_FEAT_MISC_INSTRUCTION_EXT2 /* miscellaneous-instruction-extensions facility 2 */
6032 #define FAC_MIE3        S390_FEAT_MISC_INSTRUCTION_EXT3 /* miscellaneous-instruction-extensions facility 3 */
6033 
6034 static const DisasInsn insn_info[] = {
6035 #include "insn-data.h.inc"
6036 };
6037 
6038 #undef E
6039 #define E(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, D, FL) \
6040     case OPC: return &insn_info[insn_ ## NM];
6041 
6042 static const DisasInsn *lookup_opc(uint16_t opc)
6043 {
6044     switch (opc) {
6045 #include "insn-data.h.inc"
6046     default:
6047         return NULL;
6048     }
6049 }
6050 
6051 #undef F
6052 #undef E
6053 #undef D
6054 #undef C
6055 
6056 /* Extract a field from the insn.  The INSN should be left-aligned in
6057    the uint64_t so that we can more easily utilize the big-bit-endian
6058    definitions we extract from the Principals of Operation.  */
6059 
6060 static void extract_field(DisasFields *o, const DisasField *f, uint64_t insn)
6061 {
6062     uint32_t r, m;
6063 
6064     if (f->size == 0) {
6065         return;
6066     }
6067 
6068     /* Zero extract the field from the insn.  */
6069     r = (insn << f->beg) >> (64 - f->size);
6070 
6071     /* Sign-extend, or un-swap the field as necessary.  */
6072     switch (f->type) {
6073     case 0: /* unsigned */
6074         break;
6075     case 1: /* signed */
6076         assert(f->size <= 32);
6077         m = 1u << (f->size - 1);
6078         r = (r ^ m) - m;
6079         break;
6080     case 2: /* dl+dh split, signed 20 bit. */
6081         r = ((int8_t)r << 12) | (r >> 8);
6082         break;
6083     case 3: /* MSB stored in RXB */
6084         g_assert(f->size == 4);
6085         switch (f->beg) {
6086         case 8:
6087             r |= extract64(insn, 63 - 36, 1) << 4;
6088             break;
6089         case 12:
6090             r |= extract64(insn, 63 - 37, 1) << 4;
6091             break;
6092         case 16:
6093             r |= extract64(insn, 63 - 38, 1) << 4;
6094             break;
6095         case 32:
6096             r |= extract64(insn, 63 - 39, 1) << 4;
6097             break;
6098         default:
6099             g_assert_not_reached();
6100         }
6101         break;
6102     default:
6103         abort();
6104     }
6105 
6106     /*
6107      * Validate that the "compressed" encoding we selected above is valid.
6108      * I.e. we haven't made two different original fields overlap.
6109      */
6110     assert(((o->presentC >> f->indexC) & 1) == 0);
6111     o->presentC |= 1 << f->indexC;
6112     o->presentO |= 1 << f->indexO;
6113 
6114     o->c[f->indexC] = r;
6115 }
6116 
6117 /* Lookup the insn at the current PC, extracting the operands into O and
6118    returning the info struct for the insn.  Returns NULL for invalid insn.  */
6119 
6120 static const DisasInsn *extract_insn(CPUS390XState *env, DisasContext *s)
6121 {
6122     uint64_t insn, pc = s->base.pc_next;
6123     int op, op2, ilen;
6124     const DisasInsn *info;
6125 
6126     if (unlikely(s->ex_value)) {
6127         uint64_t be_insn;
6128 
6129         /* Drop the EX data now, so that it's clear on exception paths.  */
6130         tcg_gen_st_i64(tcg_constant_i64(0), tcg_env,
6131                        offsetof(CPUS390XState, ex_value));
6132 
6133         /* Extract the values saved by EXECUTE.  */
6134         insn = s->ex_value & 0xffffffffffff0000ull;
6135         ilen = s->ex_value & 0xf;
6136         op = insn >> 56;
6137 
6138         /* Register insn bytes with translator so plugins work. */
6139         be_insn = cpu_to_be64(insn);
6140         translator_fake_ld(&s->base, &be_insn, get_ilen(op));
6141     } else {
6142         insn = ld_code2(env, s, pc);
6143         op = (insn >> 8) & 0xff;
6144         ilen = get_ilen(op);
6145         switch (ilen) {
6146         case 2:
6147             insn = insn << 48;
6148             break;
6149         case 4:
6150             insn = ld_code4(env, s, pc) << 32;
6151             break;
6152         case 6:
6153             insn = (insn << 48) | (ld_code4(env, s, pc + 2) << 16);
6154             break;
6155         default:
6156             g_assert_not_reached();
6157         }
6158     }
6159     s->pc_tmp = s->base.pc_next + ilen;
6160     s->ilen = ilen;
6161 
6162     /* We can't actually determine the insn format until we've looked up
6163        the full insn opcode.  Which we can't do without locating the
6164        secondary opcode.  Assume by default that OP2 is at bit 40; for
6165        those smaller insns that don't actually have a secondary opcode
6166        this will correctly result in OP2 = 0. */
6167     switch (op) {
6168     case 0x01: /* E */
6169     case 0x80: /* S */
6170     case 0x82: /* S */
6171     case 0x93: /* S */
6172     case 0xb2: /* S, RRF, RRE, IE */
6173     case 0xb3: /* RRE, RRD, RRF */
6174     case 0xb9: /* RRE, RRF */
6175     case 0xe5: /* SSE, SIL */
6176         op2 = (insn << 8) >> 56;
6177         break;
6178     case 0xa5: /* RI */
6179     case 0xa7: /* RI */
6180     case 0xc0: /* RIL */
6181     case 0xc2: /* RIL */
6182     case 0xc4: /* RIL */
6183     case 0xc6: /* RIL */
6184     case 0xc8: /* SSF */
6185     case 0xcc: /* RIL */
6186         op2 = (insn << 12) >> 60;
6187         break;
6188     case 0xc5: /* MII */
6189     case 0xc7: /* SMI */
6190     case 0xd0 ... 0xdf: /* SS */
6191     case 0xe1: /* SS */
6192     case 0xe2: /* SS */
6193     case 0xe8: /* SS */
6194     case 0xe9: /* SS */
6195     case 0xea: /* SS */
6196     case 0xee ... 0xf3: /* SS */
6197     case 0xf8 ... 0xfd: /* SS */
6198         op2 = 0;
6199         break;
6200     default:
6201         op2 = (insn << 40) >> 56;
6202         break;
6203     }
6204 
6205     memset(&s->fields, 0, sizeof(s->fields));
6206     s->fields.raw_insn = insn;
6207     s->fields.op = op;
6208     s->fields.op2 = op2;
6209 
6210     /* Lookup the instruction.  */
6211     info = lookup_opc(op << 8 | op2);
6212     s->insn = info;
6213 
6214     /* If we found it, extract the operands.  */
6215     if (info != NULL) {
6216         DisasFormat fmt = info->fmt;
6217         int i;
6218 
6219         for (i = 0; i < NUM_C_FIELD; ++i) {
6220             extract_field(&s->fields, &format_info[fmt].op[i], insn);
6221         }
6222     }
6223     return info;
6224 }
6225 
6226 static bool is_afp_reg(int reg)
6227 {
6228     return reg % 2 || reg > 6;
6229 }
6230 
6231 static bool is_fp_pair(int reg)
6232 {
6233     /* 0,1,4,5,8,9,12,13: to exclude the others, check for single bit */
6234     return !(reg & 0x2);
6235 }
6236 
6237 static DisasJumpType translate_one(CPUS390XState *env, DisasContext *s)
6238 {
6239     const DisasInsn *insn;
6240     DisasJumpType ret = DISAS_NEXT;
6241     DisasOps o = {};
6242     bool icount = false;
6243 
6244     /* Search for the insn in the table.  */
6245     insn = extract_insn(env, s);
6246 
6247     /* Update insn_start now that we know the ILEN.  */
6248     tcg_set_insn_start_param(s->base.insn_start, 2, s->ilen);
6249 
6250     /* Not found means unimplemented/illegal opcode.  */
6251     if (insn == NULL) {
6252         qemu_log_mask(LOG_UNIMP, "unimplemented opcode 0x%02x%02x\n",
6253                       s->fields.op, s->fields.op2);
6254         gen_illegal_opcode(s);
6255         ret = DISAS_NORETURN;
6256         goto out;
6257     }
6258 
6259 #ifndef CONFIG_USER_ONLY
6260     if (s->base.tb->flags & FLAG_MASK_PER_IFETCH) {
6261         /* With ifetch set, psw_addr and cc_op are always up-to-date. */
6262         gen_helper_per_ifetch(tcg_env, tcg_constant_i32(s->ilen));
6263     }
6264 #endif
6265 
6266     /* process flags */
6267     if (insn->flags) {
6268         /* privileged instruction */
6269         if ((s->base.tb->flags & FLAG_MASK_PSTATE) && (insn->flags & IF_PRIV)) {
6270             gen_program_exception(s, PGM_PRIVILEGED);
6271             ret = DISAS_NORETURN;
6272             goto out;
6273         }
6274 
6275         /* if AFP is not enabled, instructions and registers are forbidden */
6276         if (!(s->base.tb->flags & FLAG_MASK_AFP)) {
6277             uint8_t dxc = 0;
6278 
6279             if ((insn->flags & IF_AFP1) && is_afp_reg(get_field(s, r1))) {
6280                 dxc = 1;
6281             }
6282             if ((insn->flags & IF_AFP2) && is_afp_reg(get_field(s, r2))) {
6283                 dxc = 1;
6284             }
6285             if ((insn->flags & IF_AFP3) && is_afp_reg(get_field(s, r3))) {
6286                 dxc = 1;
6287             }
6288             if (insn->flags & IF_BFP) {
6289                 dxc = 2;
6290             }
6291             if (insn->flags & IF_DFP) {
6292                 dxc = 3;
6293             }
6294             if (insn->flags & IF_VEC) {
6295                 dxc = 0xfe;
6296             }
6297             if (dxc) {
6298                 gen_data_exception(dxc);
6299                 ret = DISAS_NORETURN;
6300                 goto out;
6301             }
6302         }
6303 
6304         /* if vector instructions not enabled, executing them is forbidden */
6305         if (insn->flags & IF_VEC) {
6306             if (!((s->base.tb->flags & FLAG_MASK_VECTOR))) {
6307                 gen_data_exception(0xfe);
6308                 ret = DISAS_NORETURN;
6309                 goto out;
6310             }
6311         }
6312 
6313         /* input/output is the special case for icount mode */
6314         if (unlikely(insn->flags & IF_IO)) {
6315             icount = translator_io_start(&s->base);
6316         }
6317     }
6318 
6319     /* Check for insn specification exceptions.  */
6320     if (insn->spec) {
6321         if ((insn->spec & SPEC_r1_even && get_field(s, r1) & 1) ||
6322             (insn->spec & SPEC_r2_even && get_field(s, r2) & 1) ||
6323             (insn->spec & SPEC_r3_even && get_field(s, r3) & 1) ||
6324             (insn->spec & SPEC_r1_f128 && !is_fp_pair(get_field(s, r1))) ||
6325             (insn->spec & SPEC_r2_f128 && !is_fp_pair(get_field(s, r2)))) {
6326             gen_program_exception(s, PGM_SPECIFICATION);
6327             ret = DISAS_NORETURN;
6328             goto out;
6329         }
6330     }
6331 
6332     /* Implement the instruction.  */
6333     if (insn->help_in1) {
6334         insn->help_in1(s, &o);
6335     }
6336     if (insn->help_in2) {
6337         insn->help_in2(s, &o);
6338     }
6339     if (insn->help_prep) {
6340         insn->help_prep(s, &o);
6341     }
6342     if (insn->help_op) {
6343         ret = insn->help_op(s, &o);
6344         if (ret == DISAS_NORETURN) {
6345             goto out;
6346         }
6347     }
6348     if (insn->help_wout) {
6349         insn->help_wout(s, &o);
6350     }
6351     if (insn->help_cout) {
6352         insn->help_cout(s, &o);
6353     }
6354 
6355     /* io should be the last instruction in tb when icount is enabled */
6356     if (unlikely(icount && ret == DISAS_NEXT)) {
6357         ret = DISAS_TOO_MANY;
6358     }
6359 
6360 #ifndef CONFIG_USER_ONLY
6361     if (s->base.tb->flags & FLAG_MASK_PER_IFETCH) {
6362         switch (ret) {
6363         case DISAS_TOO_MANY:
6364             s->base.is_jmp = DISAS_PC_CC_UPDATED;
6365             /* fall through */
6366         case DISAS_NEXT:
6367             tcg_gen_movi_i64(psw_addr, s->pc_tmp);
6368             break;
6369         default:
6370             break;
6371         }
6372         update_cc_op(s);
6373         gen_helper_per_check_exception(tcg_env);
6374     }
6375 #endif
6376 
6377 out:
6378     /* Advance to the next instruction.  */
6379     s->base.pc_next = s->pc_tmp;
6380     return ret;
6381 }
6382 
6383 static void s390x_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs)
6384 {
6385     DisasContext *dc = container_of(dcbase, DisasContext, base);
6386 
6387     /* 31-bit mode */
6388     if (!(dc->base.tb->flags & FLAG_MASK_64)) {
6389         dc->base.pc_first &= 0x7fffffff;
6390         dc->base.pc_next = dc->base.pc_first;
6391     }
6392 
6393     dc->cc_op = CC_OP_DYNAMIC;
6394     dc->ex_value = dc->base.tb->cs_base;
6395     dc->exit_to_mainloop = dc->ex_value;
6396 }
6397 
6398 static void s390x_tr_tb_start(DisasContextBase *db, CPUState *cs)
6399 {
6400 }
6401 
6402 static void s390x_tr_insn_start(DisasContextBase *dcbase, CPUState *cs)
6403 {
6404     DisasContext *dc = container_of(dcbase, DisasContext, base);
6405 
6406     /* Delay the set of ilen until we've read the insn. */
6407     tcg_gen_insn_start(dc->base.pc_next, dc->cc_op, 0);
6408 }
6409 
6410 static target_ulong get_next_pc(CPUS390XState *env, DisasContext *s,
6411                                 uint64_t pc)
6412 {
6413     uint64_t insn = translator_lduw(env, &s->base, pc);
6414 
6415     return pc + get_ilen((insn >> 8) & 0xff);
6416 }
6417 
6418 static void s390x_tr_translate_insn(DisasContextBase *dcbase, CPUState *cs)
6419 {
6420     CPUS390XState *env = cpu_env(cs);
6421     DisasContext *dc = container_of(dcbase, DisasContext, base);
6422 
6423     dc->base.is_jmp = translate_one(env, dc);
6424     if (dc->base.is_jmp == DISAS_NEXT) {
6425         if (dc->ex_value ||
6426             !is_same_page(dcbase, dc->base.pc_next) ||
6427             !is_same_page(dcbase, get_next_pc(env, dc, dc->base.pc_next))) {
6428             dc->base.is_jmp = DISAS_TOO_MANY;
6429         }
6430     }
6431 }
6432 
6433 static void s390x_tr_tb_stop(DisasContextBase *dcbase, CPUState *cs)
6434 {
6435     DisasContext *dc = container_of(dcbase, DisasContext, base);
6436 
6437     switch (dc->base.is_jmp) {
6438     case DISAS_NORETURN:
6439         break;
6440     case DISAS_TOO_MANY:
6441         update_psw_addr(dc);
6442         /* FALLTHRU */
6443     case DISAS_PC_UPDATED:
6444         /* Next TB starts off with CC_OP_DYNAMIC, so make sure the
6445            cc op type is in env */
6446         update_cc_op(dc);
6447         /* FALLTHRU */
6448     case DISAS_PC_CC_UPDATED:
6449         /* Exit the TB, either by raising a debug exception or by return.  */
6450         if (dc->exit_to_mainloop) {
6451             tcg_gen_exit_tb(NULL, 0);
6452         } else {
6453             tcg_gen_lookup_and_goto_ptr();
6454         }
6455         break;
6456     default:
6457         g_assert_not_reached();
6458     }
6459 }
6460 
6461 static bool s390x_tr_disas_log(const DisasContextBase *dcbase,
6462                                CPUState *cs, FILE *logfile)
6463 {
6464     DisasContext *dc = container_of(dcbase, DisasContext, base);
6465 
6466     if (unlikely(dc->ex_value)) {
6467         /* The ex_value has been recorded with translator_fake_ld. */
6468         fprintf(logfile, "IN: EXECUTE\n");
6469         target_disas(logfile, cs, &dc->base);
6470         return true;
6471     }
6472     return false;
6473 }
6474 
6475 static const TranslatorOps s390x_tr_ops = {
6476     .init_disas_context = s390x_tr_init_disas_context,
6477     .tb_start           = s390x_tr_tb_start,
6478     .insn_start         = s390x_tr_insn_start,
6479     .translate_insn     = s390x_tr_translate_insn,
6480     .tb_stop            = s390x_tr_tb_stop,
6481     .disas_log          = s390x_tr_disas_log,
6482 };
6483 
6484 void gen_intermediate_code(CPUState *cs, TranslationBlock *tb, int *max_insns,
6485                            vaddr pc, void *host_pc)
6486 {
6487     DisasContext dc;
6488 
6489     translator_loop(cs, tb, max_insns, pc, host_pc, &s390x_tr_ops, &dc.base);
6490 }
6491 
6492 void s390x_restore_state_to_opc(CPUState *cs,
6493                                 const TranslationBlock *tb,
6494                                 const uint64_t *data)
6495 {
6496     CPUS390XState *env = cpu_env(cs);
6497     int cc_op = data[1];
6498 
6499     env->psw.addr = data[0];
6500 
6501     /* Update the CC opcode if it is not already up-to-date.  */
6502     if ((cc_op != CC_OP_DYNAMIC) && (cc_op != CC_OP_STATIC)) {
6503         env->cc_op = cc_op;
6504     }
6505 
6506     /* Record ILEN.  */
6507     env->int_pgm_ilen = data[2];
6508 }
6509