xref: /openbmc/qemu/tcg/tcg-op.c (revision 6538692e)
1 /*
2  * Tiny Code Generator for QEMU
3  *
4  * Copyright (c) 2008 Fabrice Bellard
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22  * THE SOFTWARE.
23  */
24 
25 #include "qemu/osdep.h"
26 #include "exec/exec-all.h"
27 #include "tcg/tcg.h"
28 #include "tcg/tcg-op.h"
29 #include "tcg/tcg-mo.h"
30 #include "exec/plugin-gen.h"
31 
32 /* Reduce the number of ifdefs below.  This assumes that all uses of
33    TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
34    the compiler can eliminate.  */
35 #if TCG_TARGET_REG_BITS == 64
36 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
37 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
38 #define TCGV_LOW  TCGV_LOW_link_error
39 #define TCGV_HIGH TCGV_HIGH_link_error
40 #endif
41 
42 void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
43 {
44     TCGOp *op = tcg_emit_op(opc);
45     op->args[0] = a1;
46 }
47 
48 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
49 {
50     TCGOp *op = tcg_emit_op(opc);
51     op->args[0] = a1;
52     op->args[1] = a2;
53 }
54 
55 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
56 {
57     TCGOp *op = tcg_emit_op(opc);
58     op->args[0] = a1;
59     op->args[1] = a2;
60     op->args[2] = a3;
61 }
62 
63 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
64 {
65     TCGOp *op = tcg_emit_op(opc);
66     op->args[0] = a1;
67     op->args[1] = a2;
68     op->args[2] = a3;
69     op->args[3] = a4;
70 }
71 
72 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
73                  TCGArg a4, TCGArg a5)
74 {
75     TCGOp *op = tcg_emit_op(opc);
76     op->args[0] = a1;
77     op->args[1] = a2;
78     op->args[2] = a3;
79     op->args[3] = a4;
80     op->args[4] = a5;
81 }
82 
83 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
84                  TCGArg a4, TCGArg a5, TCGArg a6)
85 {
86     TCGOp *op = tcg_emit_op(opc);
87     op->args[0] = a1;
88     op->args[1] = a2;
89     op->args[2] = a3;
90     op->args[3] = a4;
91     op->args[4] = a5;
92     op->args[5] = a6;
93 }
94 
95 void tcg_gen_mb(TCGBar mb_type)
96 {
97     if (tcg_ctx->tb_cflags & CF_PARALLEL) {
98         tcg_gen_op1(INDEX_op_mb, mb_type);
99     }
100 }
101 
102 /* 32 bit ops */
103 
104 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg)
105 {
106     tcg_gen_mov_i32(ret, tcg_constant_i32(arg));
107 }
108 
109 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
110 {
111     /* some cases can be optimized here */
112     if (arg2 == 0) {
113         tcg_gen_mov_i32(ret, arg1);
114     } else {
115         tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2));
116     }
117 }
118 
119 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
120 {
121     if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
122         /* Don't recurse with tcg_gen_neg_i32.  */
123         tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
124     } else {
125         tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2);
126     }
127 }
128 
129 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
130 {
131     /* some cases can be optimized here */
132     if (arg2 == 0) {
133         tcg_gen_mov_i32(ret, arg1);
134     } else {
135         tcg_gen_sub_i32(ret, arg1, tcg_constant_i32(arg2));
136     }
137 }
138 
139 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
140 {
141     /* Some cases can be optimized here.  */
142     switch (arg2) {
143     case 0:
144         tcg_gen_movi_i32(ret, 0);
145         return;
146     case -1:
147         tcg_gen_mov_i32(ret, arg1);
148         return;
149     case 0xff:
150         /* Don't recurse with tcg_gen_ext8u_i32.  */
151         if (TCG_TARGET_HAS_ext8u_i32) {
152             tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
153             return;
154         }
155         break;
156     case 0xffff:
157         if (TCG_TARGET_HAS_ext16u_i32) {
158             tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
159             return;
160         }
161         break;
162     }
163 
164     tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2));
165 }
166 
167 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
168 {
169     /* Some cases can be optimized here.  */
170     if (arg2 == -1) {
171         tcg_gen_movi_i32(ret, -1);
172     } else if (arg2 == 0) {
173         tcg_gen_mov_i32(ret, arg1);
174     } else {
175         tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2));
176     }
177 }
178 
179 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
180 {
181     /* Some cases can be optimized here.  */
182     if (arg2 == 0) {
183         tcg_gen_mov_i32(ret, arg1);
184     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
185         /* Don't recurse with tcg_gen_not_i32.  */
186         tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
187     } else {
188         tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2));
189     }
190 }
191 
192 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
193 {
194     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
195     if (arg2 == 0) {
196         tcg_gen_mov_i32(ret, arg1);
197     } else {
198         tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2));
199     }
200 }
201 
202 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
203 {
204     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
205     if (arg2 == 0) {
206         tcg_gen_mov_i32(ret, arg1);
207     } else {
208         tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2));
209     }
210 }
211 
212 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
213 {
214     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
215     if (arg2 == 0) {
216         tcg_gen_mov_i32(ret, arg1);
217     } else {
218         tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2));
219     }
220 }
221 
222 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
223 {
224     if (cond == TCG_COND_ALWAYS) {
225         tcg_gen_br(l);
226     } else if (cond != TCG_COND_NEVER) {
227         l->refs++;
228         tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
229     }
230 }
231 
232 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
233 {
234     if (cond == TCG_COND_ALWAYS) {
235         tcg_gen_br(l);
236     } else if (cond != TCG_COND_NEVER) {
237         tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l);
238     }
239 }
240 
241 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
242                          TCGv_i32 arg1, TCGv_i32 arg2)
243 {
244     if (cond == TCG_COND_ALWAYS) {
245         tcg_gen_movi_i32(ret, 1);
246     } else if (cond == TCG_COND_NEVER) {
247         tcg_gen_movi_i32(ret, 0);
248     } else {
249         tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
250     }
251 }
252 
253 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
254                           TCGv_i32 arg1, int32_t arg2)
255 {
256     tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2));
257 }
258 
259 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
260 {
261     if (arg2 == 0) {
262         tcg_gen_movi_i32(ret, 0);
263     } else if (is_power_of_2(arg2)) {
264         tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
265     } else {
266         tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2));
267     }
268 }
269 
270 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
271 {
272     if (TCG_TARGET_HAS_div_i32) {
273         tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
274     } else if (TCG_TARGET_HAS_div2_i32) {
275         TCGv_i32 t0 = tcg_temp_new_i32();
276         tcg_gen_sari_i32(t0, arg1, 31);
277         tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
278         tcg_temp_free_i32(t0);
279     } else {
280         gen_helper_div_i32(ret, arg1, arg2);
281     }
282 }
283 
284 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
285 {
286     if (TCG_TARGET_HAS_rem_i32) {
287         tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
288     } else if (TCG_TARGET_HAS_div_i32) {
289         TCGv_i32 t0 = tcg_temp_new_i32();
290         tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
291         tcg_gen_mul_i32(t0, t0, arg2);
292         tcg_gen_sub_i32(ret, arg1, t0);
293         tcg_temp_free_i32(t0);
294     } else if (TCG_TARGET_HAS_div2_i32) {
295         TCGv_i32 t0 = tcg_temp_new_i32();
296         tcg_gen_sari_i32(t0, arg1, 31);
297         tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
298         tcg_temp_free_i32(t0);
299     } else {
300         gen_helper_rem_i32(ret, arg1, arg2);
301     }
302 }
303 
304 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
305 {
306     if (TCG_TARGET_HAS_div_i32) {
307         tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
308     } else if (TCG_TARGET_HAS_div2_i32) {
309         TCGv_i32 t0 = tcg_temp_new_i32();
310         tcg_gen_movi_i32(t0, 0);
311         tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
312         tcg_temp_free_i32(t0);
313     } else {
314         gen_helper_divu_i32(ret, arg1, arg2);
315     }
316 }
317 
318 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
319 {
320     if (TCG_TARGET_HAS_rem_i32) {
321         tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
322     } else if (TCG_TARGET_HAS_div_i32) {
323         TCGv_i32 t0 = tcg_temp_new_i32();
324         tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
325         tcg_gen_mul_i32(t0, t0, arg2);
326         tcg_gen_sub_i32(ret, arg1, t0);
327         tcg_temp_free_i32(t0);
328     } else if (TCG_TARGET_HAS_div2_i32) {
329         TCGv_i32 t0 = tcg_temp_new_i32();
330         tcg_gen_movi_i32(t0, 0);
331         tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
332         tcg_temp_free_i32(t0);
333     } else {
334         gen_helper_remu_i32(ret, arg1, arg2);
335     }
336 }
337 
338 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
339 {
340     if (TCG_TARGET_HAS_andc_i32) {
341         tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
342     } else {
343         TCGv_i32 t0 = tcg_temp_new_i32();
344         tcg_gen_not_i32(t0, arg2);
345         tcg_gen_and_i32(ret, arg1, t0);
346         tcg_temp_free_i32(t0);
347     }
348 }
349 
350 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
351 {
352     if (TCG_TARGET_HAS_eqv_i32) {
353         tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
354     } else {
355         tcg_gen_xor_i32(ret, arg1, arg2);
356         tcg_gen_not_i32(ret, ret);
357     }
358 }
359 
360 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
361 {
362     if (TCG_TARGET_HAS_nand_i32) {
363         tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
364     } else {
365         tcg_gen_and_i32(ret, arg1, arg2);
366         tcg_gen_not_i32(ret, ret);
367     }
368 }
369 
370 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
371 {
372     if (TCG_TARGET_HAS_nor_i32) {
373         tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
374     } else {
375         tcg_gen_or_i32(ret, arg1, arg2);
376         tcg_gen_not_i32(ret, ret);
377     }
378 }
379 
380 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
381 {
382     if (TCG_TARGET_HAS_orc_i32) {
383         tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
384     } else {
385         TCGv_i32 t0 = tcg_temp_new_i32();
386         tcg_gen_not_i32(t0, arg2);
387         tcg_gen_or_i32(ret, arg1, t0);
388         tcg_temp_free_i32(t0);
389     }
390 }
391 
392 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
393 {
394     if (TCG_TARGET_HAS_clz_i32) {
395         tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
396     } else if (TCG_TARGET_HAS_clz_i64) {
397         TCGv_i64 t1 = tcg_temp_new_i64();
398         TCGv_i64 t2 = tcg_temp_new_i64();
399         tcg_gen_extu_i32_i64(t1, arg1);
400         tcg_gen_extu_i32_i64(t2, arg2);
401         tcg_gen_addi_i64(t2, t2, 32);
402         tcg_gen_clz_i64(t1, t1, t2);
403         tcg_gen_extrl_i64_i32(ret, t1);
404         tcg_temp_free_i64(t1);
405         tcg_temp_free_i64(t2);
406         tcg_gen_subi_i32(ret, ret, 32);
407     } else {
408         gen_helper_clz_i32(ret, arg1, arg2);
409     }
410 }
411 
412 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
413 {
414     tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2));
415 }
416 
417 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
418 {
419     if (TCG_TARGET_HAS_ctz_i32) {
420         tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
421     } else if (TCG_TARGET_HAS_ctz_i64) {
422         TCGv_i64 t1 = tcg_temp_new_i64();
423         TCGv_i64 t2 = tcg_temp_new_i64();
424         tcg_gen_extu_i32_i64(t1, arg1);
425         tcg_gen_extu_i32_i64(t2, arg2);
426         tcg_gen_ctz_i64(t1, t1, t2);
427         tcg_gen_extrl_i64_i32(ret, t1);
428         tcg_temp_free_i64(t1);
429         tcg_temp_free_i64(t2);
430     } else if (TCG_TARGET_HAS_ctpop_i32
431                || TCG_TARGET_HAS_ctpop_i64
432                || TCG_TARGET_HAS_clz_i32
433                || TCG_TARGET_HAS_clz_i64) {
434         TCGv_i32 z, t = tcg_temp_new_i32();
435 
436         if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
437             tcg_gen_subi_i32(t, arg1, 1);
438             tcg_gen_andc_i32(t, t, arg1);
439             tcg_gen_ctpop_i32(t, t);
440         } else {
441             /* Since all non-x86 hosts have clz(0) == 32, don't fight it.  */
442             tcg_gen_neg_i32(t, arg1);
443             tcg_gen_and_i32(t, t, arg1);
444             tcg_gen_clzi_i32(t, t, 32);
445             tcg_gen_xori_i32(t, t, 31);
446         }
447         z = tcg_constant_i32(0);
448         tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
449         tcg_temp_free_i32(t);
450     } else {
451         gen_helper_ctz_i32(ret, arg1, arg2);
452     }
453 }
454 
455 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
456 {
457     if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
458         /* This equivalence has the advantage of not requiring a fixup.  */
459         TCGv_i32 t = tcg_temp_new_i32();
460         tcg_gen_subi_i32(t, arg1, 1);
461         tcg_gen_andc_i32(t, t, arg1);
462         tcg_gen_ctpop_i32(ret, t);
463         tcg_temp_free_i32(t);
464     } else {
465         tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2));
466     }
467 }
468 
469 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
470 {
471     if (TCG_TARGET_HAS_clz_i32) {
472         TCGv_i32 t = tcg_temp_new_i32();
473         tcg_gen_sari_i32(t, arg, 31);
474         tcg_gen_xor_i32(t, t, arg);
475         tcg_gen_clzi_i32(t, t, 32);
476         tcg_gen_subi_i32(ret, t, 1);
477         tcg_temp_free_i32(t);
478     } else {
479         gen_helper_clrsb_i32(ret, arg);
480     }
481 }
482 
483 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
484 {
485     if (TCG_TARGET_HAS_ctpop_i32) {
486         tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
487     } else if (TCG_TARGET_HAS_ctpop_i64) {
488         TCGv_i64 t = tcg_temp_new_i64();
489         tcg_gen_extu_i32_i64(t, arg1);
490         tcg_gen_ctpop_i64(t, t);
491         tcg_gen_extrl_i64_i32(ret, t);
492         tcg_temp_free_i64(t);
493     } else {
494         gen_helper_ctpop_i32(ret, arg1);
495     }
496 }
497 
498 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
499 {
500     if (TCG_TARGET_HAS_rot_i32) {
501         tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
502     } else {
503         TCGv_i32 t0, t1;
504 
505         t0 = tcg_temp_new_i32();
506         t1 = tcg_temp_new_i32();
507         tcg_gen_shl_i32(t0, arg1, arg2);
508         tcg_gen_subfi_i32(t1, 32, arg2);
509         tcg_gen_shr_i32(t1, arg1, t1);
510         tcg_gen_or_i32(ret, t0, t1);
511         tcg_temp_free_i32(t0);
512         tcg_temp_free_i32(t1);
513     }
514 }
515 
516 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
517 {
518     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
519     /* some cases can be optimized here */
520     if (arg2 == 0) {
521         tcg_gen_mov_i32(ret, arg1);
522     } else if (TCG_TARGET_HAS_rot_i32) {
523         tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2));
524     } else {
525         TCGv_i32 t0, t1;
526         t0 = tcg_temp_new_i32();
527         t1 = tcg_temp_new_i32();
528         tcg_gen_shli_i32(t0, arg1, arg2);
529         tcg_gen_shri_i32(t1, arg1, 32 - arg2);
530         tcg_gen_or_i32(ret, t0, t1);
531         tcg_temp_free_i32(t0);
532         tcg_temp_free_i32(t1);
533     }
534 }
535 
536 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
537 {
538     if (TCG_TARGET_HAS_rot_i32) {
539         tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
540     } else {
541         TCGv_i32 t0, t1;
542 
543         t0 = tcg_temp_new_i32();
544         t1 = tcg_temp_new_i32();
545         tcg_gen_shr_i32(t0, arg1, arg2);
546         tcg_gen_subfi_i32(t1, 32, arg2);
547         tcg_gen_shl_i32(t1, arg1, t1);
548         tcg_gen_or_i32(ret, t0, t1);
549         tcg_temp_free_i32(t0);
550         tcg_temp_free_i32(t1);
551     }
552 }
553 
554 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
555 {
556     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
557     /* some cases can be optimized here */
558     if (arg2 == 0) {
559         tcg_gen_mov_i32(ret, arg1);
560     } else {
561         tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
562     }
563 }
564 
565 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
566                          unsigned int ofs, unsigned int len)
567 {
568     uint32_t mask;
569     TCGv_i32 t1;
570 
571     tcg_debug_assert(ofs < 32);
572     tcg_debug_assert(len > 0);
573     tcg_debug_assert(len <= 32);
574     tcg_debug_assert(ofs + len <= 32);
575 
576     if (len == 32) {
577         tcg_gen_mov_i32(ret, arg2);
578         return;
579     }
580     if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
581         tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
582         return;
583     }
584 
585     t1 = tcg_temp_new_i32();
586 
587     if (TCG_TARGET_HAS_extract2_i32) {
588         if (ofs + len == 32) {
589             tcg_gen_shli_i32(t1, arg1, len);
590             tcg_gen_extract2_i32(ret, t1, arg2, len);
591             goto done;
592         }
593         if (ofs == 0) {
594             tcg_gen_extract2_i32(ret, arg1, arg2, len);
595             tcg_gen_rotli_i32(ret, ret, len);
596             goto done;
597         }
598     }
599 
600     mask = (1u << len) - 1;
601     if (ofs + len < 32) {
602         tcg_gen_andi_i32(t1, arg2, mask);
603         tcg_gen_shli_i32(t1, t1, ofs);
604     } else {
605         tcg_gen_shli_i32(t1, arg2, ofs);
606     }
607     tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
608     tcg_gen_or_i32(ret, ret, t1);
609  done:
610     tcg_temp_free_i32(t1);
611 }
612 
613 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
614                            unsigned int ofs, unsigned int len)
615 {
616     tcg_debug_assert(ofs < 32);
617     tcg_debug_assert(len > 0);
618     tcg_debug_assert(len <= 32);
619     tcg_debug_assert(ofs + len <= 32);
620 
621     if (ofs + len == 32) {
622         tcg_gen_shli_i32(ret, arg, ofs);
623     } else if (ofs == 0) {
624         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
625     } else if (TCG_TARGET_HAS_deposit_i32
626                && TCG_TARGET_deposit_i32_valid(ofs, len)) {
627         TCGv_i32 zero = tcg_constant_i32(0);
628         tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
629     } else {
630         /* To help two-operand hosts we prefer to zero-extend first,
631            which allows ARG to stay live.  */
632         switch (len) {
633         case 16:
634             if (TCG_TARGET_HAS_ext16u_i32) {
635                 tcg_gen_ext16u_i32(ret, arg);
636                 tcg_gen_shli_i32(ret, ret, ofs);
637                 return;
638             }
639             break;
640         case 8:
641             if (TCG_TARGET_HAS_ext8u_i32) {
642                 tcg_gen_ext8u_i32(ret, arg);
643                 tcg_gen_shli_i32(ret, ret, ofs);
644                 return;
645             }
646             break;
647         }
648         /* Otherwise prefer zero-extension over AND for code size.  */
649         switch (ofs + len) {
650         case 16:
651             if (TCG_TARGET_HAS_ext16u_i32) {
652                 tcg_gen_shli_i32(ret, arg, ofs);
653                 tcg_gen_ext16u_i32(ret, ret);
654                 return;
655             }
656             break;
657         case 8:
658             if (TCG_TARGET_HAS_ext8u_i32) {
659                 tcg_gen_shli_i32(ret, arg, ofs);
660                 tcg_gen_ext8u_i32(ret, ret);
661                 return;
662             }
663             break;
664         }
665         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
666         tcg_gen_shli_i32(ret, ret, ofs);
667     }
668 }
669 
670 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
671                          unsigned int ofs, unsigned int len)
672 {
673     tcg_debug_assert(ofs < 32);
674     tcg_debug_assert(len > 0);
675     tcg_debug_assert(len <= 32);
676     tcg_debug_assert(ofs + len <= 32);
677 
678     /* Canonicalize certain special cases, even if extract is supported.  */
679     if (ofs + len == 32) {
680         tcg_gen_shri_i32(ret, arg, 32 - len);
681         return;
682     }
683     if (ofs == 0) {
684         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
685         return;
686     }
687 
688     if (TCG_TARGET_HAS_extract_i32
689         && TCG_TARGET_extract_i32_valid(ofs, len)) {
690         tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
691         return;
692     }
693 
694     /* Assume that zero-extension, if available, is cheaper than a shift.  */
695     switch (ofs + len) {
696     case 16:
697         if (TCG_TARGET_HAS_ext16u_i32) {
698             tcg_gen_ext16u_i32(ret, arg);
699             tcg_gen_shri_i32(ret, ret, ofs);
700             return;
701         }
702         break;
703     case 8:
704         if (TCG_TARGET_HAS_ext8u_i32) {
705             tcg_gen_ext8u_i32(ret, arg);
706             tcg_gen_shri_i32(ret, ret, ofs);
707             return;
708         }
709         break;
710     }
711 
712     /* ??? Ideally we'd know what values are available for immediate AND.
713        Assume that 8 bits are available, plus the special case of 16,
714        so that we get ext8u, ext16u.  */
715     switch (len) {
716     case 1 ... 8: case 16:
717         tcg_gen_shri_i32(ret, arg, ofs);
718         tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
719         break;
720     default:
721         tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
722         tcg_gen_shri_i32(ret, ret, 32 - len);
723         break;
724     }
725 }
726 
727 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
728                           unsigned int ofs, unsigned int len)
729 {
730     tcg_debug_assert(ofs < 32);
731     tcg_debug_assert(len > 0);
732     tcg_debug_assert(len <= 32);
733     tcg_debug_assert(ofs + len <= 32);
734 
735     /* Canonicalize certain special cases, even if extract is supported.  */
736     if (ofs + len == 32) {
737         tcg_gen_sari_i32(ret, arg, 32 - len);
738         return;
739     }
740     if (ofs == 0) {
741         switch (len) {
742         case 16:
743             tcg_gen_ext16s_i32(ret, arg);
744             return;
745         case 8:
746             tcg_gen_ext8s_i32(ret, arg);
747             return;
748         }
749     }
750 
751     if (TCG_TARGET_HAS_sextract_i32
752         && TCG_TARGET_extract_i32_valid(ofs, len)) {
753         tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
754         return;
755     }
756 
757     /* Assume that sign-extension, if available, is cheaper than a shift.  */
758     switch (ofs + len) {
759     case 16:
760         if (TCG_TARGET_HAS_ext16s_i32) {
761             tcg_gen_ext16s_i32(ret, arg);
762             tcg_gen_sari_i32(ret, ret, ofs);
763             return;
764         }
765         break;
766     case 8:
767         if (TCG_TARGET_HAS_ext8s_i32) {
768             tcg_gen_ext8s_i32(ret, arg);
769             tcg_gen_sari_i32(ret, ret, ofs);
770             return;
771         }
772         break;
773     }
774     switch (len) {
775     case 16:
776         if (TCG_TARGET_HAS_ext16s_i32) {
777             tcg_gen_shri_i32(ret, arg, ofs);
778             tcg_gen_ext16s_i32(ret, ret);
779             return;
780         }
781         break;
782     case 8:
783         if (TCG_TARGET_HAS_ext8s_i32) {
784             tcg_gen_shri_i32(ret, arg, ofs);
785             tcg_gen_ext8s_i32(ret, ret);
786             return;
787         }
788         break;
789     }
790 
791     tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
792     tcg_gen_sari_i32(ret, ret, 32 - len);
793 }
794 
795 /*
796  * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
797  * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
798  */
799 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah,
800                           unsigned int ofs)
801 {
802     tcg_debug_assert(ofs <= 32);
803     if (ofs == 0) {
804         tcg_gen_mov_i32(ret, al);
805     } else if (ofs == 32) {
806         tcg_gen_mov_i32(ret, ah);
807     } else if (al == ah) {
808         tcg_gen_rotri_i32(ret, al, ofs);
809     } else if (TCG_TARGET_HAS_extract2_i32) {
810         tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs);
811     } else {
812         TCGv_i32 t0 = tcg_temp_new_i32();
813         tcg_gen_shri_i32(t0, al, ofs);
814         tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs);
815         tcg_temp_free_i32(t0);
816     }
817 }
818 
819 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
820                          TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
821 {
822     if (cond == TCG_COND_ALWAYS) {
823         tcg_gen_mov_i32(ret, v1);
824     } else if (cond == TCG_COND_NEVER) {
825         tcg_gen_mov_i32(ret, v2);
826     } else if (TCG_TARGET_HAS_movcond_i32) {
827         tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
828     } else {
829         TCGv_i32 t0 = tcg_temp_new_i32();
830         TCGv_i32 t1 = tcg_temp_new_i32();
831         tcg_gen_setcond_i32(cond, t0, c1, c2);
832         tcg_gen_neg_i32(t0, t0);
833         tcg_gen_and_i32(t1, v1, t0);
834         tcg_gen_andc_i32(ret, v2, t0);
835         tcg_gen_or_i32(ret, ret, t1);
836         tcg_temp_free_i32(t0);
837         tcg_temp_free_i32(t1);
838     }
839 }
840 
841 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
842                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
843 {
844     if (TCG_TARGET_HAS_add2_i32) {
845         tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
846     } else {
847         TCGv_i64 t0 = tcg_temp_new_i64();
848         TCGv_i64 t1 = tcg_temp_new_i64();
849         tcg_gen_concat_i32_i64(t0, al, ah);
850         tcg_gen_concat_i32_i64(t1, bl, bh);
851         tcg_gen_add_i64(t0, t0, t1);
852         tcg_gen_extr_i64_i32(rl, rh, t0);
853         tcg_temp_free_i64(t0);
854         tcg_temp_free_i64(t1);
855     }
856 }
857 
858 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
859                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
860 {
861     if (TCG_TARGET_HAS_sub2_i32) {
862         tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
863     } else {
864         TCGv_i64 t0 = tcg_temp_new_i64();
865         TCGv_i64 t1 = tcg_temp_new_i64();
866         tcg_gen_concat_i32_i64(t0, al, ah);
867         tcg_gen_concat_i32_i64(t1, bl, bh);
868         tcg_gen_sub_i64(t0, t0, t1);
869         tcg_gen_extr_i64_i32(rl, rh, t0);
870         tcg_temp_free_i64(t0);
871         tcg_temp_free_i64(t1);
872     }
873 }
874 
875 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
876 {
877     if (TCG_TARGET_HAS_mulu2_i32) {
878         tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
879     } else if (TCG_TARGET_HAS_muluh_i32) {
880         TCGv_i32 t = tcg_temp_new_i32();
881         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
882         tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
883         tcg_gen_mov_i32(rl, t);
884         tcg_temp_free_i32(t);
885     } else {
886         TCGv_i64 t0 = tcg_temp_new_i64();
887         TCGv_i64 t1 = tcg_temp_new_i64();
888         tcg_gen_extu_i32_i64(t0, arg1);
889         tcg_gen_extu_i32_i64(t1, arg2);
890         tcg_gen_mul_i64(t0, t0, t1);
891         tcg_gen_extr_i64_i32(rl, rh, t0);
892         tcg_temp_free_i64(t0);
893         tcg_temp_free_i64(t1);
894     }
895 }
896 
897 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
898 {
899     if (TCG_TARGET_HAS_muls2_i32) {
900         tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
901     } else if (TCG_TARGET_HAS_mulsh_i32) {
902         TCGv_i32 t = tcg_temp_new_i32();
903         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
904         tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
905         tcg_gen_mov_i32(rl, t);
906         tcg_temp_free_i32(t);
907     } else if (TCG_TARGET_REG_BITS == 32) {
908         TCGv_i32 t0 = tcg_temp_new_i32();
909         TCGv_i32 t1 = tcg_temp_new_i32();
910         TCGv_i32 t2 = tcg_temp_new_i32();
911         TCGv_i32 t3 = tcg_temp_new_i32();
912         tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
913         /* Adjust for negative inputs.  */
914         tcg_gen_sari_i32(t2, arg1, 31);
915         tcg_gen_sari_i32(t3, arg2, 31);
916         tcg_gen_and_i32(t2, t2, arg2);
917         tcg_gen_and_i32(t3, t3, arg1);
918         tcg_gen_sub_i32(rh, t1, t2);
919         tcg_gen_sub_i32(rh, rh, t3);
920         tcg_gen_mov_i32(rl, t0);
921         tcg_temp_free_i32(t0);
922         tcg_temp_free_i32(t1);
923         tcg_temp_free_i32(t2);
924         tcg_temp_free_i32(t3);
925     } else {
926         TCGv_i64 t0 = tcg_temp_new_i64();
927         TCGv_i64 t1 = tcg_temp_new_i64();
928         tcg_gen_ext_i32_i64(t0, arg1);
929         tcg_gen_ext_i32_i64(t1, arg2);
930         tcg_gen_mul_i64(t0, t0, t1);
931         tcg_gen_extr_i64_i32(rl, rh, t0);
932         tcg_temp_free_i64(t0);
933         tcg_temp_free_i64(t1);
934     }
935 }
936 
937 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
938 {
939     if (TCG_TARGET_REG_BITS == 32) {
940         TCGv_i32 t0 = tcg_temp_new_i32();
941         TCGv_i32 t1 = tcg_temp_new_i32();
942         TCGv_i32 t2 = tcg_temp_new_i32();
943         tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
944         /* Adjust for negative input for the signed arg1.  */
945         tcg_gen_sari_i32(t2, arg1, 31);
946         tcg_gen_and_i32(t2, t2, arg2);
947         tcg_gen_sub_i32(rh, t1, t2);
948         tcg_gen_mov_i32(rl, t0);
949         tcg_temp_free_i32(t0);
950         tcg_temp_free_i32(t1);
951         tcg_temp_free_i32(t2);
952     } else {
953         TCGv_i64 t0 = tcg_temp_new_i64();
954         TCGv_i64 t1 = tcg_temp_new_i64();
955         tcg_gen_ext_i32_i64(t0, arg1);
956         tcg_gen_extu_i32_i64(t1, arg2);
957         tcg_gen_mul_i64(t0, t0, t1);
958         tcg_gen_extr_i64_i32(rl, rh, t0);
959         tcg_temp_free_i64(t0);
960         tcg_temp_free_i64(t1);
961     }
962 }
963 
964 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
965 {
966     if (TCG_TARGET_HAS_ext8s_i32) {
967         tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
968     } else {
969         tcg_gen_shli_i32(ret, arg, 24);
970         tcg_gen_sari_i32(ret, ret, 24);
971     }
972 }
973 
974 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
975 {
976     if (TCG_TARGET_HAS_ext16s_i32) {
977         tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
978     } else {
979         tcg_gen_shli_i32(ret, arg, 16);
980         tcg_gen_sari_i32(ret, ret, 16);
981     }
982 }
983 
984 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
985 {
986     if (TCG_TARGET_HAS_ext8u_i32) {
987         tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
988     } else {
989         tcg_gen_andi_i32(ret, arg, 0xffu);
990     }
991 }
992 
993 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
994 {
995     if (TCG_TARGET_HAS_ext16u_i32) {
996         tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
997     } else {
998         tcg_gen_andi_i32(ret, arg, 0xffffu);
999     }
1000 }
1001 
1002 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags)
1003 {
1004     /* Only one extension flag may be present. */
1005     tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1006 
1007     if (TCG_TARGET_HAS_bswap16_i32) {
1008         tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags);
1009     } else {
1010         TCGv_i32 t0 = tcg_temp_new_i32();
1011         TCGv_i32 t1 = tcg_temp_new_i32();
1012 
1013         tcg_gen_shri_i32(t0, arg, 8);
1014         if (!(flags & TCG_BSWAP_IZ)) {
1015             tcg_gen_ext8u_i32(t0, t0);
1016         }
1017 
1018         if (flags & TCG_BSWAP_OS) {
1019             tcg_gen_shli_i32(t1, arg, 24);
1020             tcg_gen_sari_i32(t1, t1, 16);
1021         } else if (flags & TCG_BSWAP_OZ) {
1022             tcg_gen_ext8u_i32(t1, arg);
1023             tcg_gen_shli_i32(t1, t1, 8);
1024         } else {
1025             tcg_gen_shli_i32(t1, arg, 8);
1026         }
1027 
1028         tcg_gen_or_i32(ret, t0, t1);
1029         tcg_temp_free_i32(t0);
1030         tcg_temp_free_i32(t1);
1031     }
1032 }
1033 
1034 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1035 {
1036     if (TCG_TARGET_HAS_bswap32_i32) {
1037         tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0);
1038     } else {
1039         TCGv_i32 t0 = tcg_temp_new_i32();
1040         TCGv_i32 t1 = tcg_temp_new_i32();
1041         TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff);
1042 
1043                                         /* arg = abcd */
1044         tcg_gen_shri_i32(t0, arg, 8);   /*  t0 = .abc */
1045         tcg_gen_and_i32(t1, arg, t2);   /*  t1 = .b.d */
1046         tcg_gen_and_i32(t0, t0, t2);    /*  t0 = .a.c */
1047         tcg_gen_shli_i32(t1, t1, 8);    /*  t1 = b.d. */
1048         tcg_gen_or_i32(ret, t0, t1);    /* ret = badc */
1049 
1050         tcg_gen_shri_i32(t0, ret, 16);  /*  t0 = ..ba */
1051         tcg_gen_shli_i32(t1, ret, 16);  /*  t1 = dc.. */
1052         tcg_gen_or_i32(ret, t0, t1);    /* ret = dcba */
1053 
1054         tcg_temp_free_i32(t0);
1055         tcg_temp_free_i32(t1);
1056     }
1057 }
1058 
1059 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1060 {
1061     tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1062 }
1063 
1064 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1065 {
1066     tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1067 }
1068 
1069 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1070 {
1071     tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1072 }
1073 
1074 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1075 {
1076     tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1077 }
1078 
1079 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a)
1080 {
1081     TCGv_i32 t = tcg_temp_new_i32();
1082 
1083     tcg_gen_sari_i32(t, a, 31);
1084     tcg_gen_xor_i32(ret, a, t);
1085     tcg_gen_sub_i32(ret, ret, t);
1086     tcg_temp_free_i32(t);
1087 }
1088 
1089 /* 64-bit ops */
1090 
1091 #if TCG_TARGET_REG_BITS == 32
1092 /* These are all inline for TCG_TARGET_REG_BITS == 64.  */
1093 
1094 void tcg_gen_discard_i64(TCGv_i64 arg)
1095 {
1096     tcg_gen_discard_i32(TCGV_LOW(arg));
1097     tcg_gen_discard_i32(TCGV_HIGH(arg));
1098 }
1099 
1100 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1101 {
1102     TCGTemp *ts = tcgv_i64_temp(arg);
1103 
1104     /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */
1105     if (ts->kind == TEMP_CONST) {
1106         tcg_gen_movi_i64(ret, ts->val);
1107     } else {
1108         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1109         tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1110     }
1111 }
1112 
1113 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1114 {
1115     tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1116     tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1117 }
1118 
1119 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1120 {
1121     tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1122     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1123 }
1124 
1125 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1126 {
1127     tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
1128     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1129 }
1130 
1131 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1132 {
1133     tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1134     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1135 }
1136 
1137 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1138 {
1139     tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1140     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1141 }
1142 
1143 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1144 {
1145     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1146     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1147 }
1148 
1149 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1150 {
1151     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1152     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1153 }
1154 
1155 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1156 {
1157     /* Since arg2 and ret have different types,
1158        they cannot be the same temporary */
1159 #ifdef HOST_WORDS_BIGENDIAN
1160     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1161     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1162 #else
1163     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1164     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1165 #endif
1166 }
1167 
1168 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1169 {
1170 #ifdef HOST_WORDS_BIGENDIAN
1171     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1172     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1173 #else
1174     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1175     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1176 #endif
1177 }
1178 
1179 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1180 {
1181     tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1182     tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1183 }
1184 
1185 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1186 {
1187     tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1188     tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1189 }
1190 
1191 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1192 {
1193     tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1194     tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1195 }
1196 
1197 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1198 {
1199     gen_helper_shl_i64(ret, arg1, arg2);
1200 }
1201 
1202 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1203 {
1204     gen_helper_shr_i64(ret, arg1, arg2);
1205 }
1206 
1207 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1208 {
1209     gen_helper_sar_i64(ret, arg1, arg2);
1210 }
1211 
1212 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1213 {
1214     TCGv_i64 t0;
1215     TCGv_i32 t1;
1216 
1217     t0 = tcg_temp_new_i64();
1218     t1 = tcg_temp_new_i32();
1219 
1220     tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1221                       TCGV_LOW(arg1), TCGV_LOW(arg2));
1222 
1223     tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1224     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1225     tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1226     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1227 
1228     tcg_gen_mov_i64(ret, t0);
1229     tcg_temp_free_i64(t0);
1230     tcg_temp_free_i32(t1);
1231 }
1232 
1233 #else
1234 
1235 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1236 {
1237     tcg_gen_mov_i64(ret, tcg_constant_i64(arg));
1238 }
1239 
1240 #endif /* TCG_TARGET_REG_SIZE == 32 */
1241 
1242 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1243 {
1244     /* some cases can be optimized here */
1245     if (arg2 == 0) {
1246         tcg_gen_mov_i64(ret, arg1);
1247     } else if (TCG_TARGET_REG_BITS == 64) {
1248         tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2));
1249     } else {
1250         tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1251                          TCGV_LOW(arg1), TCGV_HIGH(arg1),
1252                          tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
1253     }
1254 }
1255 
1256 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1257 {
1258     if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1259         /* Don't recurse with tcg_gen_neg_i64.  */
1260         tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1261     } else if (TCG_TARGET_REG_BITS == 64) {
1262         tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2);
1263     } else {
1264         tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1265                          tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32),
1266                          TCGV_LOW(arg2), TCGV_HIGH(arg2));
1267     }
1268 }
1269 
1270 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1271 {
1272     /* some cases can be optimized here */
1273     if (arg2 == 0) {
1274         tcg_gen_mov_i64(ret, arg1);
1275     } else if (TCG_TARGET_REG_BITS == 64) {
1276         tcg_gen_sub_i64(ret, arg1, tcg_constant_i64(arg2));
1277     } else {
1278         tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1279                          TCGV_LOW(arg1), TCGV_HIGH(arg1),
1280                          tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
1281     }
1282 }
1283 
1284 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1285 {
1286     if (TCG_TARGET_REG_BITS == 32) {
1287         tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1288         tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1289         return;
1290     }
1291 
1292     /* Some cases can be optimized here.  */
1293     switch (arg2) {
1294     case 0:
1295         tcg_gen_movi_i64(ret, 0);
1296         return;
1297     case -1:
1298         tcg_gen_mov_i64(ret, arg1);
1299         return;
1300     case 0xff:
1301         /* Don't recurse with tcg_gen_ext8u_i64.  */
1302         if (TCG_TARGET_HAS_ext8u_i64) {
1303             tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1304             return;
1305         }
1306         break;
1307     case 0xffff:
1308         if (TCG_TARGET_HAS_ext16u_i64) {
1309             tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1310             return;
1311         }
1312         break;
1313     case 0xffffffffu:
1314         if (TCG_TARGET_HAS_ext32u_i64) {
1315             tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1316             return;
1317         }
1318         break;
1319     }
1320 
1321     tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2));
1322 }
1323 
1324 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1325 {
1326     if (TCG_TARGET_REG_BITS == 32) {
1327         tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1328         tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1329         return;
1330     }
1331     /* Some cases can be optimized here.  */
1332     if (arg2 == -1) {
1333         tcg_gen_movi_i64(ret, -1);
1334     } else if (arg2 == 0) {
1335         tcg_gen_mov_i64(ret, arg1);
1336     } else {
1337         tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2));
1338     }
1339 }
1340 
1341 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1342 {
1343     if (TCG_TARGET_REG_BITS == 32) {
1344         tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1345         tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1346         return;
1347     }
1348     /* Some cases can be optimized here.  */
1349     if (arg2 == 0) {
1350         tcg_gen_mov_i64(ret, arg1);
1351     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1352         /* Don't recurse with tcg_gen_not_i64.  */
1353         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1354     } else {
1355         tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2));
1356     }
1357 }
1358 
1359 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1360                                       unsigned c, bool right, bool arith)
1361 {
1362     tcg_debug_assert(c < 64);
1363     if (c == 0) {
1364         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1365         tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1366     } else if (c >= 32) {
1367         c -= 32;
1368         if (right) {
1369             if (arith) {
1370                 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1371                 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1372             } else {
1373                 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1374                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1375             }
1376         } else {
1377             tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1378             tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1379         }
1380     } else if (right) {
1381         if (TCG_TARGET_HAS_extract2_i32) {
1382             tcg_gen_extract2_i32(TCGV_LOW(ret),
1383                                  TCGV_LOW(arg1), TCGV_HIGH(arg1), c);
1384         } else {
1385             tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1386             tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret),
1387                                 TCGV_HIGH(arg1), 32 - c, c);
1388         }
1389         if (arith) {
1390             tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1391         } else {
1392             tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1393         }
1394     } else {
1395         if (TCG_TARGET_HAS_extract2_i32) {
1396             tcg_gen_extract2_i32(TCGV_HIGH(ret),
1397                                  TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c);
1398         } else {
1399             TCGv_i32 t0 = tcg_temp_new_i32();
1400             tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1401             tcg_gen_deposit_i32(TCGV_HIGH(ret), t0,
1402                                 TCGV_HIGH(arg1), c, 32 - c);
1403             tcg_temp_free_i32(t0);
1404         }
1405         tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1406     }
1407 }
1408 
1409 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1410 {
1411     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1412     if (TCG_TARGET_REG_BITS == 32) {
1413         tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1414     } else if (arg2 == 0) {
1415         tcg_gen_mov_i64(ret, arg1);
1416     } else {
1417         tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2));
1418     }
1419 }
1420 
1421 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1422 {
1423     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1424     if (TCG_TARGET_REG_BITS == 32) {
1425         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1426     } else if (arg2 == 0) {
1427         tcg_gen_mov_i64(ret, arg1);
1428     } else {
1429         tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2));
1430     }
1431 }
1432 
1433 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1434 {
1435     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1436     if (TCG_TARGET_REG_BITS == 32) {
1437         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1438     } else if (arg2 == 0) {
1439         tcg_gen_mov_i64(ret, arg1);
1440     } else {
1441         tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2));
1442     }
1443 }
1444 
1445 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1446 {
1447     if (cond == TCG_COND_ALWAYS) {
1448         tcg_gen_br(l);
1449     } else if (cond != TCG_COND_NEVER) {
1450         l->refs++;
1451         if (TCG_TARGET_REG_BITS == 32) {
1452             tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1453                               TCGV_HIGH(arg1), TCGV_LOW(arg2),
1454                               TCGV_HIGH(arg2), cond, label_arg(l));
1455         } else {
1456             tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1457                               label_arg(l));
1458         }
1459     }
1460 }
1461 
1462 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1463 {
1464     if (TCG_TARGET_REG_BITS == 64) {
1465         tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l);
1466     } else if (cond == TCG_COND_ALWAYS) {
1467         tcg_gen_br(l);
1468     } else if (cond != TCG_COND_NEVER) {
1469         l->refs++;
1470         tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
1471                           TCGV_LOW(arg1), TCGV_HIGH(arg1),
1472                           tcg_constant_i32(arg2),
1473                           tcg_constant_i32(arg2 >> 32),
1474                           cond, label_arg(l));
1475     }
1476 }
1477 
1478 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1479                          TCGv_i64 arg1, TCGv_i64 arg2)
1480 {
1481     if (cond == TCG_COND_ALWAYS) {
1482         tcg_gen_movi_i64(ret, 1);
1483     } else if (cond == TCG_COND_NEVER) {
1484         tcg_gen_movi_i64(ret, 0);
1485     } else {
1486         if (TCG_TARGET_REG_BITS == 32) {
1487             tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1488                              TCGV_LOW(arg1), TCGV_HIGH(arg1),
1489                              TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1490             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1491         } else {
1492             tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1493         }
1494     }
1495 }
1496 
1497 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1498                           TCGv_i64 arg1, int64_t arg2)
1499 {
1500     if (TCG_TARGET_REG_BITS == 64) {
1501         tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2));
1502     } else if (cond == TCG_COND_ALWAYS) {
1503         tcg_gen_movi_i64(ret, 1);
1504     } else if (cond == TCG_COND_NEVER) {
1505         tcg_gen_movi_i64(ret, 0);
1506     } else {
1507         tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1508                          TCGV_LOW(arg1), TCGV_HIGH(arg1),
1509                          tcg_constant_i32(arg2),
1510                          tcg_constant_i32(arg2 >> 32), cond);
1511         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1512     }
1513 }
1514 
1515 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1516 {
1517     if (arg2 == 0) {
1518         tcg_gen_movi_i64(ret, 0);
1519     } else if (is_power_of_2(arg2)) {
1520         tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1521     } else {
1522         TCGv_i64 t0 = tcg_const_i64(arg2);
1523         tcg_gen_mul_i64(ret, arg1, t0);
1524         tcg_temp_free_i64(t0);
1525     }
1526 }
1527 
1528 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1529 {
1530     if (TCG_TARGET_HAS_div_i64) {
1531         tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1532     } else if (TCG_TARGET_HAS_div2_i64) {
1533         TCGv_i64 t0 = tcg_temp_new_i64();
1534         tcg_gen_sari_i64(t0, arg1, 63);
1535         tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1536         tcg_temp_free_i64(t0);
1537     } else {
1538         gen_helper_div_i64(ret, arg1, arg2);
1539     }
1540 }
1541 
1542 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1543 {
1544     if (TCG_TARGET_HAS_rem_i64) {
1545         tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1546     } else if (TCG_TARGET_HAS_div_i64) {
1547         TCGv_i64 t0 = tcg_temp_new_i64();
1548         tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1549         tcg_gen_mul_i64(t0, t0, arg2);
1550         tcg_gen_sub_i64(ret, arg1, t0);
1551         tcg_temp_free_i64(t0);
1552     } else if (TCG_TARGET_HAS_div2_i64) {
1553         TCGv_i64 t0 = tcg_temp_new_i64();
1554         tcg_gen_sari_i64(t0, arg1, 63);
1555         tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1556         tcg_temp_free_i64(t0);
1557     } else {
1558         gen_helper_rem_i64(ret, arg1, arg2);
1559     }
1560 }
1561 
1562 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1563 {
1564     if (TCG_TARGET_HAS_div_i64) {
1565         tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1566     } else if (TCG_TARGET_HAS_div2_i64) {
1567         TCGv_i64 t0 = tcg_temp_new_i64();
1568         tcg_gen_movi_i64(t0, 0);
1569         tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1570         tcg_temp_free_i64(t0);
1571     } else {
1572         gen_helper_divu_i64(ret, arg1, arg2);
1573     }
1574 }
1575 
1576 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1577 {
1578     if (TCG_TARGET_HAS_rem_i64) {
1579         tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1580     } else if (TCG_TARGET_HAS_div_i64) {
1581         TCGv_i64 t0 = tcg_temp_new_i64();
1582         tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1583         tcg_gen_mul_i64(t0, t0, arg2);
1584         tcg_gen_sub_i64(ret, arg1, t0);
1585         tcg_temp_free_i64(t0);
1586     } else if (TCG_TARGET_HAS_div2_i64) {
1587         TCGv_i64 t0 = tcg_temp_new_i64();
1588         tcg_gen_movi_i64(t0, 0);
1589         tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1590         tcg_temp_free_i64(t0);
1591     } else {
1592         gen_helper_remu_i64(ret, arg1, arg2);
1593     }
1594 }
1595 
1596 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1597 {
1598     if (TCG_TARGET_REG_BITS == 32) {
1599         tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1600         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1601     } else if (TCG_TARGET_HAS_ext8s_i64) {
1602         tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1603     } else {
1604         tcg_gen_shli_i64(ret, arg, 56);
1605         tcg_gen_sari_i64(ret, ret, 56);
1606     }
1607 }
1608 
1609 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1610 {
1611     if (TCG_TARGET_REG_BITS == 32) {
1612         tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1613         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1614     } else if (TCG_TARGET_HAS_ext16s_i64) {
1615         tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1616     } else {
1617         tcg_gen_shli_i64(ret, arg, 48);
1618         tcg_gen_sari_i64(ret, ret, 48);
1619     }
1620 }
1621 
1622 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1623 {
1624     if (TCG_TARGET_REG_BITS == 32) {
1625         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1626         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1627     } else if (TCG_TARGET_HAS_ext32s_i64) {
1628         tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1629     } else {
1630         tcg_gen_shli_i64(ret, arg, 32);
1631         tcg_gen_sari_i64(ret, ret, 32);
1632     }
1633 }
1634 
1635 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1636 {
1637     if (TCG_TARGET_REG_BITS == 32) {
1638         tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1639         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1640     } else if (TCG_TARGET_HAS_ext8u_i64) {
1641         tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1642     } else {
1643         tcg_gen_andi_i64(ret, arg, 0xffu);
1644     }
1645 }
1646 
1647 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1648 {
1649     if (TCG_TARGET_REG_BITS == 32) {
1650         tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1651         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1652     } else if (TCG_TARGET_HAS_ext16u_i64) {
1653         tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1654     } else {
1655         tcg_gen_andi_i64(ret, arg, 0xffffu);
1656     }
1657 }
1658 
1659 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1660 {
1661     if (TCG_TARGET_REG_BITS == 32) {
1662         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1663         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1664     } else if (TCG_TARGET_HAS_ext32u_i64) {
1665         tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1666     } else {
1667         tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1668     }
1669 }
1670 
1671 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
1672 {
1673     /* Only one extension flag may be present. */
1674     tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1675 
1676     if (TCG_TARGET_REG_BITS == 32) {
1677         tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags);
1678         if (flags & TCG_BSWAP_OS) {
1679             tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1680         } else {
1681             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1682         }
1683     } else if (TCG_TARGET_HAS_bswap16_i64) {
1684         tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags);
1685     } else {
1686         TCGv_i64 t0 = tcg_temp_new_i64();
1687         TCGv_i64 t1 = tcg_temp_new_i64();
1688 
1689         tcg_gen_shri_i64(t0, arg, 8);
1690         if (!(flags & TCG_BSWAP_IZ)) {
1691             tcg_gen_ext8u_i64(t0, t0);
1692         }
1693 
1694         if (flags & TCG_BSWAP_OS) {
1695             tcg_gen_shli_i64(t1, arg, 56);
1696             tcg_gen_sari_i64(t1, t1, 48);
1697         } else if (flags & TCG_BSWAP_OZ) {
1698             tcg_gen_ext8u_i64(t1, arg);
1699             tcg_gen_shli_i64(t1, t1, 8);
1700         } else {
1701             tcg_gen_shli_i64(t1, arg, 8);
1702         }
1703 
1704         tcg_gen_or_i64(ret, t0, t1);
1705         tcg_temp_free_i64(t0);
1706         tcg_temp_free_i64(t1);
1707     }
1708 }
1709 
1710 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
1711 {
1712     /* Only one extension flag may be present. */
1713     tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1714 
1715     if (TCG_TARGET_REG_BITS == 32) {
1716         tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1717         if (flags & TCG_BSWAP_OS) {
1718             tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1719         } else {
1720             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1721         }
1722     } else if (TCG_TARGET_HAS_bswap32_i64) {
1723         tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags);
1724     } else {
1725         TCGv_i64 t0 = tcg_temp_new_i64();
1726         TCGv_i64 t1 = tcg_temp_new_i64();
1727         TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff);
1728 
1729                                             /* arg = xxxxabcd */
1730         tcg_gen_shri_i64(t0, arg, 8);       /*  t0 = .xxxxabc */
1731         tcg_gen_and_i64(t1, arg, t2);       /*  t1 = .....b.d */
1732         tcg_gen_and_i64(t0, t0, t2);        /*  t0 = .....a.c */
1733         tcg_gen_shli_i64(t1, t1, 8);        /*  t1 = ....b.d. */
1734         tcg_gen_or_i64(ret, t0, t1);        /* ret = ....badc */
1735 
1736         tcg_gen_shli_i64(t1, ret, 48);      /*  t1 = dc...... */
1737         tcg_gen_shri_i64(t0, ret, 16);      /*  t0 = ......ba */
1738         if (flags & TCG_BSWAP_OS) {
1739             tcg_gen_sari_i64(t1, t1, 32);   /*  t1 = ssssdc.. */
1740         } else {
1741             tcg_gen_shri_i64(t1, t1, 32);   /*  t1 = ....dc.. */
1742         }
1743         tcg_gen_or_i64(ret, t0, t1);        /* ret = ssssdcba */
1744 
1745         tcg_temp_free_i64(t0);
1746         tcg_temp_free_i64(t1);
1747     }
1748 }
1749 
1750 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1751 {
1752     if (TCG_TARGET_REG_BITS == 32) {
1753         TCGv_i32 t0, t1;
1754         t0 = tcg_temp_new_i32();
1755         t1 = tcg_temp_new_i32();
1756 
1757         tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1758         tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1759         tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1760         tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1761         tcg_temp_free_i32(t0);
1762         tcg_temp_free_i32(t1);
1763     } else if (TCG_TARGET_HAS_bswap64_i64) {
1764         tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0);
1765     } else {
1766         TCGv_i64 t0 = tcg_temp_new_i64();
1767         TCGv_i64 t1 = tcg_temp_new_i64();
1768         TCGv_i64 t2 = tcg_temp_new_i64();
1769 
1770                                         /* arg = abcdefgh */
1771         tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
1772         tcg_gen_shri_i64(t0, arg, 8);   /*  t0 = .abcdefg */
1773         tcg_gen_and_i64(t1, arg, t2);   /*  t1 = .b.d.f.h */
1774         tcg_gen_and_i64(t0, t0, t2);    /*  t0 = .a.c.e.g */
1775         tcg_gen_shli_i64(t1, t1, 8);    /*  t1 = b.d.f.h. */
1776         tcg_gen_or_i64(ret, t0, t1);    /* ret = badcfehg */
1777 
1778         tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
1779         tcg_gen_shri_i64(t0, ret, 16);  /*  t0 = ..badcfe */
1780         tcg_gen_and_i64(t1, ret, t2);   /*  t1 = ..dc..hg */
1781         tcg_gen_and_i64(t0, t0, t2);    /*  t0 = ..ba..fe */
1782         tcg_gen_shli_i64(t1, t1, 16);   /*  t1 = dc..hg.. */
1783         tcg_gen_or_i64(ret, t0, t1);    /* ret = dcbahgfe */
1784 
1785         tcg_gen_shri_i64(t0, ret, 32);  /*  t0 = ....dcba */
1786         tcg_gen_shli_i64(t1, ret, 32);  /*  t1 = hgfe.... */
1787         tcg_gen_or_i64(ret, t0, t1);    /* ret = hgfedcba */
1788 
1789         tcg_temp_free_i64(t0);
1790         tcg_temp_free_i64(t1);
1791         tcg_temp_free_i64(t2);
1792     }
1793 }
1794 
1795 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1796 {
1797     if (TCG_TARGET_REG_BITS == 32) {
1798         tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1799         tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1800     } else if (TCG_TARGET_HAS_not_i64) {
1801         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1802     } else {
1803         tcg_gen_xori_i64(ret, arg, -1);
1804     }
1805 }
1806 
1807 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1808 {
1809     if (TCG_TARGET_REG_BITS == 32) {
1810         tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1811         tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1812     } else if (TCG_TARGET_HAS_andc_i64) {
1813         tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1814     } else {
1815         TCGv_i64 t0 = tcg_temp_new_i64();
1816         tcg_gen_not_i64(t0, arg2);
1817         tcg_gen_and_i64(ret, arg1, t0);
1818         tcg_temp_free_i64(t0);
1819     }
1820 }
1821 
1822 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1823 {
1824     if (TCG_TARGET_REG_BITS == 32) {
1825         tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1826         tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1827     } else if (TCG_TARGET_HAS_eqv_i64) {
1828         tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1829     } else {
1830         tcg_gen_xor_i64(ret, arg1, arg2);
1831         tcg_gen_not_i64(ret, ret);
1832     }
1833 }
1834 
1835 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1836 {
1837     if (TCG_TARGET_REG_BITS == 32) {
1838         tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1839         tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1840     } else if (TCG_TARGET_HAS_nand_i64) {
1841         tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1842     } else {
1843         tcg_gen_and_i64(ret, arg1, arg2);
1844         tcg_gen_not_i64(ret, ret);
1845     }
1846 }
1847 
1848 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1849 {
1850     if (TCG_TARGET_REG_BITS == 32) {
1851         tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1852         tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1853     } else if (TCG_TARGET_HAS_nor_i64) {
1854         tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1855     } else {
1856         tcg_gen_or_i64(ret, arg1, arg2);
1857         tcg_gen_not_i64(ret, ret);
1858     }
1859 }
1860 
1861 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1862 {
1863     if (TCG_TARGET_REG_BITS == 32) {
1864         tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1865         tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1866     } else if (TCG_TARGET_HAS_orc_i64) {
1867         tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1868     } else {
1869         TCGv_i64 t0 = tcg_temp_new_i64();
1870         tcg_gen_not_i64(t0, arg2);
1871         tcg_gen_or_i64(ret, arg1, t0);
1872         tcg_temp_free_i64(t0);
1873     }
1874 }
1875 
1876 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1877 {
1878     if (TCG_TARGET_HAS_clz_i64) {
1879         tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1880     } else {
1881         gen_helper_clz_i64(ret, arg1, arg2);
1882     }
1883 }
1884 
1885 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1886 {
1887     if (TCG_TARGET_REG_BITS == 32
1888         && TCG_TARGET_HAS_clz_i32
1889         && arg2 <= 0xffffffffu) {
1890         TCGv_i32 t = tcg_temp_new_i32();
1891         tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32);
1892         tcg_gen_addi_i32(t, t, 32);
1893         tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1894         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1895         tcg_temp_free_i32(t);
1896     } else {
1897         TCGv_i64 t0 = tcg_const_i64(arg2);
1898         tcg_gen_clz_i64(ret, arg1, t0);
1899         tcg_temp_free_i64(t0);
1900     }
1901 }
1902 
1903 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1904 {
1905     if (TCG_TARGET_HAS_ctz_i64) {
1906         tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
1907     } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1908         TCGv_i64 z, t = tcg_temp_new_i64();
1909 
1910         if (TCG_TARGET_HAS_ctpop_i64) {
1911             tcg_gen_subi_i64(t, arg1, 1);
1912             tcg_gen_andc_i64(t, t, arg1);
1913             tcg_gen_ctpop_i64(t, t);
1914         } else {
1915             /* Since all non-x86 hosts have clz(0) == 64, don't fight it.  */
1916             tcg_gen_neg_i64(t, arg1);
1917             tcg_gen_and_i64(t, t, arg1);
1918             tcg_gen_clzi_i64(t, t, 64);
1919             tcg_gen_xori_i64(t, t, 63);
1920         }
1921         z = tcg_constant_i64(0);
1922         tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1923         tcg_temp_free_i64(t);
1924         tcg_temp_free_i64(z);
1925     } else {
1926         gen_helper_ctz_i64(ret, arg1, arg2);
1927     }
1928 }
1929 
1930 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1931 {
1932     if (TCG_TARGET_REG_BITS == 32
1933         && TCG_TARGET_HAS_ctz_i32
1934         && arg2 <= 0xffffffffu) {
1935         TCGv_i32 t32 = tcg_temp_new_i32();
1936         tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32);
1937         tcg_gen_addi_i32(t32, t32, 32);
1938         tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1939         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1940         tcg_temp_free_i32(t32);
1941     } else if (!TCG_TARGET_HAS_ctz_i64
1942                && TCG_TARGET_HAS_ctpop_i64
1943                && arg2 == 64) {
1944         /* This equivalence has the advantage of not requiring a fixup.  */
1945         TCGv_i64 t = tcg_temp_new_i64();
1946         tcg_gen_subi_i64(t, arg1, 1);
1947         tcg_gen_andc_i64(t, t, arg1);
1948         tcg_gen_ctpop_i64(ret, t);
1949         tcg_temp_free_i64(t);
1950     } else {
1951         TCGv_i64 t0 = tcg_const_i64(arg2);
1952         tcg_gen_ctz_i64(ret, arg1, t0);
1953         tcg_temp_free_i64(t0);
1954     }
1955 }
1956 
1957 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1958 {
1959     if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1960         TCGv_i64 t = tcg_temp_new_i64();
1961         tcg_gen_sari_i64(t, arg, 63);
1962         tcg_gen_xor_i64(t, t, arg);
1963         tcg_gen_clzi_i64(t, t, 64);
1964         tcg_gen_subi_i64(ret, t, 1);
1965         tcg_temp_free_i64(t);
1966     } else {
1967         gen_helper_clrsb_i64(ret, arg);
1968     }
1969 }
1970 
1971 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
1972 {
1973     if (TCG_TARGET_HAS_ctpop_i64) {
1974         tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
1975     } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
1976         tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1977         tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1978         tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
1979         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1980     } else {
1981         gen_helper_ctpop_i64(ret, arg1);
1982     }
1983 }
1984 
1985 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1986 {
1987     if (TCG_TARGET_HAS_rot_i64) {
1988         tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1989     } else {
1990         TCGv_i64 t0, t1;
1991         t0 = tcg_temp_new_i64();
1992         t1 = tcg_temp_new_i64();
1993         tcg_gen_shl_i64(t0, arg1, arg2);
1994         tcg_gen_subfi_i64(t1, 64, arg2);
1995         tcg_gen_shr_i64(t1, arg1, t1);
1996         tcg_gen_or_i64(ret, t0, t1);
1997         tcg_temp_free_i64(t0);
1998         tcg_temp_free_i64(t1);
1999     }
2000 }
2001 
2002 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2003 {
2004     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2005     /* some cases can be optimized here */
2006     if (arg2 == 0) {
2007         tcg_gen_mov_i64(ret, arg1);
2008     } else if (TCG_TARGET_HAS_rot_i64) {
2009         tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2));
2010     } else {
2011         TCGv_i64 t0, t1;
2012         t0 = tcg_temp_new_i64();
2013         t1 = tcg_temp_new_i64();
2014         tcg_gen_shli_i64(t0, arg1, arg2);
2015         tcg_gen_shri_i64(t1, arg1, 64 - arg2);
2016         tcg_gen_or_i64(ret, t0, t1);
2017         tcg_temp_free_i64(t0);
2018         tcg_temp_free_i64(t1);
2019     }
2020 }
2021 
2022 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2023 {
2024     if (TCG_TARGET_HAS_rot_i64) {
2025         tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
2026     } else {
2027         TCGv_i64 t0, t1;
2028         t0 = tcg_temp_new_i64();
2029         t1 = tcg_temp_new_i64();
2030         tcg_gen_shr_i64(t0, arg1, arg2);
2031         tcg_gen_subfi_i64(t1, 64, arg2);
2032         tcg_gen_shl_i64(t1, arg1, t1);
2033         tcg_gen_or_i64(ret, t0, t1);
2034         tcg_temp_free_i64(t0);
2035         tcg_temp_free_i64(t1);
2036     }
2037 }
2038 
2039 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2040 {
2041     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2042     /* some cases can be optimized here */
2043     if (arg2 == 0) {
2044         tcg_gen_mov_i64(ret, arg1);
2045     } else {
2046         tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
2047     }
2048 }
2049 
2050 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
2051                          unsigned int ofs, unsigned int len)
2052 {
2053     uint64_t mask;
2054     TCGv_i64 t1;
2055 
2056     tcg_debug_assert(ofs < 64);
2057     tcg_debug_assert(len > 0);
2058     tcg_debug_assert(len <= 64);
2059     tcg_debug_assert(ofs + len <= 64);
2060 
2061     if (len == 64) {
2062         tcg_gen_mov_i64(ret, arg2);
2063         return;
2064     }
2065     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2066         tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
2067         return;
2068     }
2069 
2070     if (TCG_TARGET_REG_BITS == 32) {
2071         if (ofs >= 32) {
2072             tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
2073                                 TCGV_LOW(arg2), ofs - 32, len);
2074             tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2075             return;
2076         }
2077         if (ofs + len <= 32) {
2078             tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
2079                                 TCGV_LOW(arg2), ofs, len);
2080             tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2081             return;
2082         }
2083     }
2084 
2085     t1 = tcg_temp_new_i64();
2086 
2087     if (TCG_TARGET_HAS_extract2_i64) {
2088         if (ofs + len == 64) {
2089             tcg_gen_shli_i64(t1, arg1, len);
2090             tcg_gen_extract2_i64(ret, t1, arg2, len);
2091             goto done;
2092         }
2093         if (ofs == 0) {
2094             tcg_gen_extract2_i64(ret, arg1, arg2, len);
2095             tcg_gen_rotli_i64(ret, ret, len);
2096             goto done;
2097         }
2098     }
2099 
2100     mask = (1ull << len) - 1;
2101     if (ofs + len < 64) {
2102         tcg_gen_andi_i64(t1, arg2, mask);
2103         tcg_gen_shli_i64(t1, t1, ofs);
2104     } else {
2105         tcg_gen_shli_i64(t1, arg2, ofs);
2106     }
2107     tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2108     tcg_gen_or_i64(ret, ret, t1);
2109  done:
2110     tcg_temp_free_i64(t1);
2111 }
2112 
2113 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2114                            unsigned int ofs, unsigned int len)
2115 {
2116     tcg_debug_assert(ofs < 64);
2117     tcg_debug_assert(len > 0);
2118     tcg_debug_assert(len <= 64);
2119     tcg_debug_assert(ofs + len <= 64);
2120 
2121     if (ofs + len == 64) {
2122         tcg_gen_shli_i64(ret, arg, ofs);
2123     } else if (ofs == 0) {
2124         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2125     } else if (TCG_TARGET_HAS_deposit_i64
2126                && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2127         TCGv_i64 zero = tcg_constant_i64(0);
2128         tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2129     } else {
2130         if (TCG_TARGET_REG_BITS == 32) {
2131             if (ofs >= 32) {
2132                 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2133                                       ofs - 32, len);
2134                 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2135                 return;
2136             }
2137             if (ofs + len <= 32) {
2138                 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2139                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2140                 return;
2141             }
2142         }
2143         /* To help two-operand hosts we prefer to zero-extend first,
2144            which allows ARG to stay live.  */
2145         switch (len) {
2146         case 32:
2147             if (TCG_TARGET_HAS_ext32u_i64) {
2148                 tcg_gen_ext32u_i64(ret, arg);
2149                 tcg_gen_shli_i64(ret, ret, ofs);
2150                 return;
2151             }
2152             break;
2153         case 16:
2154             if (TCG_TARGET_HAS_ext16u_i64) {
2155                 tcg_gen_ext16u_i64(ret, arg);
2156                 tcg_gen_shli_i64(ret, ret, ofs);
2157                 return;
2158             }
2159             break;
2160         case 8:
2161             if (TCG_TARGET_HAS_ext8u_i64) {
2162                 tcg_gen_ext8u_i64(ret, arg);
2163                 tcg_gen_shli_i64(ret, ret, ofs);
2164                 return;
2165             }
2166             break;
2167         }
2168         /* Otherwise prefer zero-extension over AND for code size.  */
2169         switch (ofs + len) {
2170         case 32:
2171             if (TCG_TARGET_HAS_ext32u_i64) {
2172                 tcg_gen_shli_i64(ret, arg, ofs);
2173                 tcg_gen_ext32u_i64(ret, ret);
2174                 return;
2175             }
2176             break;
2177         case 16:
2178             if (TCG_TARGET_HAS_ext16u_i64) {
2179                 tcg_gen_shli_i64(ret, arg, ofs);
2180                 tcg_gen_ext16u_i64(ret, ret);
2181                 return;
2182             }
2183             break;
2184         case 8:
2185             if (TCG_TARGET_HAS_ext8u_i64) {
2186                 tcg_gen_shli_i64(ret, arg, ofs);
2187                 tcg_gen_ext8u_i64(ret, ret);
2188                 return;
2189             }
2190             break;
2191         }
2192         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2193         tcg_gen_shli_i64(ret, ret, ofs);
2194     }
2195 }
2196 
2197 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2198                          unsigned int ofs, unsigned int len)
2199 {
2200     tcg_debug_assert(ofs < 64);
2201     tcg_debug_assert(len > 0);
2202     tcg_debug_assert(len <= 64);
2203     tcg_debug_assert(ofs + len <= 64);
2204 
2205     /* Canonicalize certain special cases, even if extract is supported.  */
2206     if (ofs + len == 64) {
2207         tcg_gen_shri_i64(ret, arg, 64 - len);
2208         return;
2209     }
2210     if (ofs == 0) {
2211         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2212         return;
2213     }
2214 
2215     if (TCG_TARGET_REG_BITS == 32) {
2216         /* Look for a 32-bit extract within one of the two words.  */
2217         if (ofs >= 32) {
2218             tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2219             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2220             return;
2221         }
2222         if (ofs + len <= 32) {
2223             tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2224             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2225             return;
2226         }
2227         /* The field is split across two words.  One double-word
2228            shift is better than two double-word shifts.  */
2229         goto do_shift_and;
2230     }
2231 
2232     if (TCG_TARGET_HAS_extract_i64
2233         && TCG_TARGET_extract_i64_valid(ofs, len)) {
2234         tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2235         return;
2236     }
2237 
2238     /* Assume that zero-extension, if available, is cheaper than a shift.  */
2239     switch (ofs + len) {
2240     case 32:
2241         if (TCG_TARGET_HAS_ext32u_i64) {
2242             tcg_gen_ext32u_i64(ret, arg);
2243             tcg_gen_shri_i64(ret, ret, ofs);
2244             return;
2245         }
2246         break;
2247     case 16:
2248         if (TCG_TARGET_HAS_ext16u_i64) {
2249             tcg_gen_ext16u_i64(ret, arg);
2250             tcg_gen_shri_i64(ret, ret, ofs);
2251             return;
2252         }
2253         break;
2254     case 8:
2255         if (TCG_TARGET_HAS_ext8u_i64) {
2256             tcg_gen_ext8u_i64(ret, arg);
2257             tcg_gen_shri_i64(ret, ret, ofs);
2258             return;
2259         }
2260         break;
2261     }
2262 
2263     /* ??? Ideally we'd know what values are available for immediate AND.
2264        Assume that 8 bits are available, plus the special cases of 16 and 32,
2265        so that we get ext8u, ext16u, and ext32u.  */
2266     switch (len) {
2267     case 1 ... 8: case 16: case 32:
2268     do_shift_and:
2269         tcg_gen_shri_i64(ret, arg, ofs);
2270         tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2271         break;
2272     default:
2273         tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2274         tcg_gen_shri_i64(ret, ret, 64 - len);
2275         break;
2276     }
2277 }
2278 
2279 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2280                           unsigned int ofs, unsigned int len)
2281 {
2282     tcg_debug_assert(ofs < 64);
2283     tcg_debug_assert(len > 0);
2284     tcg_debug_assert(len <= 64);
2285     tcg_debug_assert(ofs + len <= 64);
2286 
2287     /* Canonicalize certain special cases, even if sextract is supported.  */
2288     if (ofs + len == 64) {
2289         tcg_gen_sari_i64(ret, arg, 64 - len);
2290         return;
2291     }
2292     if (ofs == 0) {
2293         switch (len) {
2294         case 32:
2295             tcg_gen_ext32s_i64(ret, arg);
2296             return;
2297         case 16:
2298             tcg_gen_ext16s_i64(ret, arg);
2299             return;
2300         case 8:
2301             tcg_gen_ext8s_i64(ret, arg);
2302             return;
2303         }
2304     }
2305 
2306     if (TCG_TARGET_REG_BITS == 32) {
2307         /* Look for a 32-bit extract within one of the two words.  */
2308         if (ofs >= 32) {
2309             tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2310         } else if (ofs + len <= 32) {
2311             tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2312         } else if (ofs == 0) {
2313             tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2314             tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2315             return;
2316         } else if (len > 32) {
2317             TCGv_i32 t = tcg_temp_new_i32();
2318             /* Extract the bits for the high word normally.  */
2319             tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2320             /* Shift the field down for the low part.  */
2321             tcg_gen_shri_i64(ret, arg, ofs);
2322             /* Overwrite the shift into the high part.  */
2323             tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2324             tcg_temp_free_i32(t);
2325             return;
2326         } else {
2327             /* Shift the field down for the low part, such that the
2328                field sits at the MSB.  */
2329             tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2330             /* Shift the field down from the MSB, sign extending.  */
2331             tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2332         }
2333         /* Sign-extend the field from 32 bits.  */
2334         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2335         return;
2336     }
2337 
2338     if (TCG_TARGET_HAS_sextract_i64
2339         && TCG_TARGET_extract_i64_valid(ofs, len)) {
2340         tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2341         return;
2342     }
2343 
2344     /* Assume that sign-extension, if available, is cheaper than a shift.  */
2345     switch (ofs + len) {
2346     case 32:
2347         if (TCG_TARGET_HAS_ext32s_i64) {
2348             tcg_gen_ext32s_i64(ret, arg);
2349             tcg_gen_sari_i64(ret, ret, ofs);
2350             return;
2351         }
2352         break;
2353     case 16:
2354         if (TCG_TARGET_HAS_ext16s_i64) {
2355             tcg_gen_ext16s_i64(ret, arg);
2356             tcg_gen_sari_i64(ret, ret, ofs);
2357             return;
2358         }
2359         break;
2360     case 8:
2361         if (TCG_TARGET_HAS_ext8s_i64) {
2362             tcg_gen_ext8s_i64(ret, arg);
2363             tcg_gen_sari_i64(ret, ret, ofs);
2364             return;
2365         }
2366         break;
2367     }
2368     switch (len) {
2369     case 32:
2370         if (TCG_TARGET_HAS_ext32s_i64) {
2371             tcg_gen_shri_i64(ret, arg, ofs);
2372             tcg_gen_ext32s_i64(ret, ret);
2373             return;
2374         }
2375         break;
2376     case 16:
2377         if (TCG_TARGET_HAS_ext16s_i64) {
2378             tcg_gen_shri_i64(ret, arg, ofs);
2379             tcg_gen_ext16s_i64(ret, ret);
2380             return;
2381         }
2382         break;
2383     case 8:
2384         if (TCG_TARGET_HAS_ext8s_i64) {
2385             tcg_gen_shri_i64(ret, arg, ofs);
2386             tcg_gen_ext8s_i64(ret, ret);
2387             return;
2388         }
2389         break;
2390     }
2391     tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2392     tcg_gen_sari_i64(ret, ret, 64 - len);
2393 }
2394 
2395 /*
2396  * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2397  * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2398  */
2399 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah,
2400                           unsigned int ofs)
2401 {
2402     tcg_debug_assert(ofs <= 64);
2403     if (ofs == 0) {
2404         tcg_gen_mov_i64(ret, al);
2405     } else if (ofs == 64) {
2406         tcg_gen_mov_i64(ret, ah);
2407     } else if (al == ah) {
2408         tcg_gen_rotri_i64(ret, al, ofs);
2409     } else if (TCG_TARGET_HAS_extract2_i64) {
2410         tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs);
2411     } else {
2412         TCGv_i64 t0 = tcg_temp_new_i64();
2413         tcg_gen_shri_i64(t0, al, ofs);
2414         tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs);
2415         tcg_temp_free_i64(t0);
2416     }
2417 }
2418 
2419 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2420                          TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2421 {
2422     if (cond == TCG_COND_ALWAYS) {
2423         tcg_gen_mov_i64(ret, v1);
2424     } else if (cond == TCG_COND_NEVER) {
2425         tcg_gen_mov_i64(ret, v2);
2426     } else if (TCG_TARGET_REG_BITS == 32) {
2427         TCGv_i32 t0 = tcg_temp_new_i32();
2428         TCGv_i32 t1 = tcg_temp_new_i32();
2429         tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2430                          TCGV_LOW(c1), TCGV_HIGH(c1),
2431                          TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2432 
2433         if (TCG_TARGET_HAS_movcond_i32) {
2434             tcg_gen_movi_i32(t1, 0);
2435             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2436                                 TCGV_LOW(v1), TCGV_LOW(v2));
2437             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2438                                 TCGV_HIGH(v1), TCGV_HIGH(v2));
2439         } else {
2440             tcg_gen_neg_i32(t0, t0);
2441 
2442             tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2443             tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2444             tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
2445 
2446             tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2447             tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2448             tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2449         }
2450         tcg_temp_free_i32(t0);
2451         tcg_temp_free_i32(t1);
2452     } else if (TCG_TARGET_HAS_movcond_i64) {
2453         tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2454     } else {
2455         TCGv_i64 t0 = tcg_temp_new_i64();
2456         TCGv_i64 t1 = tcg_temp_new_i64();
2457         tcg_gen_setcond_i64(cond, t0, c1, c2);
2458         tcg_gen_neg_i64(t0, t0);
2459         tcg_gen_and_i64(t1, v1, t0);
2460         tcg_gen_andc_i64(ret, v2, t0);
2461         tcg_gen_or_i64(ret, ret, t1);
2462         tcg_temp_free_i64(t0);
2463         tcg_temp_free_i64(t1);
2464     }
2465 }
2466 
2467 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2468                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2469 {
2470     if (TCG_TARGET_HAS_add2_i64) {
2471         tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
2472     } else {
2473         TCGv_i64 t0 = tcg_temp_new_i64();
2474         TCGv_i64 t1 = tcg_temp_new_i64();
2475         tcg_gen_add_i64(t0, al, bl);
2476         tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2477         tcg_gen_add_i64(rh, ah, bh);
2478         tcg_gen_add_i64(rh, rh, t1);
2479         tcg_gen_mov_i64(rl, t0);
2480         tcg_temp_free_i64(t0);
2481         tcg_temp_free_i64(t1);
2482     }
2483 }
2484 
2485 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2486                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2487 {
2488     if (TCG_TARGET_HAS_sub2_i64) {
2489         tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
2490     } else {
2491         TCGv_i64 t0 = tcg_temp_new_i64();
2492         TCGv_i64 t1 = tcg_temp_new_i64();
2493         tcg_gen_sub_i64(t0, al, bl);
2494         tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2495         tcg_gen_sub_i64(rh, ah, bh);
2496         tcg_gen_sub_i64(rh, rh, t1);
2497         tcg_gen_mov_i64(rl, t0);
2498         tcg_temp_free_i64(t0);
2499         tcg_temp_free_i64(t1);
2500     }
2501 }
2502 
2503 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2504 {
2505     if (TCG_TARGET_HAS_mulu2_i64) {
2506         tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
2507     } else if (TCG_TARGET_HAS_muluh_i64) {
2508         TCGv_i64 t = tcg_temp_new_i64();
2509         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2510         tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2511         tcg_gen_mov_i64(rl, t);
2512         tcg_temp_free_i64(t);
2513     } else {
2514         TCGv_i64 t0 = tcg_temp_new_i64();
2515         tcg_gen_mul_i64(t0, arg1, arg2);
2516         gen_helper_muluh_i64(rh, arg1, arg2);
2517         tcg_gen_mov_i64(rl, t0);
2518         tcg_temp_free_i64(t0);
2519     }
2520 }
2521 
2522 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2523 {
2524     if (TCG_TARGET_HAS_muls2_i64) {
2525         tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
2526     } else if (TCG_TARGET_HAS_mulsh_i64) {
2527         TCGv_i64 t = tcg_temp_new_i64();
2528         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2529         tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2530         tcg_gen_mov_i64(rl, t);
2531         tcg_temp_free_i64(t);
2532     } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2533         TCGv_i64 t0 = tcg_temp_new_i64();
2534         TCGv_i64 t1 = tcg_temp_new_i64();
2535         TCGv_i64 t2 = tcg_temp_new_i64();
2536         TCGv_i64 t3 = tcg_temp_new_i64();
2537         tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2538         /* Adjust for negative inputs.  */
2539         tcg_gen_sari_i64(t2, arg1, 63);
2540         tcg_gen_sari_i64(t3, arg2, 63);
2541         tcg_gen_and_i64(t2, t2, arg2);
2542         tcg_gen_and_i64(t3, t3, arg1);
2543         tcg_gen_sub_i64(rh, t1, t2);
2544         tcg_gen_sub_i64(rh, rh, t3);
2545         tcg_gen_mov_i64(rl, t0);
2546         tcg_temp_free_i64(t0);
2547         tcg_temp_free_i64(t1);
2548         tcg_temp_free_i64(t2);
2549         tcg_temp_free_i64(t3);
2550     } else {
2551         TCGv_i64 t0 = tcg_temp_new_i64();
2552         tcg_gen_mul_i64(t0, arg1, arg2);
2553         gen_helper_mulsh_i64(rh, arg1, arg2);
2554         tcg_gen_mov_i64(rl, t0);
2555         tcg_temp_free_i64(t0);
2556     }
2557 }
2558 
2559 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2560 {
2561     TCGv_i64 t0 = tcg_temp_new_i64();
2562     TCGv_i64 t1 = tcg_temp_new_i64();
2563     TCGv_i64 t2 = tcg_temp_new_i64();
2564     tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2565     /* Adjust for negative input for the signed arg1.  */
2566     tcg_gen_sari_i64(t2, arg1, 63);
2567     tcg_gen_and_i64(t2, t2, arg2);
2568     tcg_gen_sub_i64(rh, t1, t2);
2569     tcg_gen_mov_i64(rl, t0);
2570     tcg_temp_free_i64(t0);
2571     tcg_temp_free_i64(t1);
2572     tcg_temp_free_i64(t2);
2573 }
2574 
2575 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2576 {
2577     tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2578 }
2579 
2580 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2581 {
2582     tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2583 }
2584 
2585 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2586 {
2587     tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2588 }
2589 
2590 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2591 {
2592     tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2593 }
2594 
2595 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a)
2596 {
2597     TCGv_i64 t = tcg_temp_new_i64();
2598 
2599     tcg_gen_sari_i64(t, a, 63);
2600     tcg_gen_xor_i64(ret, a, t);
2601     tcg_gen_sub_i64(ret, ret, t);
2602     tcg_temp_free_i64(t);
2603 }
2604 
2605 /* Size changing operations.  */
2606 
2607 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2608 {
2609     if (TCG_TARGET_REG_BITS == 32) {
2610         tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2611     } else if (TCG_TARGET_HAS_extrl_i64_i32) {
2612         tcg_gen_op2(INDEX_op_extrl_i64_i32,
2613                     tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2614     } else {
2615         tcg_gen_mov_i32(ret, (TCGv_i32)arg);
2616     }
2617 }
2618 
2619 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2620 {
2621     if (TCG_TARGET_REG_BITS == 32) {
2622         tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2623     } else if (TCG_TARGET_HAS_extrh_i64_i32) {
2624         tcg_gen_op2(INDEX_op_extrh_i64_i32,
2625                     tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2626     } else {
2627         TCGv_i64 t = tcg_temp_new_i64();
2628         tcg_gen_shri_i64(t, arg, 32);
2629         tcg_gen_mov_i32(ret, (TCGv_i32)t);
2630         tcg_temp_free_i64(t);
2631     }
2632 }
2633 
2634 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2635 {
2636     if (TCG_TARGET_REG_BITS == 32) {
2637         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2638         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2639     } else {
2640         tcg_gen_op2(INDEX_op_extu_i32_i64,
2641                     tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2642     }
2643 }
2644 
2645 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2646 {
2647     if (TCG_TARGET_REG_BITS == 32) {
2648         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2649         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2650     } else {
2651         tcg_gen_op2(INDEX_op_ext_i32_i64,
2652                     tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2653     }
2654 }
2655 
2656 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2657 {
2658     TCGv_i64 tmp;
2659 
2660     if (TCG_TARGET_REG_BITS == 32) {
2661         tcg_gen_mov_i32(TCGV_LOW(dest), low);
2662         tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2663         return;
2664     }
2665 
2666     tmp = tcg_temp_new_i64();
2667     /* These extensions are only needed for type correctness.
2668        We may be able to do better given target specific information.  */
2669     tcg_gen_extu_i32_i64(tmp, high);
2670     tcg_gen_extu_i32_i64(dest, low);
2671     /* If deposit is available, use it.  Otherwise use the extra
2672        knowledge that we have of the zero-extensions above.  */
2673     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2674         tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2675     } else {
2676         tcg_gen_shli_i64(tmp, tmp, 32);
2677         tcg_gen_or_i64(dest, dest, tmp);
2678     }
2679     tcg_temp_free_i64(tmp);
2680 }
2681 
2682 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2683 {
2684     if (TCG_TARGET_REG_BITS == 32) {
2685         tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2686         tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2687     } else {
2688         tcg_gen_extrl_i64_i32(lo, arg);
2689         tcg_gen_extrh_i64_i32(hi, arg);
2690     }
2691 }
2692 
2693 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2694 {
2695     tcg_gen_ext32u_i64(lo, arg);
2696     tcg_gen_shri_i64(hi, arg, 32);
2697 }
2698 
2699 /* QEMU specific operations.  */
2700 
2701 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx)
2702 {
2703     /*
2704      * Let the jit code return the read-only version of the
2705      * TranslationBlock, so that we minimize the pc-relative
2706      * distance of the address of the exit_tb code to TB.
2707      * This will improve utilization of pc-relative address loads.
2708      *
2709      * TODO: Move this to translator_loop, so that all const
2710      * TranslationBlock pointers refer to read-only memory.
2711      * This requires coordination with targets that do not use
2712      * the translator_loop.
2713      */
2714     uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx;
2715 
2716     if (tb == NULL) {
2717         tcg_debug_assert(idx == 0);
2718     } else if (idx <= TB_EXIT_IDXMAX) {
2719 #ifdef CONFIG_DEBUG_TCG
2720         /* This is an exit following a goto_tb.  Verify that we have
2721            seen this numbered exit before, via tcg_gen_goto_tb.  */
2722         tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2723 #endif
2724     } else {
2725         /* This is an exit via the exitreq label.  */
2726         tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2727     }
2728 
2729     plugin_gen_disable_mem_helpers();
2730     tcg_gen_op1i(INDEX_op_exit_tb, val);
2731 }
2732 
2733 void tcg_gen_goto_tb(unsigned idx)
2734 {
2735     /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */
2736     tcg_debug_assert(!(tcg_ctx->tb_cflags & CF_NO_GOTO_TB));
2737     /* We only support two chained exits.  */
2738     tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
2739 #ifdef CONFIG_DEBUG_TCG
2740     /* Verify that we haven't seen this numbered exit before.  */
2741     tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2742     tcg_ctx->goto_tb_issue_mask |= 1 << idx;
2743 #endif
2744     plugin_gen_disable_mem_helpers();
2745     tcg_gen_op1i(INDEX_op_goto_tb, idx);
2746 }
2747 
2748 void tcg_gen_lookup_and_goto_ptr(void)
2749 {
2750     TCGv_ptr ptr;
2751 
2752     if (tcg_ctx->tb_cflags & CF_NO_GOTO_PTR) {
2753         tcg_gen_exit_tb(NULL, 0);
2754         return;
2755     }
2756 
2757     plugin_gen_disable_mem_helpers();
2758     ptr = tcg_temp_new_ptr();
2759     gen_helper_lookup_tb_ptr(ptr, cpu_env);
2760     tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
2761     tcg_temp_free_ptr(ptr);
2762 }
2763 
2764 static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
2765 {
2766     /* Trigger the asserts within as early as possible.  */
2767     unsigned a_bits = get_alignment_bits(op);
2768 
2769     /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */
2770     if (a_bits == (op & MO_SIZE)) {
2771         op = (op & ~MO_AMASK) | MO_ALIGN;
2772     }
2773 
2774     switch (op & MO_SIZE) {
2775     case MO_8:
2776         op &= ~MO_BSWAP;
2777         break;
2778     case MO_16:
2779         break;
2780     case MO_32:
2781         if (!is64) {
2782             op &= ~MO_SIGN;
2783         }
2784         break;
2785     case MO_64:
2786         if (is64) {
2787             op &= ~MO_SIGN;
2788             break;
2789         }
2790         /* fall through */
2791     default:
2792         g_assert_not_reached();
2793     }
2794     if (st) {
2795         op &= ~MO_SIGN;
2796     }
2797     return op;
2798 }
2799 
2800 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2801                          MemOp memop, TCGArg idx)
2802 {
2803     MemOpIdx oi = make_memop_idx(memop, idx);
2804 #if TARGET_LONG_BITS == 32
2805     tcg_gen_op3i_i32(opc, val, addr, oi);
2806 #else
2807     if (TCG_TARGET_REG_BITS == 32) {
2808         tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2809     } else {
2810         tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
2811     }
2812 #endif
2813 }
2814 
2815 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2816                          MemOp memop, TCGArg idx)
2817 {
2818     MemOpIdx oi = make_memop_idx(memop, idx);
2819 #if TARGET_LONG_BITS == 32
2820     if (TCG_TARGET_REG_BITS == 32) {
2821         tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
2822     } else {
2823         tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
2824     }
2825 #else
2826     if (TCG_TARGET_REG_BITS == 32) {
2827         tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2828                          TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2829     } else {
2830         tcg_gen_op3i_i64(opc, val, addr, oi);
2831     }
2832 #endif
2833 }
2834 
2835 static void tcg_gen_req_mo(TCGBar type)
2836 {
2837 #ifdef TCG_GUEST_DEFAULT_MO
2838     type &= TCG_GUEST_DEFAULT_MO;
2839 #endif
2840     type &= ~TCG_TARGET_DEFAULT_MO;
2841     if (type) {
2842         tcg_gen_mb(type | TCG_BAR_SC);
2843     }
2844 }
2845 
2846 static inline TCGv plugin_prep_mem_callbacks(TCGv vaddr)
2847 {
2848 #ifdef CONFIG_PLUGIN
2849     if (tcg_ctx->plugin_insn != NULL) {
2850         /* Save a copy of the vaddr for use after a load.  */
2851         TCGv temp = tcg_temp_new();
2852         tcg_gen_mov_tl(temp, vaddr);
2853         return temp;
2854     }
2855 #endif
2856     return vaddr;
2857 }
2858 
2859 static void plugin_gen_mem_callbacks(TCGv vaddr, MemOpIdx oi,
2860                                      enum qemu_plugin_mem_rw rw)
2861 {
2862 #ifdef CONFIG_PLUGIN
2863     if (tcg_ctx->plugin_insn != NULL) {
2864         qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw);
2865         plugin_gen_empty_mem_callback(vaddr, info);
2866         tcg_temp_free(vaddr);
2867     }
2868 #endif
2869 }
2870 
2871 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
2872 {
2873     MemOp orig_memop;
2874     MemOpIdx oi;
2875 
2876     tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2877     memop = tcg_canonicalize_memop(memop, 0, 0);
2878     oi = make_memop_idx(memop, idx);
2879 
2880     orig_memop = memop;
2881     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2882         memop &= ~MO_BSWAP;
2883         /* The bswap primitive benefits from zero-extended input.  */
2884         if ((memop & MO_SSIZE) == MO_SW) {
2885             memop &= ~MO_SIGN;
2886         }
2887     }
2888 
2889     addr = plugin_prep_mem_callbacks(addr);
2890     gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
2891     plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
2892 
2893     if ((orig_memop ^ memop) & MO_BSWAP) {
2894         switch (orig_memop & MO_SIZE) {
2895         case MO_16:
2896             tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN
2897                                            ? TCG_BSWAP_IZ | TCG_BSWAP_OS
2898                                            : TCG_BSWAP_IZ | TCG_BSWAP_OZ));
2899             break;
2900         case MO_32:
2901             tcg_gen_bswap32_i32(val, val);
2902             break;
2903         default:
2904             g_assert_not_reached();
2905         }
2906     }
2907 }
2908 
2909 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
2910 {
2911     TCGv_i32 swap = NULL;
2912     MemOpIdx oi;
2913 
2914     tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2915     memop = tcg_canonicalize_memop(memop, 0, 1);
2916     oi = make_memop_idx(memop, idx);
2917 
2918     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2919         swap = tcg_temp_new_i32();
2920         switch (memop & MO_SIZE) {
2921         case MO_16:
2922             tcg_gen_bswap16_i32(swap, val, 0);
2923             break;
2924         case MO_32:
2925             tcg_gen_bswap32_i32(swap, val);
2926             break;
2927         default:
2928             g_assert_not_reached();
2929         }
2930         val = swap;
2931         memop &= ~MO_BSWAP;
2932     }
2933 
2934     addr = plugin_prep_mem_callbacks(addr);
2935     if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) {
2936         gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx);
2937     } else {
2938         gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
2939     }
2940     plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
2941 
2942     if (swap) {
2943         tcg_temp_free_i32(swap);
2944     }
2945 }
2946 
2947 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
2948 {
2949     MemOp orig_memop;
2950     MemOpIdx oi;
2951 
2952     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2953         tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2954         if (memop & MO_SIGN) {
2955             tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2956         } else {
2957             tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2958         }
2959         return;
2960     }
2961 
2962     tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2963     memop = tcg_canonicalize_memop(memop, 1, 0);
2964     oi = make_memop_idx(memop, idx);
2965 
2966     orig_memop = memop;
2967     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2968         memop &= ~MO_BSWAP;
2969         /* The bswap primitive benefits from zero-extended input.  */
2970         if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
2971             memop &= ~MO_SIGN;
2972         }
2973     }
2974 
2975     addr = plugin_prep_mem_callbacks(addr);
2976     gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
2977     plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
2978 
2979     if ((orig_memop ^ memop) & MO_BSWAP) {
2980         int flags = (orig_memop & MO_SIGN
2981                      ? TCG_BSWAP_IZ | TCG_BSWAP_OS
2982                      : TCG_BSWAP_IZ | TCG_BSWAP_OZ);
2983         switch (orig_memop & MO_SIZE) {
2984         case MO_16:
2985             tcg_gen_bswap16_i64(val, val, flags);
2986             break;
2987         case MO_32:
2988             tcg_gen_bswap32_i64(val, val, flags);
2989             break;
2990         case MO_64:
2991             tcg_gen_bswap64_i64(val, val);
2992             break;
2993         default:
2994             g_assert_not_reached();
2995         }
2996     }
2997 }
2998 
2999 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
3000 {
3001     TCGv_i64 swap = NULL;
3002     MemOpIdx oi;
3003 
3004     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
3005         tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
3006         return;
3007     }
3008 
3009     tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
3010     memop = tcg_canonicalize_memop(memop, 1, 1);
3011     oi = make_memop_idx(memop, idx);
3012 
3013     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
3014         swap = tcg_temp_new_i64();
3015         switch (memop & MO_SIZE) {
3016         case MO_16:
3017             tcg_gen_bswap16_i64(swap, val, 0);
3018             break;
3019         case MO_32:
3020             tcg_gen_bswap32_i64(swap, val, 0);
3021             break;
3022         case MO_64:
3023             tcg_gen_bswap64_i64(swap, val);
3024             break;
3025         default:
3026             g_assert_not_reached();
3027         }
3028         val = swap;
3029         memop &= ~MO_BSWAP;
3030     }
3031 
3032     addr = plugin_prep_mem_callbacks(addr);
3033     gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
3034     plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
3035 
3036     if (swap) {
3037         tcg_temp_free_i64(swap);
3038     }
3039 }
3040 
3041 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc)
3042 {
3043     switch (opc & MO_SSIZE) {
3044     case MO_SB:
3045         tcg_gen_ext8s_i32(ret, val);
3046         break;
3047     case MO_UB:
3048         tcg_gen_ext8u_i32(ret, val);
3049         break;
3050     case MO_SW:
3051         tcg_gen_ext16s_i32(ret, val);
3052         break;
3053     case MO_UW:
3054         tcg_gen_ext16u_i32(ret, val);
3055         break;
3056     default:
3057         tcg_gen_mov_i32(ret, val);
3058         break;
3059     }
3060 }
3061 
3062 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc)
3063 {
3064     switch (opc & MO_SSIZE) {
3065     case MO_SB:
3066         tcg_gen_ext8s_i64(ret, val);
3067         break;
3068     case MO_UB:
3069         tcg_gen_ext8u_i64(ret, val);
3070         break;
3071     case MO_SW:
3072         tcg_gen_ext16s_i64(ret, val);
3073         break;
3074     case MO_UW:
3075         tcg_gen_ext16u_i64(ret, val);
3076         break;
3077     case MO_SL:
3078         tcg_gen_ext32s_i64(ret, val);
3079         break;
3080     case MO_UL:
3081         tcg_gen_ext32u_i64(ret, val);
3082         break;
3083     default:
3084         tcg_gen_mov_i64(ret, val);
3085         break;
3086     }
3087 }
3088 
3089 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
3090                                   TCGv_i32, TCGv_i32, TCGv_i32);
3091 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
3092                                   TCGv_i64, TCGv_i64, TCGv_i32);
3093 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
3094                                   TCGv_i32, TCGv_i32);
3095 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
3096                                   TCGv_i64, TCGv_i32);
3097 
3098 #ifdef CONFIG_ATOMIC64
3099 # define WITH_ATOMIC64(X) X,
3100 #else
3101 # define WITH_ATOMIC64(X)
3102 #endif
3103 
3104 static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
3105     [MO_8] = gen_helper_atomic_cmpxchgb,
3106     [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
3107     [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
3108     [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
3109     [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
3110     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
3111     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
3112 };
3113 
3114 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
3115                                 TCGv_i32 newv, TCGArg idx, MemOp memop)
3116 {
3117     memop = tcg_canonicalize_memop(memop, 0, 0);
3118 
3119     if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
3120         TCGv_i32 t1 = tcg_temp_new_i32();
3121         TCGv_i32 t2 = tcg_temp_new_i32();
3122 
3123         tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
3124 
3125         tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3126         tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
3127         tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3128         tcg_temp_free_i32(t2);
3129 
3130         if (memop & MO_SIGN) {
3131             tcg_gen_ext_i32(retv, t1, memop);
3132         } else {
3133             tcg_gen_mov_i32(retv, t1);
3134         }
3135         tcg_temp_free_i32(t1);
3136     } else {
3137         gen_atomic_cx_i32 gen;
3138         MemOpIdx oi;
3139 
3140         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3141         tcg_debug_assert(gen != NULL);
3142 
3143         oi = make_memop_idx(memop & ~MO_SIGN, idx);
3144         gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3145 
3146         if (memop & MO_SIGN) {
3147             tcg_gen_ext_i32(retv, retv, memop);
3148         }
3149     }
3150 }
3151 
3152 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
3153                                 TCGv_i64 newv, TCGArg idx, MemOp memop)
3154 {
3155     memop = tcg_canonicalize_memop(memop, 1, 0);
3156 
3157     if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
3158         TCGv_i64 t1 = tcg_temp_new_i64();
3159         TCGv_i64 t2 = tcg_temp_new_i64();
3160 
3161         tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
3162 
3163         tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3164         tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
3165         tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3166         tcg_temp_free_i64(t2);
3167 
3168         if (memop & MO_SIGN) {
3169             tcg_gen_ext_i64(retv, t1, memop);
3170         } else {
3171             tcg_gen_mov_i64(retv, t1);
3172         }
3173         tcg_temp_free_i64(t1);
3174     } else if ((memop & MO_SIZE) == MO_64) {
3175 #ifdef CONFIG_ATOMIC64
3176         gen_atomic_cx_i64 gen;
3177         MemOpIdx oi;
3178 
3179         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3180         tcg_debug_assert(gen != NULL);
3181 
3182         oi = make_memop_idx(memop, idx);
3183         gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3184 #else
3185         gen_helper_exit_atomic(cpu_env);
3186         /* Produce a result, so that we have a well-formed opcode stream
3187            with respect to uses of the result in the (dead) code following.  */
3188         tcg_gen_movi_i64(retv, 0);
3189 #endif /* CONFIG_ATOMIC64 */
3190     } else {
3191         TCGv_i32 c32 = tcg_temp_new_i32();
3192         TCGv_i32 n32 = tcg_temp_new_i32();
3193         TCGv_i32 r32 = tcg_temp_new_i32();
3194 
3195         tcg_gen_extrl_i64_i32(c32, cmpv);
3196         tcg_gen_extrl_i64_i32(n32, newv);
3197         tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
3198         tcg_temp_free_i32(c32);
3199         tcg_temp_free_i32(n32);
3200 
3201         tcg_gen_extu_i32_i64(retv, r32);
3202         tcg_temp_free_i32(r32);
3203 
3204         if (memop & MO_SIGN) {
3205             tcg_gen_ext_i64(retv, retv, memop);
3206         }
3207     }
3208 }
3209 
3210 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3211                                 TCGArg idx, MemOp memop, bool new_val,
3212                                 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
3213 {
3214     TCGv_i32 t1 = tcg_temp_new_i32();
3215     TCGv_i32 t2 = tcg_temp_new_i32();
3216 
3217     memop = tcg_canonicalize_memop(memop, 0, 0);
3218 
3219     tcg_gen_qemu_ld_i32(t1, addr, idx, memop);
3220     tcg_gen_ext_i32(t2, val, memop);
3221     gen(t2, t1, t2);
3222     tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3223 
3224     tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
3225     tcg_temp_free_i32(t1);
3226     tcg_temp_free_i32(t2);
3227 }
3228 
3229 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3230                              TCGArg idx, MemOp memop, void * const table[])
3231 {
3232     gen_atomic_op_i32 gen;
3233     MemOpIdx oi;
3234 
3235     memop = tcg_canonicalize_memop(memop, 0, 0);
3236 
3237     gen = table[memop & (MO_SIZE | MO_BSWAP)];
3238     tcg_debug_assert(gen != NULL);
3239 
3240     oi = make_memop_idx(memop & ~MO_SIGN, idx);
3241     gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
3242 
3243     if (memop & MO_SIGN) {
3244         tcg_gen_ext_i32(ret, ret, memop);
3245     }
3246 }
3247 
3248 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3249                                 TCGArg idx, MemOp memop, bool new_val,
3250                                 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
3251 {
3252     TCGv_i64 t1 = tcg_temp_new_i64();
3253     TCGv_i64 t2 = tcg_temp_new_i64();
3254 
3255     memop = tcg_canonicalize_memop(memop, 1, 0);
3256 
3257     tcg_gen_qemu_ld_i64(t1, addr, idx, memop);
3258     tcg_gen_ext_i64(t2, val, memop);
3259     gen(t2, t1, t2);
3260     tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3261 
3262     tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
3263     tcg_temp_free_i64(t1);
3264     tcg_temp_free_i64(t2);
3265 }
3266 
3267 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3268                              TCGArg idx, MemOp memop, void * const table[])
3269 {
3270     memop = tcg_canonicalize_memop(memop, 1, 0);
3271 
3272     if ((memop & MO_SIZE) == MO_64) {
3273 #ifdef CONFIG_ATOMIC64
3274         gen_atomic_op_i64 gen;
3275         MemOpIdx oi;
3276 
3277         gen = table[memop & (MO_SIZE | MO_BSWAP)];
3278         tcg_debug_assert(gen != NULL);
3279 
3280         oi = make_memop_idx(memop & ~MO_SIGN, idx);
3281         gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
3282 #else
3283         gen_helper_exit_atomic(cpu_env);
3284         /* Produce a result, so that we have a well-formed opcode stream
3285            with respect to uses of the result in the (dead) code following.  */
3286         tcg_gen_movi_i64(ret, 0);
3287 #endif /* CONFIG_ATOMIC64 */
3288     } else {
3289         TCGv_i32 v32 = tcg_temp_new_i32();
3290         TCGv_i32 r32 = tcg_temp_new_i32();
3291 
3292         tcg_gen_extrl_i64_i32(v32, val);
3293         do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3294         tcg_temp_free_i32(v32);
3295 
3296         tcg_gen_extu_i32_i64(ret, r32);
3297         tcg_temp_free_i32(r32);
3298 
3299         if (memop & MO_SIGN) {
3300             tcg_gen_ext_i64(ret, ret, memop);
3301         }
3302     }
3303 }
3304 
3305 #define GEN_ATOMIC_HELPER(NAME, OP, NEW)                                \
3306 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = {          \
3307     [MO_8] = gen_helper_atomic_##NAME##b,                               \
3308     [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le,                   \
3309     [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be,                   \
3310     [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le,                   \
3311     [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be,                   \
3312     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le)     \
3313     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be)     \
3314 };                                                                      \
3315 void tcg_gen_atomic_##NAME##_i32                                        \
3316     (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop)    \
3317 {                                                                       \
3318     if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
3319         do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME);     \
3320     } else {                                                            \
3321         do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW,            \
3322                             tcg_gen_##OP##_i32);                        \
3323     }                                                                   \
3324 }                                                                       \
3325 void tcg_gen_atomic_##NAME##_i64                                        \
3326     (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop)    \
3327 {                                                                       \
3328     if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
3329         do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME);     \
3330     } else {                                                            \
3331         do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW,            \
3332                             tcg_gen_##OP##_i64);                        \
3333     }                                                                   \
3334 }
3335 
3336 GEN_ATOMIC_HELPER(fetch_add, add, 0)
3337 GEN_ATOMIC_HELPER(fetch_and, and, 0)
3338 GEN_ATOMIC_HELPER(fetch_or, or, 0)
3339 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
3340 GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3341 GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3342 GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3343 GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
3344 
3345 GEN_ATOMIC_HELPER(add_fetch, add, 1)
3346 GEN_ATOMIC_HELPER(and_fetch, and, 1)
3347 GEN_ATOMIC_HELPER(or_fetch, or, 1)
3348 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
3349 GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3350 GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3351 GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3352 GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
3353 
3354 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3355 {
3356     tcg_gen_mov_i32(r, b);
3357 }
3358 
3359 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3360 {
3361     tcg_gen_mov_i64(r, b);
3362 }
3363 
3364 GEN_ATOMIC_HELPER(xchg, mov2, 0)
3365 
3366 #undef GEN_ATOMIC_HELPER
3367