xref: /openbmc/qemu/tcg/tcg-op.c (revision 438c78da)
1 /*
2  * Tiny Code Generator for QEMU
3  *
4  * Copyright (c) 2008 Fabrice Bellard
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22  * THE SOFTWARE.
23  */
24 
25 #include "qemu/osdep.h"
26 #include "qemu-common.h"
27 #include "cpu.h"
28 #include "exec/exec-all.h"
29 #include "tcg.h"
30 #include "tcg-op.h"
31 #include "tcg-mo.h"
32 #include "trace-tcg.h"
33 #include "trace/mem.h"
34 
35 /* Reduce the number of ifdefs below.  This assumes that all uses of
36    TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
37    the compiler can eliminate.  */
38 #if TCG_TARGET_REG_BITS == 64
39 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
40 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
41 #define TCGV_LOW  TCGV_LOW_link_error
42 #define TCGV_HIGH TCGV_HIGH_link_error
43 #endif
44 
45 void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
46 {
47     TCGOp *op = tcg_emit_op(opc);
48     op->args[0] = a1;
49 }
50 
51 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
52 {
53     TCGOp *op = tcg_emit_op(opc);
54     op->args[0] = a1;
55     op->args[1] = a2;
56 }
57 
58 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
59 {
60     TCGOp *op = tcg_emit_op(opc);
61     op->args[0] = a1;
62     op->args[1] = a2;
63     op->args[2] = a3;
64 }
65 
66 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
67 {
68     TCGOp *op = tcg_emit_op(opc);
69     op->args[0] = a1;
70     op->args[1] = a2;
71     op->args[2] = a3;
72     op->args[3] = a4;
73 }
74 
75 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
76                  TCGArg a4, TCGArg a5)
77 {
78     TCGOp *op = tcg_emit_op(opc);
79     op->args[0] = a1;
80     op->args[1] = a2;
81     op->args[2] = a3;
82     op->args[3] = a4;
83     op->args[4] = a5;
84 }
85 
86 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
87                  TCGArg a4, TCGArg a5, TCGArg a6)
88 {
89     TCGOp *op = tcg_emit_op(opc);
90     op->args[0] = a1;
91     op->args[1] = a2;
92     op->args[2] = a3;
93     op->args[3] = a4;
94     op->args[4] = a5;
95     op->args[5] = a6;
96 }
97 
98 void tcg_gen_mb(TCGBar mb_type)
99 {
100     if (tcg_ctx->tb_cflags & CF_PARALLEL) {
101         tcg_gen_op1(INDEX_op_mb, mb_type);
102     }
103 }
104 
105 /* 32 bit ops */
106 
107 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
108 {
109     /* some cases can be optimized here */
110     if (arg2 == 0) {
111         tcg_gen_mov_i32(ret, arg1);
112     } else {
113         TCGv_i32 t0 = tcg_const_i32(arg2);
114         tcg_gen_add_i32(ret, arg1, t0);
115         tcg_temp_free_i32(t0);
116     }
117 }
118 
119 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
120 {
121     if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
122         /* Don't recurse with tcg_gen_neg_i32.  */
123         tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
124     } else {
125         TCGv_i32 t0 = tcg_const_i32(arg1);
126         tcg_gen_sub_i32(ret, t0, arg2);
127         tcg_temp_free_i32(t0);
128     }
129 }
130 
131 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
132 {
133     /* some cases can be optimized here */
134     if (arg2 == 0) {
135         tcg_gen_mov_i32(ret, arg1);
136     } else {
137         TCGv_i32 t0 = tcg_const_i32(arg2);
138         tcg_gen_sub_i32(ret, arg1, t0);
139         tcg_temp_free_i32(t0);
140     }
141 }
142 
143 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
144 {
145     TCGv_i32 t0;
146     /* Some cases can be optimized here.  */
147     switch (arg2) {
148     case 0:
149         tcg_gen_movi_i32(ret, 0);
150         return;
151     case -1:
152         tcg_gen_mov_i32(ret, arg1);
153         return;
154     case 0xff:
155         /* Don't recurse with tcg_gen_ext8u_i32.  */
156         if (TCG_TARGET_HAS_ext8u_i32) {
157             tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
158             return;
159         }
160         break;
161     case 0xffff:
162         if (TCG_TARGET_HAS_ext16u_i32) {
163             tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
164             return;
165         }
166         break;
167     }
168     t0 = tcg_const_i32(arg2);
169     tcg_gen_and_i32(ret, arg1, t0);
170     tcg_temp_free_i32(t0);
171 }
172 
173 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
174 {
175     /* Some cases can be optimized here.  */
176     if (arg2 == -1) {
177         tcg_gen_movi_i32(ret, -1);
178     } else if (arg2 == 0) {
179         tcg_gen_mov_i32(ret, arg1);
180     } else {
181         TCGv_i32 t0 = tcg_const_i32(arg2);
182         tcg_gen_or_i32(ret, arg1, t0);
183         tcg_temp_free_i32(t0);
184     }
185 }
186 
187 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
188 {
189     /* Some cases can be optimized here.  */
190     if (arg2 == 0) {
191         tcg_gen_mov_i32(ret, arg1);
192     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
193         /* Don't recurse with tcg_gen_not_i32.  */
194         tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
195     } else {
196         TCGv_i32 t0 = tcg_const_i32(arg2);
197         tcg_gen_xor_i32(ret, arg1, t0);
198         tcg_temp_free_i32(t0);
199     }
200 }
201 
202 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
203 {
204     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
205     if (arg2 == 0) {
206         tcg_gen_mov_i32(ret, arg1);
207     } else {
208         TCGv_i32 t0 = tcg_const_i32(arg2);
209         tcg_gen_shl_i32(ret, arg1, t0);
210         tcg_temp_free_i32(t0);
211     }
212 }
213 
214 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
215 {
216     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
217     if (arg2 == 0) {
218         tcg_gen_mov_i32(ret, arg1);
219     } else {
220         TCGv_i32 t0 = tcg_const_i32(arg2);
221         tcg_gen_shr_i32(ret, arg1, t0);
222         tcg_temp_free_i32(t0);
223     }
224 }
225 
226 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
227 {
228     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
229     if (arg2 == 0) {
230         tcg_gen_mov_i32(ret, arg1);
231     } else {
232         TCGv_i32 t0 = tcg_const_i32(arg2);
233         tcg_gen_sar_i32(ret, arg1, t0);
234         tcg_temp_free_i32(t0);
235     }
236 }
237 
238 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
239 {
240     if (cond == TCG_COND_ALWAYS) {
241         tcg_gen_br(l);
242     } else if (cond != TCG_COND_NEVER) {
243         tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
244     }
245 }
246 
247 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
248 {
249     if (cond == TCG_COND_ALWAYS) {
250         tcg_gen_br(l);
251     } else if (cond != TCG_COND_NEVER) {
252         TCGv_i32 t0 = tcg_const_i32(arg2);
253         tcg_gen_brcond_i32(cond, arg1, t0, l);
254         tcg_temp_free_i32(t0);
255     }
256 }
257 
258 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
259                          TCGv_i32 arg1, TCGv_i32 arg2)
260 {
261     if (cond == TCG_COND_ALWAYS) {
262         tcg_gen_movi_i32(ret, 1);
263     } else if (cond == TCG_COND_NEVER) {
264         tcg_gen_movi_i32(ret, 0);
265     } else {
266         tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
267     }
268 }
269 
270 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
271                           TCGv_i32 arg1, int32_t arg2)
272 {
273     TCGv_i32 t0 = tcg_const_i32(arg2);
274     tcg_gen_setcond_i32(cond, ret, arg1, t0);
275     tcg_temp_free_i32(t0);
276 }
277 
278 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
279 {
280     if (arg2 == 0) {
281         tcg_gen_movi_i32(ret, 0);
282     } else if (is_power_of_2(arg2)) {
283         tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
284     } else {
285         TCGv_i32 t0 = tcg_const_i32(arg2);
286         tcg_gen_mul_i32(ret, arg1, t0);
287         tcg_temp_free_i32(t0);
288     }
289 }
290 
291 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
292 {
293     if (TCG_TARGET_HAS_div_i32) {
294         tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
295     } else if (TCG_TARGET_HAS_div2_i32) {
296         TCGv_i32 t0 = tcg_temp_new_i32();
297         tcg_gen_sari_i32(t0, arg1, 31);
298         tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
299         tcg_temp_free_i32(t0);
300     } else {
301         gen_helper_div_i32(ret, arg1, arg2);
302     }
303 }
304 
305 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
306 {
307     if (TCG_TARGET_HAS_rem_i32) {
308         tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
309     } else if (TCG_TARGET_HAS_div_i32) {
310         TCGv_i32 t0 = tcg_temp_new_i32();
311         tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
312         tcg_gen_mul_i32(t0, t0, arg2);
313         tcg_gen_sub_i32(ret, arg1, t0);
314         tcg_temp_free_i32(t0);
315     } else if (TCG_TARGET_HAS_div2_i32) {
316         TCGv_i32 t0 = tcg_temp_new_i32();
317         tcg_gen_sari_i32(t0, arg1, 31);
318         tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
319         tcg_temp_free_i32(t0);
320     } else {
321         gen_helper_rem_i32(ret, arg1, arg2);
322     }
323 }
324 
325 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
326 {
327     if (TCG_TARGET_HAS_div_i32) {
328         tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
329     } else if (TCG_TARGET_HAS_div2_i32) {
330         TCGv_i32 t0 = tcg_temp_new_i32();
331         tcg_gen_movi_i32(t0, 0);
332         tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
333         tcg_temp_free_i32(t0);
334     } else {
335         gen_helper_divu_i32(ret, arg1, arg2);
336     }
337 }
338 
339 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
340 {
341     if (TCG_TARGET_HAS_rem_i32) {
342         tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
343     } else if (TCG_TARGET_HAS_div_i32) {
344         TCGv_i32 t0 = tcg_temp_new_i32();
345         tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
346         tcg_gen_mul_i32(t0, t0, arg2);
347         tcg_gen_sub_i32(ret, arg1, t0);
348         tcg_temp_free_i32(t0);
349     } else if (TCG_TARGET_HAS_div2_i32) {
350         TCGv_i32 t0 = tcg_temp_new_i32();
351         tcg_gen_movi_i32(t0, 0);
352         tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
353         tcg_temp_free_i32(t0);
354     } else {
355         gen_helper_remu_i32(ret, arg1, arg2);
356     }
357 }
358 
359 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
360 {
361     if (TCG_TARGET_HAS_andc_i32) {
362         tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
363     } else {
364         TCGv_i32 t0 = tcg_temp_new_i32();
365         tcg_gen_not_i32(t0, arg2);
366         tcg_gen_and_i32(ret, arg1, t0);
367         tcg_temp_free_i32(t0);
368     }
369 }
370 
371 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
372 {
373     if (TCG_TARGET_HAS_eqv_i32) {
374         tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
375     } else {
376         tcg_gen_xor_i32(ret, arg1, arg2);
377         tcg_gen_not_i32(ret, ret);
378     }
379 }
380 
381 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
382 {
383     if (TCG_TARGET_HAS_nand_i32) {
384         tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
385     } else {
386         tcg_gen_and_i32(ret, arg1, arg2);
387         tcg_gen_not_i32(ret, ret);
388     }
389 }
390 
391 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
392 {
393     if (TCG_TARGET_HAS_nor_i32) {
394         tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
395     } else {
396         tcg_gen_or_i32(ret, arg1, arg2);
397         tcg_gen_not_i32(ret, ret);
398     }
399 }
400 
401 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
402 {
403     if (TCG_TARGET_HAS_orc_i32) {
404         tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
405     } else {
406         TCGv_i32 t0 = tcg_temp_new_i32();
407         tcg_gen_not_i32(t0, arg2);
408         tcg_gen_or_i32(ret, arg1, t0);
409         tcg_temp_free_i32(t0);
410     }
411 }
412 
413 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
414 {
415     if (TCG_TARGET_HAS_clz_i32) {
416         tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
417     } else if (TCG_TARGET_HAS_clz_i64) {
418         TCGv_i64 t1 = tcg_temp_new_i64();
419         TCGv_i64 t2 = tcg_temp_new_i64();
420         tcg_gen_extu_i32_i64(t1, arg1);
421         tcg_gen_extu_i32_i64(t2, arg2);
422         tcg_gen_addi_i64(t2, t2, 32);
423         tcg_gen_clz_i64(t1, t1, t2);
424         tcg_gen_extrl_i64_i32(ret, t1);
425         tcg_temp_free_i64(t1);
426         tcg_temp_free_i64(t2);
427         tcg_gen_subi_i32(ret, ret, 32);
428     } else {
429         gen_helper_clz_i32(ret, arg1, arg2);
430     }
431 }
432 
433 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
434 {
435     TCGv_i32 t = tcg_const_i32(arg2);
436     tcg_gen_clz_i32(ret, arg1, t);
437     tcg_temp_free_i32(t);
438 }
439 
440 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
441 {
442     if (TCG_TARGET_HAS_ctz_i32) {
443         tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
444     } else if (TCG_TARGET_HAS_ctz_i64) {
445         TCGv_i64 t1 = tcg_temp_new_i64();
446         TCGv_i64 t2 = tcg_temp_new_i64();
447         tcg_gen_extu_i32_i64(t1, arg1);
448         tcg_gen_extu_i32_i64(t2, arg2);
449         tcg_gen_ctz_i64(t1, t1, t2);
450         tcg_gen_extrl_i64_i32(ret, t1);
451         tcg_temp_free_i64(t1);
452         tcg_temp_free_i64(t2);
453     } else if (TCG_TARGET_HAS_ctpop_i32
454                || TCG_TARGET_HAS_ctpop_i64
455                || TCG_TARGET_HAS_clz_i32
456                || TCG_TARGET_HAS_clz_i64) {
457         TCGv_i32 z, t = tcg_temp_new_i32();
458 
459         if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
460             tcg_gen_subi_i32(t, arg1, 1);
461             tcg_gen_andc_i32(t, t, arg1);
462             tcg_gen_ctpop_i32(t, t);
463         } else {
464             /* Since all non-x86 hosts have clz(0) == 32, don't fight it.  */
465             tcg_gen_neg_i32(t, arg1);
466             tcg_gen_and_i32(t, t, arg1);
467             tcg_gen_clzi_i32(t, t, 32);
468             tcg_gen_xori_i32(t, t, 31);
469         }
470         z = tcg_const_i32(0);
471         tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
472         tcg_temp_free_i32(t);
473         tcg_temp_free_i32(z);
474     } else {
475         gen_helper_ctz_i32(ret, arg1, arg2);
476     }
477 }
478 
479 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
480 {
481     if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
482         /* This equivalence has the advantage of not requiring a fixup.  */
483         TCGv_i32 t = tcg_temp_new_i32();
484         tcg_gen_subi_i32(t, arg1, 1);
485         tcg_gen_andc_i32(t, t, arg1);
486         tcg_gen_ctpop_i32(ret, t);
487         tcg_temp_free_i32(t);
488     } else {
489         TCGv_i32 t = tcg_const_i32(arg2);
490         tcg_gen_ctz_i32(ret, arg1, t);
491         tcg_temp_free_i32(t);
492     }
493 }
494 
495 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
496 {
497     if (TCG_TARGET_HAS_clz_i32) {
498         TCGv_i32 t = tcg_temp_new_i32();
499         tcg_gen_sari_i32(t, arg, 31);
500         tcg_gen_xor_i32(t, t, arg);
501         tcg_gen_clzi_i32(t, t, 32);
502         tcg_gen_subi_i32(ret, t, 1);
503         tcg_temp_free_i32(t);
504     } else {
505         gen_helper_clrsb_i32(ret, arg);
506     }
507 }
508 
509 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
510 {
511     if (TCG_TARGET_HAS_ctpop_i32) {
512         tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
513     } else if (TCG_TARGET_HAS_ctpop_i64) {
514         TCGv_i64 t = tcg_temp_new_i64();
515         tcg_gen_extu_i32_i64(t, arg1);
516         tcg_gen_ctpop_i64(t, t);
517         tcg_gen_extrl_i64_i32(ret, t);
518         tcg_temp_free_i64(t);
519     } else {
520         gen_helper_ctpop_i32(ret, arg1);
521     }
522 }
523 
524 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
525 {
526     if (TCG_TARGET_HAS_rot_i32) {
527         tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
528     } else {
529         TCGv_i32 t0, t1;
530 
531         t0 = tcg_temp_new_i32();
532         t1 = tcg_temp_new_i32();
533         tcg_gen_shl_i32(t0, arg1, arg2);
534         tcg_gen_subfi_i32(t1, 32, arg2);
535         tcg_gen_shr_i32(t1, arg1, t1);
536         tcg_gen_or_i32(ret, t0, t1);
537         tcg_temp_free_i32(t0);
538         tcg_temp_free_i32(t1);
539     }
540 }
541 
542 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
543 {
544     tcg_debug_assert(arg2 < 32);
545     /* some cases can be optimized here */
546     if (arg2 == 0) {
547         tcg_gen_mov_i32(ret, arg1);
548     } else if (TCG_TARGET_HAS_rot_i32) {
549         TCGv_i32 t0 = tcg_const_i32(arg2);
550         tcg_gen_rotl_i32(ret, arg1, t0);
551         tcg_temp_free_i32(t0);
552     } else {
553         TCGv_i32 t0, t1;
554         t0 = tcg_temp_new_i32();
555         t1 = tcg_temp_new_i32();
556         tcg_gen_shli_i32(t0, arg1, arg2);
557         tcg_gen_shri_i32(t1, arg1, 32 - arg2);
558         tcg_gen_or_i32(ret, t0, t1);
559         tcg_temp_free_i32(t0);
560         tcg_temp_free_i32(t1);
561     }
562 }
563 
564 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
565 {
566     if (TCG_TARGET_HAS_rot_i32) {
567         tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
568     } else {
569         TCGv_i32 t0, t1;
570 
571         t0 = tcg_temp_new_i32();
572         t1 = tcg_temp_new_i32();
573         tcg_gen_shr_i32(t0, arg1, arg2);
574         tcg_gen_subfi_i32(t1, 32, arg2);
575         tcg_gen_shl_i32(t1, arg1, t1);
576         tcg_gen_or_i32(ret, t0, t1);
577         tcg_temp_free_i32(t0);
578         tcg_temp_free_i32(t1);
579     }
580 }
581 
582 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
583 {
584     tcg_debug_assert(arg2 < 32);
585     /* some cases can be optimized here */
586     if (arg2 == 0) {
587         tcg_gen_mov_i32(ret, arg1);
588     } else {
589         tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
590     }
591 }
592 
593 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
594                          unsigned int ofs, unsigned int len)
595 {
596     uint32_t mask;
597     TCGv_i32 t1;
598 
599     tcg_debug_assert(ofs < 32);
600     tcg_debug_assert(len > 0);
601     tcg_debug_assert(len <= 32);
602     tcg_debug_assert(ofs + len <= 32);
603 
604     if (len == 32) {
605         tcg_gen_mov_i32(ret, arg2);
606         return;
607     }
608     if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
609         tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
610         return;
611     }
612 
613     mask = (1u << len) - 1;
614     t1 = tcg_temp_new_i32();
615 
616     if (ofs + len < 32) {
617         tcg_gen_andi_i32(t1, arg2, mask);
618         tcg_gen_shli_i32(t1, t1, ofs);
619     } else {
620         tcg_gen_shli_i32(t1, arg2, ofs);
621     }
622     tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
623     tcg_gen_or_i32(ret, ret, t1);
624 
625     tcg_temp_free_i32(t1);
626 }
627 
628 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
629                            unsigned int ofs, unsigned int len)
630 {
631     tcg_debug_assert(ofs < 32);
632     tcg_debug_assert(len > 0);
633     tcg_debug_assert(len <= 32);
634     tcg_debug_assert(ofs + len <= 32);
635 
636     if (ofs + len == 32) {
637         tcg_gen_shli_i32(ret, arg, ofs);
638     } else if (ofs == 0) {
639         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
640     } else if (TCG_TARGET_HAS_deposit_i32
641                && TCG_TARGET_deposit_i32_valid(ofs, len)) {
642         TCGv_i32 zero = tcg_const_i32(0);
643         tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
644         tcg_temp_free_i32(zero);
645     } else {
646         /* To help two-operand hosts we prefer to zero-extend first,
647            which allows ARG to stay live.  */
648         switch (len) {
649         case 16:
650             if (TCG_TARGET_HAS_ext16u_i32) {
651                 tcg_gen_ext16u_i32(ret, arg);
652                 tcg_gen_shli_i32(ret, ret, ofs);
653                 return;
654             }
655             break;
656         case 8:
657             if (TCG_TARGET_HAS_ext8u_i32) {
658                 tcg_gen_ext8u_i32(ret, arg);
659                 tcg_gen_shli_i32(ret, ret, ofs);
660                 return;
661             }
662             break;
663         }
664         /* Otherwise prefer zero-extension over AND for code size.  */
665         switch (ofs + len) {
666         case 16:
667             if (TCG_TARGET_HAS_ext16u_i32) {
668                 tcg_gen_shli_i32(ret, arg, ofs);
669                 tcg_gen_ext16u_i32(ret, ret);
670                 return;
671             }
672             break;
673         case 8:
674             if (TCG_TARGET_HAS_ext8u_i32) {
675                 tcg_gen_shli_i32(ret, arg, ofs);
676                 tcg_gen_ext8u_i32(ret, ret);
677                 return;
678             }
679             break;
680         }
681         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
682         tcg_gen_shli_i32(ret, ret, ofs);
683     }
684 }
685 
686 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
687                          unsigned int ofs, unsigned int len)
688 {
689     tcg_debug_assert(ofs < 32);
690     tcg_debug_assert(len > 0);
691     tcg_debug_assert(len <= 32);
692     tcg_debug_assert(ofs + len <= 32);
693 
694     /* Canonicalize certain special cases, even if extract is supported.  */
695     if (ofs + len == 32) {
696         tcg_gen_shri_i32(ret, arg, 32 - len);
697         return;
698     }
699     if (ofs == 0) {
700         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
701         return;
702     }
703 
704     if (TCG_TARGET_HAS_extract_i32
705         && TCG_TARGET_extract_i32_valid(ofs, len)) {
706         tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
707         return;
708     }
709 
710     /* Assume that zero-extension, if available, is cheaper than a shift.  */
711     switch (ofs + len) {
712     case 16:
713         if (TCG_TARGET_HAS_ext16u_i32) {
714             tcg_gen_ext16u_i32(ret, arg);
715             tcg_gen_shri_i32(ret, ret, ofs);
716             return;
717         }
718         break;
719     case 8:
720         if (TCG_TARGET_HAS_ext8u_i32) {
721             tcg_gen_ext8u_i32(ret, arg);
722             tcg_gen_shri_i32(ret, ret, ofs);
723             return;
724         }
725         break;
726     }
727 
728     /* ??? Ideally we'd know what values are available for immediate AND.
729        Assume that 8 bits are available, plus the special case of 16,
730        so that we get ext8u, ext16u.  */
731     switch (len) {
732     case 1 ... 8: case 16:
733         tcg_gen_shri_i32(ret, arg, ofs);
734         tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
735         break;
736     default:
737         tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
738         tcg_gen_shri_i32(ret, ret, 32 - len);
739         break;
740     }
741 }
742 
743 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
744                           unsigned int ofs, unsigned int len)
745 {
746     tcg_debug_assert(ofs < 32);
747     tcg_debug_assert(len > 0);
748     tcg_debug_assert(len <= 32);
749     tcg_debug_assert(ofs + len <= 32);
750 
751     /* Canonicalize certain special cases, even if extract is supported.  */
752     if (ofs + len == 32) {
753         tcg_gen_sari_i32(ret, arg, 32 - len);
754         return;
755     }
756     if (ofs == 0) {
757         switch (len) {
758         case 16:
759             tcg_gen_ext16s_i32(ret, arg);
760             return;
761         case 8:
762             tcg_gen_ext8s_i32(ret, arg);
763             return;
764         }
765     }
766 
767     if (TCG_TARGET_HAS_sextract_i32
768         && TCG_TARGET_extract_i32_valid(ofs, len)) {
769         tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
770         return;
771     }
772 
773     /* Assume that sign-extension, if available, is cheaper than a shift.  */
774     switch (ofs + len) {
775     case 16:
776         if (TCG_TARGET_HAS_ext16s_i32) {
777             tcg_gen_ext16s_i32(ret, arg);
778             tcg_gen_sari_i32(ret, ret, ofs);
779             return;
780         }
781         break;
782     case 8:
783         if (TCG_TARGET_HAS_ext8s_i32) {
784             tcg_gen_ext8s_i32(ret, arg);
785             tcg_gen_sari_i32(ret, ret, ofs);
786             return;
787         }
788         break;
789     }
790     switch (len) {
791     case 16:
792         if (TCG_TARGET_HAS_ext16s_i32) {
793             tcg_gen_shri_i32(ret, arg, ofs);
794             tcg_gen_ext16s_i32(ret, ret);
795             return;
796         }
797         break;
798     case 8:
799         if (TCG_TARGET_HAS_ext8s_i32) {
800             tcg_gen_shri_i32(ret, arg, ofs);
801             tcg_gen_ext8s_i32(ret, ret);
802             return;
803         }
804         break;
805     }
806 
807     tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
808     tcg_gen_sari_i32(ret, ret, 32 - len);
809 }
810 
811 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
812                          TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
813 {
814     if (cond == TCG_COND_ALWAYS) {
815         tcg_gen_mov_i32(ret, v1);
816     } else if (cond == TCG_COND_NEVER) {
817         tcg_gen_mov_i32(ret, v2);
818     } else if (TCG_TARGET_HAS_movcond_i32) {
819         tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
820     } else {
821         TCGv_i32 t0 = tcg_temp_new_i32();
822         TCGv_i32 t1 = tcg_temp_new_i32();
823         tcg_gen_setcond_i32(cond, t0, c1, c2);
824         tcg_gen_neg_i32(t0, t0);
825         tcg_gen_and_i32(t1, v1, t0);
826         tcg_gen_andc_i32(ret, v2, t0);
827         tcg_gen_or_i32(ret, ret, t1);
828         tcg_temp_free_i32(t0);
829         tcg_temp_free_i32(t1);
830     }
831 }
832 
833 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
834                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
835 {
836     if (TCG_TARGET_HAS_add2_i32) {
837         tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
838     } else {
839         TCGv_i64 t0 = tcg_temp_new_i64();
840         TCGv_i64 t1 = tcg_temp_new_i64();
841         tcg_gen_concat_i32_i64(t0, al, ah);
842         tcg_gen_concat_i32_i64(t1, bl, bh);
843         tcg_gen_add_i64(t0, t0, t1);
844         tcg_gen_extr_i64_i32(rl, rh, t0);
845         tcg_temp_free_i64(t0);
846         tcg_temp_free_i64(t1);
847     }
848 }
849 
850 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
851                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
852 {
853     if (TCG_TARGET_HAS_sub2_i32) {
854         tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
855     } else {
856         TCGv_i64 t0 = tcg_temp_new_i64();
857         TCGv_i64 t1 = tcg_temp_new_i64();
858         tcg_gen_concat_i32_i64(t0, al, ah);
859         tcg_gen_concat_i32_i64(t1, bl, bh);
860         tcg_gen_sub_i64(t0, t0, t1);
861         tcg_gen_extr_i64_i32(rl, rh, t0);
862         tcg_temp_free_i64(t0);
863         tcg_temp_free_i64(t1);
864     }
865 }
866 
867 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
868 {
869     if (TCG_TARGET_HAS_mulu2_i32) {
870         tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
871     } else if (TCG_TARGET_HAS_muluh_i32) {
872         TCGv_i32 t = tcg_temp_new_i32();
873         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
874         tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
875         tcg_gen_mov_i32(rl, t);
876         tcg_temp_free_i32(t);
877     } else {
878         TCGv_i64 t0 = tcg_temp_new_i64();
879         TCGv_i64 t1 = tcg_temp_new_i64();
880         tcg_gen_extu_i32_i64(t0, arg1);
881         tcg_gen_extu_i32_i64(t1, arg2);
882         tcg_gen_mul_i64(t0, t0, t1);
883         tcg_gen_extr_i64_i32(rl, rh, t0);
884         tcg_temp_free_i64(t0);
885         tcg_temp_free_i64(t1);
886     }
887 }
888 
889 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
890 {
891     if (TCG_TARGET_HAS_muls2_i32) {
892         tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
893     } else if (TCG_TARGET_HAS_mulsh_i32) {
894         TCGv_i32 t = tcg_temp_new_i32();
895         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
896         tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
897         tcg_gen_mov_i32(rl, t);
898         tcg_temp_free_i32(t);
899     } else if (TCG_TARGET_REG_BITS == 32) {
900         TCGv_i32 t0 = tcg_temp_new_i32();
901         TCGv_i32 t1 = tcg_temp_new_i32();
902         TCGv_i32 t2 = tcg_temp_new_i32();
903         TCGv_i32 t3 = tcg_temp_new_i32();
904         tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
905         /* Adjust for negative inputs.  */
906         tcg_gen_sari_i32(t2, arg1, 31);
907         tcg_gen_sari_i32(t3, arg2, 31);
908         tcg_gen_and_i32(t2, t2, arg2);
909         tcg_gen_and_i32(t3, t3, arg1);
910         tcg_gen_sub_i32(rh, t1, t2);
911         tcg_gen_sub_i32(rh, rh, t3);
912         tcg_gen_mov_i32(rl, t0);
913         tcg_temp_free_i32(t0);
914         tcg_temp_free_i32(t1);
915         tcg_temp_free_i32(t2);
916         tcg_temp_free_i32(t3);
917     } else {
918         TCGv_i64 t0 = tcg_temp_new_i64();
919         TCGv_i64 t1 = tcg_temp_new_i64();
920         tcg_gen_ext_i32_i64(t0, arg1);
921         tcg_gen_ext_i32_i64(t1, arg2);
922         tcg_gen_mul_i64(t0, t0, t1);
923         tcg_gen_extr_i64_i32(rl, rh, t0);
924         tcg_temp_free_i64(t0);
925         tcg_temp_free_i64(t1);
926     }
927 }
928 
929 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
930 {
931     if (TCG_TARGET_REG_BITS == 32) {
932         TCGv_i32 t0 = tcg_temp_new_i32();
933         TCGv_i32 t1 = tcg_temp_new_i32();
934         TCGv_i32 t2 = tcg_temp_new_i32();
935         tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
936         /* Adjust for negative input for the signed arg1.  */
937         tcg_gen_sari_i32(t2, arg1, 31);
938         tcg_gen_and_i32(t2, t2, arg2);
939         tcg_gen_sub_i32(rh, t1, t2);
940         tcg_gen_mov_i32(rl, t0);
941         tcg_temp_free_i32(t0);
942         tcg_temp_free_i32(t1);
943         tcg_temp_free_i32(t2);
944     } else {
945         TCGv_i64 t0 = tcg_temp_new_i64();
946         TCGv_i64 t1 = tcg_temp_new_i64();
947         tcg_gen_ext_i32_i64(t0, arg1);
948         tcg_gen_extu_i32_i64(t1, arg2);
949         tcg_gen_mul_i64(t0, t0, t1);
950         tcg_gen_extr_i64_i32(rl, rh, t0);
951         tcg_temp_free_i64(t0);
952         tcg_temp_free_i64(t1);
953     }
954 }
955 
956 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
957 {
958     if (TCG_TARGET_HAS_ext8s_i32) {
959         tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
960     } else {
961         tcg_gen_shli_i32(ret, arg, 24);
962         tcg_gen_sari_i32(ret, ret, 24);
963     }
964 }
965 
966 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
967 {
968     if (TCG_TARGET_HAS_ext16s_i32) {
969         tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
970     } else {
971         tcg_gen_shli_i32(ret, arg, 16);
972         tcg_gen_sari_i32(ret, ret, 16);
973     }
974 }
975 
976 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
977 {
978     if (TCG_TARGET_HAS_ext8u_i32) {
979         tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
980     } else {
981         tcg_gen_andi_i32(ret, arg, 0xffu);
982     }
983 }
984 
985 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
986 {
987     if (TCG_TARGET_HAS_ext16u_i32) {
988         tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
989     } else {
990         tcg_gen_andi_i32(ret, arg, 0xffffu);
991     }
992 }
993 
994 /* Note: we assume the two high bytes are set to zero */
995 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg)
996 {
997     if (TCG_TARGET_HAS_bswap16_i32) {
998         tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg);
999     } else {
1000         TCGv_i32 t0 = tcg_temp_new_i32();
1001 
1002         tcg_gen_ext8u_i32(t0, arg);
1003         tcg_gen_shli_i32(t0, t0, 8);
1004         tcg_gen_shri_i32(ret, arg, 8);
1005         tcg_gen_or_i32(ret, ret, t0);
1006         tcg_temp_free_i32(t0);
1007     }
1008 }
1009 
1010 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1011 {
1012     if (TCG_TARGET_HAS_bswap32_i32) {
1013         tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg);
1014     } else {
1015         TCGv_i32 t0, t1;
1016         t0 = tcg_temp_new_i32();
1017         t1 = tcg_temp_new_i32();
1018 
1019         tcg_gen_shli_i32(t0, arg, 24);
1020 
1021         tcg_gen_andi_i32(t1, arg, 0x0000ff00);
1022         tcg_gen_shli_i32(t1, t1, 8);
1023         tcg_gen_or_i32(t0, t0, t1);
1024 
1025         tcg_gen_shri_i32(t1, arg, 8);
1026         tcg_gen_andi_i32(t1, t1, 0x0000ff00);
1027         tcg_gen_or_i32(t0, t0, t1);
1028 
1029         tcg_gen_shri_i32(t1, arg, 24);
1030         tcg_gen_or_i32(ret, t0, t1);
1031         tcg_temp_free_i32(t0);
1032         tcg_temp_free_i32(t1);
1033     }
1034 }
1035 
1036 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1037 {
1038     tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1039 }
1040 
1041 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1042 {
1043     tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1044 }
1045 
1046 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1047 {
1048     tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1049 }
1050 
1051 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1052 {
1053     tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1054 }
1055 
1056 /* 64-bit ops */
1057 
1058 #if TCG_TARGET_REG_BITS == 32
1059 /* These are all inline for TCG_TARGET_REG_BITS == 64.  */
1060 
1061 void tcg_gen_discard_i64(TCGv_i64 arg)
1062 {
1063     tcg_gen_discard_i32(TCGV_LOW(arg));
1064     tcg_gen_discard_i32(TCGV_HIGH(arg));
1065 }
1066 
1067 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1068 {
1069     tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1070     tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1071 }
1072 
1073 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1074 {
1075     tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1076     tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1077 }
1078 
1079 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1080 {
1081     tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1082     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1083 }
1084 
1085 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1086 {
1087     tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
1088     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1089 }
1090 
1091 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1092 {
1093     tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1094     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1095 }
1096 
1097 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1098 {
1099     tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1100     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1101 }
1102 
1103 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1104 {
1105     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1106     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1107 }
1108 
1109 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1110 {
1111     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1112     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1113 }
1114 
1115 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1116 {
1117     /* Since arg2 and ret have different types,
1118        they cannot be the same temporary */
1119 #ifdef HOST_WORDS_BIGENDIAN
1120     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1121     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1122 #else
1123     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1124     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1125 #endif
1126 }
1127 
1128 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1129 {
1130 #ifdef HOST_WORDS_BIGENDIAN
1131     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1132     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1133 #else
1134     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1135     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1136 #endif
1137 }
1138 
1139 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1140 {
1141     tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1142     tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1143 }
1144 
1145 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1146 {
1147     tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1148     tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1149 }
1150 
1151 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1152 {
1153     tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1154     tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1155 }
1156 
1157 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1158 {
1159     gen_helper_shl_i64(ret, arg1, arg2);
1160 }
1161 
1162 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1163 {
1164     gen_helper_shr_i64(ret, arg1, arg2);
1165 }
1166 
1167 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1168 {
1169     gen_helper_sar_i64(ret, arg1, arg2);
1170 }
1171 
1172 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1173 {
1174     TCGv_i64 t0;
1175     TCGv_i32 t1;
1176 
1177     t0 = tcg_temp_new_i64();
1178     t1 = tcg_temp_new_i32();
1179 
1180     tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1181                       TCGV_LOW(arg1), TCGV_LOW(arg2));
1182 
1183     tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1184     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1185     tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1186     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1187 
1188     tcg_gen_mov_i64(ret, t0);
1189     tcg_temp_free_i64(t0);
1190     tcg_temp_free_i32(t1);
1191 }
1192 #endif /* TCG_TARGET_REG_SIZE == 32 */
1193 
1194 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1195 {
1196     /* some cases can be optimized here */
1197     if (arg2 == 0) {
1198         tcg_gen_mov_i64(ret, arg1);
1199     } else {
1200         TCGv_i64 t0 = tcg_const_i64(arg2);
1201         tcg_gen_add_i64(ret, arg1, t0);
1202         tcg_temp_free_i64(t0);
1203     }
1204 }
1205 
1206 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1207 {
1208     if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1209         /* Don't recurse with tcg_gen_neg_i64.  */
1210         tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1211     } else {
1212         TCGv_i64 t0 = tcg_const_i64(arg1);
1213         tcg_gen_sub_i64(ret, t0, arg2);
1214         tcg_temp_free_i64(t0);
1215     }
1216 }
1217 
1218 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1219 {
1220     /* some cases can be optimized here */
1221     if (arg2 == 0) {
1222         tcg_gen_mov_i64(ret, arg1);
1223     } else {
1224         TCGv_i64 t0 = tcg_const_i64(arg2);
1225         tcg_gen_sub_i64(ret, arg1, t0);
1226         tcg_temp_free_i64(t0);
1227     }
1228 }
1229 
1230 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1231 {
1232     TCGv_i64 t0;
1233 
1234     if (TCG_TARGET_REG_BITS == 32) {
1235         tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1236         tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1237         return;
1238     }
1239 
1240     /* Some cases can be optimized here.  */
1241     switch (arg2) {
1242     case 0:
1243         tcg_gen_movi_i64(ret, 0);
1244         return;
1245     case -1:
1246         tcg_gen_mov_i64(ret, arg1);
1247         return;
1248     case 0xff:
1249         /* Don't recurse with tcg_gen_ext8u_i64.  */
1250         if (TCG_TARGET_HAS_ext8u_i64) {
1251             tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1252             return;
1253         }
1254         break;
1255     case 0xffff:
1256         if (TCG_TARGET_HAS_ext16u_i64) {
1257             tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1258             return;
1259         }
1260         break;
1261     case 0xffffffffu:
1262         if (TCG_TARGET_HAS_ext32u_i64) {
1263             tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1264             return;
1265         }
1266         break;
1267     }
1268     t0 = tcg_const_i64(arg2);
1269     tcg_gen_and_i64(ret, arg1, t0);
1270     tcg_temp_free_i64(t0);
1271 }
1272 
1273 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1274 {
1275     if (TCG_TARGET_REG_BITS == 32) {
1276         tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1277         tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1278         return;
1279     }
1280     /* Some cases can be optimized here.  */
1281     if (arg2 == -1) {
1282         tcg_gen_movi_i64(ret, -1);
1283     } else if (arg2 == 0) {
1284         tcg_gen_mov_i64(ret, arg1);
1285     } else {
1286         TCGv_i64 t0 = tcg_const_i64(arg2);
1287         tcg_gen_or_i64(ret, arg1, t0);
1288         tcg_temp_free_i64(t0);
1289     }
1290 }
1291 
1292 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1293 {
1294     if (TCG_TARGET_REG_BITS == 32) {
1295         tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1296         tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1297         return;
1298     }
1299     /* Some cases can be optimized here.  */
1300     if (arg2 == 0) {
1301         tcg_gen_mov_i64(ret, arg1);
1302     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1303         /* Don't recurse with tcg_gen_not_i64.  */
1304         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1305     } else {
1306         TCGv_i64 t0 = tcg_const_i64(arg2);
1307         tcg_gen_xor_i64(ret, arg1, t0);
1308         tcg_temp_free_i64(t0);
1309     }
1310 }
1311 
1312 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1313                                       unsigned c, bool right, bool arith)
1314 {
1315     tcg_debug_assert(c < 64);
1316     if (c == 0) {
1317         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1318         tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1319     } else if (c >= 32) {
1320         c -= 32;
1321         if (right) {
1322             if (arith) {
1323                 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1324                 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1325             } else {
1326                 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1327                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1328             }
1329         } else {
1330             tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1331             tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1332         }
1333     } else {
1334         TCGv_i32 t0, t1;
1335 
1336         t0 = tcg_temp_new_i32();
1337         t1 = tcg_temp_new_i32();
1338         if (right) {
1339             tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c);
1340             if (arith) {
1341                 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c);
1342             } else {
1343                 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c);
1344             }
1345             tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1346             tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0);
1347             tcg_gen_mov_i32(TCGV_HIGH(ret), t1);
1348         } else {
1349             tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1350             /* Note: ret can be the same as arg1, so we use t1 */
1351             tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c);
1352             tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1353             tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0);
1354             tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1355         }
1356         tcg_temp_free_i32(t0);
1357         tcg_temp_free_i32(t1);
1358     }
1359 }
1360 
1361 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1362 {
1363     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1364     if (TCG_TARGET_REG_BITS == 32) {
1365         tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1366     } else if (arg2 == 0) {
1367         tcg_gen_mov_i64(ret, arg1);
1368     } else {
1369         TCGv_i64 t0 = tcg_const_i64(arg2);
1370         tcg_gen_shl_i64(ret, arg1, t0);
1371         tcg_temp_free_i64(t0);
1372     }
1373 }
1374 
1375 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1376 {
1377     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1378     if (TCG_TARGET_REG_BITS == 32) {
1379         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1380     } else if (arg2 == 0) {
1381         tcg_gen_mov_i64(ret, arg1);
1382     } else {
1383         TCGv_i64 t0 = tcg_const_i64(arg2);
1384         tcg_gen_shr_i64(ret, arg1, t0);
1385         tcg_temp_free_i64(t0);
1386     }
1387 }
1388 
1389 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1390 {
1391     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1392     if (TCG_TARGET_REG_BITS == 32) {
1393         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1394     } else if (arg2 == 0) {
1395         tcg_gen_mov_i64(ret, arg1);
1396     } else {
1397         TCGv_i64 t0 = tcg_const_i64(arg2);
1398         tcg_gen_sar_i64(ret, arg1, t0);
1399         tcg_temp_free_i64(t0);
1400     }
1401 }
1402 
1403 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1404 {
1405     if (cond == TCG_COND_ALWAYS) {
1406         tcg_gen_br(l);
1407     } else if (cond != TCG_COND_NEVER) {
1408         if (TCG_TARGET_REG_BITS == 32) {
1409             tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1410                               TCGV_HIGH(arg1), TCGV_LOW(arg2),
1411                               TCGV_HIGH(arg2), cond, label_arg(l));
1412         } else {
1413             tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1414                               label_arg(l));
1415         }
1416     }
1417 }
1418 
1419 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1420 {
1421     if (cond == TCG_COND_ALWAYS) {
1422         tcg_gen_br(l);
1423     } else if (cond != TCG_COND_NEVER) {
1424         TCGv_i64 t0 = tcg_const_i64(arg2);
1425         tcg_gen_brcond_i64(cond, arg1, t0, l);
1426         tcg_temp_free_i64(t0);
1427     }
1428 }
1429 
1430 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1431                          TCGv_i64 arg1, TCGv_i64 arg2)
1432 {
1433     if (cond == TCG_COND_ALWAYS) {
1434         tcg_gen_movi_i64(ret, 1);
1435     } else if (cond == TCG_COND_NEVER) {
1436         tcg_gen_movi_i64(ret, 0);
1437     } else {
1438         if (TCG_TARGET_REG_BITS == 32) {
1439             tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1440                              TCGV_LOW(arg1), TCGV_HIGH(arg1),
1441                              TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1442             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1443         } else {
1444             tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1445         }
1446     }
1447 }
1448 
1449 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1450                           TCGv_i64 arg1, int64_t arg2)
1451 {
1452     TCGv_i64 t0 = tcg_const_i64(arg2);
1453     tcg_gen_setcond_i64(cond, ret, arg1, t0);
1454     tcg_temp_free_i64(t0);
1455 }
1456 
1457 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1458 {
1459     if (arg2 == 0) {
1460         tcg_gen_movi_i64(ret, 0);
1461     } else if (is_power_of_2(arg2)) {
1462         tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1463     } else {
1464         TCGv_i64 t0 = tcg_const_i64(arg2);
1465         tcg_gen_mul_i64(ret, arg1, t0);
1466         tcg_temp_free_i64(t0);
1467     }
1468 }
1469 
1470 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1471 {
1472     if (TCG_TARGET_HAS_div_i64) {
1473         tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1474     } else if (TCG_TARGET_HAS_div2_i64) {
1475         TCGv_i64 t0 = tcg_temp_new_i64();
1476         tcg_gen_sari_i64(t0, arg1, 63);
1477         tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1478         tcg_temp_free_i64(t0);
1479     } else {
1480         gen_helper_div_i64(ret, arg1, arg2);
1481     }
1482 }
1483 
1484 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1485 {
1486     if (TCG_TARGET_HAS_rem_i64) {
1487         tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1488     } else if (TCG_TARGET_HAS_div_i64) {
1489         TCGv_i64 t0 = tcg_temp_new_i64();
1490         tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1491         tcg_gen_mul_i64(t0, t0, arg2);
1492         tcg_gen_sub_i64(ret, arg1, t0);
1493         tcg_temp_free_i64(t0);
1494     } else if (TCG_TARGET_HAS_div2_i64) {
1495         TCGv_i64 t0 = tcg_temp_new_i64();
1496         tcg_gen_sari_i64(t0, arg1, 63);
1497         tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1498         tcg_temp_free_i64(t0);
1499     } else {
1500         gen_helper_rem_i64(ret, arg1, arg2);
1501     }
1502 }
1503 
1504 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1505 {
1506     if (TCG_TARGET_HAS_div_i64) {
1507         tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1508     } else if (TCG_TARGET_HAS_div2_i64) {
1509         TCGv_i64 t0 = tcg_temp_new_i64();
1510         tcg_gen_movi_i64(t0, 0);
1511         tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1512         tcg_temp_free_i64(t0);
1513     } else {
1514         gen_helper_divu_i64(ret, arg1, arg2);
1515     }
1516 }
1517 
1518 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1519 {
1520     if (TCG_TARGET_HAS_rem_i64) {
1521         tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1522     } else if (TCG_TARGET_HAS_div_i64) {
1523         TCGv_i64 t0 = tcg_temp_new_i64();
1524         tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1525         tcg_gen_mul_i64(t0, t0, arg2);
1526         tcg_gen_sub_i64(ret, arg1, t0);
1527         tcg_temp_free_i64(t0);
1528     } else if (TCG_TARGET_HAS_div2_i64) {
1529         TCGv_i64 t0 = tcg_temp_new_i64();
1530         tcg_gen_movi_i64(t0, 0);
1531         tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1532         tcg_temp_free_i64(t0);
1533     } else {
1534         gen_helper_remu_i64(ret, arg1, arg2);
1535     }
1536 }
1537 
1538 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1539 {
1540     if (TCG_TARGET_REG_BITS == 32) {
1541         tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1542         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1543     } else if (TCG_TARGET_HAS_ext8s_i64) {
1544         tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1545     } else {
1546         tcg_gen_shli_i64(ret, arg, 56);
1547         tcg_gen_sari_i64(ret, ret, 56);
1548     }
1549 }
1550 
1551 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1552 {
1553     if (TCG_TARGET_REG_BITS == 32) {
1554         tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1555         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1556     } else if (TCG_TARGET_HAS_ext16s_i64) {
1557         tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1558     } else {
1559         tcg_gen_shli_i64(ret, arg, 48);
1560         tcg_gen_sari_i64(ret, ret, 48);
1561     }
1562 }
1563 
1564 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1565 {
1566     if (TCG_TARGET_REG_BITS == 32) {
1567         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1568         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1569     } else if (TCG_TARGET_HAS_ext32s_i64) {
1570         tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1571     } else {
1572         tcg_gen_shli_i64(ret, arg, 32);
1573         tcg_gen_sari_i64(ret, ret, 32);
1574     }
1575 }
1576 
1577 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1578 {
1579     if (TCG_TARGET_REG_BITS == 32) {
1580         tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1581         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1582     } else if (TCG_TARGET_HAS_ext8u_i64) {
1583         tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1584     } else {
1585         tcg_gen_andi_i64(ret, arg, 0xffu);
1586     }
1587 }
1588 
1589 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1590 {
1591     if (TCG_TARGET_REG_BITS == 32) {
1592         tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1593         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1594     } else if (TCG_TARGET_HAS_ext16u_i64) {
1595         tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1596     } else {
1597         tcg_gen_andi_i64(ret, arg, 0xffffu);
1598     }
1599 }
1600 
1601 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1602 {
1603     if (TCG_TARGET_REG_BITS == 32) {
1604         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1605         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1606     } else if (TCG_TARGET_HAS_ext32u_i64) {
1607         tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1608     } else {
1609         tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1610     }
1611 }
1612 
1613 /* Note: we assume the six high bytes are set to zero */
1614 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg)
1615 {
1616     if (TCG_TARGET_REG_BITS == 32) {
1617         tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1618         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1619     } else if (TCG_TARGET_HAS_bswap16_i64) {
1620         tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg);
1621     } else {
1622         TCGv_i64 t0 = tcg_temp_new_i64();
1623 
1624         tcg_gen_ext8u_i64(t0, arg);
1625         tcg_gen_shli_i64(t0, t0, 8);
1626         tcg_gen_shri_i64(ret, arg, 8);
1627         tcg_gen_or_i64(ret, ret, t0);
1628         tcg_temp_free_i64(t0);
1629     }
1630 }
1631 
1632 /* Note: we assume the four high bytes are set to zero */
1633 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg)
1634 {
1635     if (TCG_TARGET_REG_BITS == 32) {
1636         tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1637         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1638     } else if (TCG_TARGET_HAS_bswap32_i64) {
1639         tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg);
1640     } else {
1641         TCGv_i64 t0, t1;
1642         t0 = tcg_temp_new_i64();
1643         t1 = tcg_temp_new_i64();
1644 
1645         tcg_gen_shli_i64(t0, arg, 24);
1646         tcg_gen_ext32u_i64(t0, t0);
1647 
1648         tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1649         tcg_gen_shli_i64(t1, t1, 8);
1650         tcg_gen_or_i64(t0, t0, t1);
1651 
1652         tcg_gen_shri_i64(t1, arg, 8);
1653         tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1654         tcg_gen_or_i64(t0, t0, t1);
1655 
1656         tcg_gen_shri_i64(t1, arg, 24);
1657         tcg_gen_or_i64(ret, t0, t1);
1658         tcg_temp_free_i64(t0);
1659         tcg_temp_free_i64(t1);
1660     }
1661 }
1662 
1663 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1664 {
1665     if (TCG_TARGET_REG_BITS == 32) {
1666         TCGv_i32 t0, t1;
1667         t0 = tcg_temp_new_i32();
1668         t1 = tcg_temp_new_i32();
1669 
1670         tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1671         tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1672         tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1673         tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1674         tcg_temp_free_i32(t0);
1675         tcg_temp_free_i32(t1);
1676     } else if (TCG_TARGET_HAS_bswap64_i64) {
1677         tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg);
1678     } else {
1679         TCGv_i64 t0 = tcg_temp_new_i64();
1680         TCGv_i64 t1 = tcg_temp_new_i64();
1681 
1682         tcg_gen_shli_i64(t0, arg, 56);
1683 
1684         tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1685         tcg_gen_shli_i64(t1, t1, 40);
1686         tcg_gen_or_i64(t0, t0, t1);
1687 
1688         tcg_gen_andi_i64(t1, arg, 0x00ff0000);
1689         tcg_gen_shli_i64(t1, t1, 24);
1690         tcg_gen_or_i64(t0, t0, t1);
1691 
1692         tcg_gen_andi_i64(t1, arg, 0xff000000);
1693         tcg_gen_shli_i64(t1, t1, 8);
1694         tcg_gen_or_i64(t0, t0, t1);
1695 
1696         tcg_gen_shri_i64(t1, arg, 8);
1697         tcg_gen_andi_i64(t1, t1, 0xff000000);
1698         tcg_gen_or_i64(t0, t0, t1);
1699 
1700         tcg_gen_shri_i64(t1, arg, 24);
1701         tcg_gen_andi_i64(t1, t1, 0x00ff0000);
1702         tcg_gen_or_i64(t0, t0, t1);
1703 
1704         tcg_gen_shri_i64(t1, arg, 40);
1705         tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1706         tcg_gen_or_i64(t0, t0, t1);
1707 
1708         tcg_gen_shri_i64(t1, arg, 56);
1709         tcg_gen_or_i64(ret, t0, t1);
1710         tcg_temp_free_i64(t0);
1711         tcg_temp_free_i64(t1);
1712     }
1713 }
1714 
1715 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1716 {
1717     if (TCG_TARGET_REG_BITS == 32) {
1718         tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1719         tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1720     } else if (TCG_TARGET_HAS_not_i64) {
1721         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1722     } else {
1723         tcg_gen_xori_i64(ret, arg, -1);
1724     }
1725 }
1726 
1727 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1728 {
1729     if (TCG_TARGET_REG_BITS == 32) {
1730         tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1731         tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1732     } else if (TCG_TARGET_HAS_andc_i64) {
1733         tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1734     } else {
1735         TCGv_i64 t0 = tcg_temp_new_i64();
1736         tcg_gen_not_i64(t0, arg2);
1737         tcg_gen_and_i64(ret, arg1, t0);
1738         tcg_temp_free_i64(t0);
1739     }
1740 }
1741 
1742 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1743 {
1744     if (TCG_TARGET_REG_BITS == 32) {
1745         tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1746         tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1747     } else if (TCG_TARGET_HAS_eqv_i64) {
1748         tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1749     } else {
1750         tcg_gen_xor_i64(ret, arg1, arg2);
1751         tcg_gen_not_i64(ret, ret);
1752     }
1753 }
1754 
1755 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1756 {
1757     if (TCG_TARGET_REG_BITS == 32) {
1758         tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1759         tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1760     } else if (TCG_TARGET_HAS_nand_i64) {
1761         tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1762     } else {
1763         tcg_gen_and_i64(ret, arg1, arg2);
1764         tcg_gen_not_i64(ret, ret);
1765     }
1766 }
1767 
1768 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1769 {
1770     if (TCG_TARGET_REG_BITS == 32) {
1771         tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1772         tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1773     } else if (TCG_TARGET_HAS_nor_i64) {
1774         tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1775     } else {
1776         tcg_gen_or_i64(ret, arg1, arg2);
1777         tcg_gen_not_i64(ret, ret);
1778     }
1779 }
1780 
1781 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1782 {
1783     if (TCG_TARGET_REG_BITS == 32) {
1784         tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1785         tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1786     } else if (TCG_TARGET_HAS_orc_i64) {
1787         tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1788     } else {
1789         TCGv_i64 t0 = tcg_temp_new_i64();
1790         tcg_gen_not_i64(t0, arg2);
1791         tcg_gen_or_i64(ret, arg1, t0);
1792         tcg_temp_free_i64(t0);
1793     }
1794 }
1795 
1796 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1797 {
1798     if (TCG_TARGET_HAS_clz_i64) {
1799         tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1800     } else {
1801         gen_helper_clz_i64(ret, arg1, arg2);
1802     }
1803 }
1804 
1805 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1806 {
1807     if (TCG_TARGET_REG_BITS == 32
1808         && TCG_TARGET_HAS_clz_i32
1809         && arg2 <= 0xffffffffu) {
1810         TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32);
1811         tcg_gen_clz_i32(t, TCGV_LOW(arg1), t);
1812         tcg_gen_addi_i32(t, t, 32);
1813         tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1814         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1815         tcg_temp_free_i32(t);
1816     } else {
1817         TCGv_i64 t = tcg_const_i64(arg2);
1818         tcg_gen_clz_i64(ret, arg1, t);
1819         tcg_temp_free_i64(t);
1820     }
1821 }
1822 
1823 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1824 {
1825     if (TCG_TARGET_HAS_ctz_i64) {
1826         tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
1827     } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1828         TCGv_i64 z, t = tcg_temp_new_i64();
1829 
1830         if (TCG_TARGET_HAS_ctpop_i64) {
1831             tcg_gen_subi_i64(t, arg1, 1);
1832             tcg_gen_andc_i64(t, t, arg1);
1833             tcg_gen_ctpop_i64(t, t);
1834         } else {
1835             /* Since all non-x86 hosts have clz(0) == 64, don't fight it.  */
1836             tcg_gen_neg_i64(t, arg1);
1837             tcg_gen_and_i64(t, t, arg1);
1838             tcg_gen_clzi_i64(t, t, 64);
1839             tcg_gen_xori_i64(t, t, 63);
1840         }
1841         z = tcg_const_i64(0);
1842         tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1843         tcg_temp_free_i64(t);
1844         tcg_temp_free_i64(z);
1845     } else {
1846         gen_helper_ctz_i64(ret, arg1, arg2);
1847     }
1848 }
1849 
1850 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1851 {
1852     if (TCG_TARGET_REG_BITS == 32
1853         && TCG_TARGET_HAS_ctz_i32
1854         && arg2 <= 0xffffffffu) {
1855         TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32);
1856         tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32);
1857         tcg_gen_addi_i32(t32, t32, 32);
1858         tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1859         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1860         tcg_temp_free_i32(t32);
1861     } else if (!TCG_TARGET_HAS_ctz_i64
1862                && TCG_TARGET_HAS_ctpop_i64
1863                && arg2 == 64) {
1864         /* This equivalence has the advantage of not requiring a fixup.  */
1865         TCGv_i64 t = tcg_temp_new_i64();
1866         tcg_gen_subi_i64(t, arg1, 1);
1867         tcg_gen_andc_i64(t, t, arg1);
1868         tcg_gen_ctpop_i64(ret, t);
1869         tcg_temp_free_i64(t);
1870     } else {
1871         TCGv_i64 t64 = tcg_const_i64(arg2);
1872         tcg_gen_ctz_i64(ret, arg1, t64);
1873         tcg_temp_free_i64(t64);
1874     }
1875 }
1876 
1877 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1878 {
1879     if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1880         TCGv_i64 t = tcg_temp_new_i64();
1881         tcg_gen_sari_i64(t, arg, 63);
1882         tcg_gen_xor_i64(t, t, arg);
1883         tcg_gen_clzi_i64(t, t, 64);
1884         tcg_gen_subi_i64(ret, t, 1);
1885         tcg_temp_free_i64(t);
1886     } else {
1887         gen_helper_clrsb_i64(ret, arg);
1888     }
1889 }
1890 
1891 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
1892 {
1893     if (TCG_TARGET_HAS_ctpop_i64) {
1894         tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
1895     } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
1896         tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1897         tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1898         tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
1899         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1900     } else {
1901         gen_helper_ctpop_i64(ret, arg1);
1902     }
1903 }
1904 
1905 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1906 {
1907     if (TCG_TARGET_HAS_rot_i64) {
1908         tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1909     } else {
1910         TCGv_i64 t0, t1;
1911         t0 = tcg_temp_new_i64();
1912         t1 = tcg_temp_new_i64();
1913         tcg_gen_shl_i64(t0, arg1, arg2);
1914         tcg_gen_subfi_i64(t1, 64, arg2);
1915         tcg_gen_shr_i64(t1, arg1, t1);
1916         tcg_gen_or_i64(ret, t0, t1);
1917         tcg_temp_free_i64(t0);
1918         tcg_temp_free_i64(t1);
1919     }
1920 }
1921 
1922 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1923 {
1924     tcg_debug_assert(arg2 < 64);
1925     /* some cases can be optimized here */
1926     if (arg2 == 0) {
1927         tcg_gen_mov_i64(ret, arg1);
1928     } else if (TCG_TARGET_HAS_rot_i64) {
1929         TCGv_i64 t0 = tcg_const_i64(arg2);
1930         tcg_gen_rotl_i64(ret, arg1, t0);
1931         tcg_temp_free_i64(t0);
1932     } else {
1933         TCGv_i64 t0, t1;
1934         t0 = tcg_temp_new_i64();
1935         t1 = tcg_temp_new_i64();
1936         tcg_gen_shli_i64(t0, arg1, arg2);
1937         tcg_gen_shri_i64(t1, arg1, 64 - arg2);
1938         tcg_gen_or_i64(ret, t0, t1);
1939         tcg_temp_free_i64(t0);
1940         tcg_temp_free_i64(t1);
1941     }
1942 }
1943 
1944 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1945 {
1946     if (TCG_TARGET_HAS_rot_i64) {
1947         tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
1948     } else {
1949         TCGv_i64 t0, t1;
1950         t0 = tcg_temp_new_i64();
1951         t1 = tcg_temp_new_i64();
1952         tcg_gen_shr_i64(t0, arg1, arg2);
1953         tcg_gen_subfi_i64(t1, 64, arg2);
1954         tcg_gen_shl_i64(t1, arg1, t1);
1955         tcg_gen_or_i64(ret, t0, t1);
1956         tcg_temp_free_i64(t0);
1957         tcg_temp_free_i64(t1);
1958     }
1959 }
1960 
1961 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1962 {
1963     tcg_debug_assert(arg2 < 64);
1964     /* some cases can be optimized here */
1965     if (arg2 == 0) {
1966         tcg_gen_mov_i64(ret, arg1);
1967     } else {
1968         tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
1969     }
1970 }
1971 
1972 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
1973                          unsigned int ofs, unsigned int len)
1974 {
1975     uint64_t mask;
1976     TCGv_i64 t1;
1977 
1978     tcg_debug_assert(ofs < 64);
1979     tcg_debug_assert(len > 0);
1980     tcg_debug_assert(len <= 64);
1981     tcg_debug_assert(ofs + len <= 64);
1982 
1983     if (len == 64) {
1984         tcg_gen_mov_i64(ret, arg2);
1985         return;
1986     }
1987     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
1988         tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
1989         return;
1990     }
1991 
1992     if (TCG_TARGET_REG_BITS == 32) {
1993         if (ofs >= 32) {
1994             tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
1995                                 TCGV_LOW(arg2), ofs - 32, len);
1996             tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1997             return;
1998         }
1999         if (ofs + len <= 32) {
2000             tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
2001                                 TCGV_LOW(arg2), ofs, len);
2002             tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2003             return;
2004         }
2005     }
2006 
2007     mask = (1ull << len) - 1;
2008     t1 = tcg_temp_new_i64();
2009 
2010     if (ofs + len < 64) {
2011         tcg_gen_andi_i64(t1, arg2, mask);
2012         tcg_gen_shli_i64(t1, t1, ofs);
2013     } else {
2014         tcg_gen_shli_i64(t1, arg2, ofs);
2015     }
2016     tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2017     tcg_gen_or_i64(ret, ret, t1);
2018 
2019     tcg_temp_free_i64(t1);
2020 }
2021 
2022 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2023                            unsigned int ofs, unsigned int len)
2024 {
2025     tcg_debug_assert(ofs < 64);
2026     tcg_debug_assert(len > 0);
2027     tcg_debug_assert(len <= 64);
2028     tcg_debug_assert(ofs + len <= 64);
2029 
2030     if (ofs + len == 64) {
2031         tcg_gen_shli_i64(ret, arg, ofs);
2032     } else if (ofs == 0) {
2033         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2034     } else if (TCG_TARGET_HAS_deposit_i64
2035                && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2036         TCGv_i64 zero = tcg_const_i64(0);
2037         tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2038         tcg_temp_free_i64(zero);
2039     } else {
2040         if (TCG_TARGET_REG_BITS == 32) {
2041             if (ofs >= 32) {
2042                 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2043                                       ofs - 32, len);
2044                 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2045                 return;
2046             }
2047             if (ofs + len <= 32) {
2048                 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2049                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2050                 return;
2051             }
2052         }
2053         /* To help two-operand hosts we prefer to zero-extend first,
2054            which allows ARG to stay live.  */
2055         switch (len) {
2056         case 32:
2057             if (TCG_TARGET_HAS_ext32u_i64) {
2058                 tcg_gen_ext32u_i64(ret, arg);
2059                 tcg_gen_shli_i64(ret, ret, ofs);
2060                 return;
2061             }
2062             break;
2063         case 16:
2064             if (TCG_TARGET_HAS_ext16u_i64) {
2065                 tcg_gen_ext16u_i64(ret, arg);
2066                 tcg_gen_shli_i64(ret, ret, ofs);
2067                 return;
2068             }
2069             break;
2070         case 8:
2071             if (TCG_TARGET_HAS_ext8u_i64) {
2072                 tcg_gen_ext8u_i64(ret, arg);
2073                 tcg_gen_shli_i64(ret, ret, ofs);
2074                 return;
2075             }
2076             break;
2077         }
2078         /* Otherwise prefer zero-extension over AND for code size.  */
2079         switch (ofs + len) {
2080         case 32:
2081             if (TCG_TARGET_HAS_ext32u_i64) {
2082                 tcg_gen_shli_i64(ret, arg, ofs);
2083                 tcg_gen_ext32u_i64(ret, ret);
2084                 return;
2085             }
2086             break;
2087         case 16:
2088             if (TCG_TARGET_HAS_ext16u_i64) {
2089                 tcg_gen_shli_i64(ret, arg, ofs);
2090                 tcg_gen_ext16u_i64(ret, ret);
2091                 return;
2092             }
2093             break;
2094         case 8:
2095             if (TCG_TARGET_HAS_ext8u_i64) {
2096                 tcg_gen_shli_i64(ret, arg, ofs);
2097                 tcg_gen_ext8u_i64(ret, ret);
2098                 return;
2099             }
2100             break;
2101         }
2102         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2103         tcg_gen_shli_i64(ret, ret, ofs);
2104     }
2105 }
2106 
2107 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2108                          unsigned int ofs, unsigned int len)
2109 {
2110     tcg_debug_assert(ofs < 64);
2111     tcg_debug_assert(len > 0);
2112     tcg_debug_assert(len <= 64);
2113     tcg_debug_assert(ofs + len <= 64);
2114 
2115     /* Canonicalize certain special cases, even if extract is supported.  */
2116     if (ofs + len == 64) {
2117         tcg_gen_shri_i64(ret, arg, 64 - len);
2118         return;
2119     }
2120     if (ofs == 0) {
2121         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2122         return;
2123     }
2124 
2125     if (TCG_TARGET_REG_BITS == 32) {
2126         /* Look for a 32-bit extract within one of the two words.  */
2127         if (ofs >= 32) {
2128             tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2129             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2130             return;
2131         }
2132         if (ofs + len <= 32) {
2133             tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2134             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2135             return;
2136         }
2137         /* The field is split across two words.  One double-word
2138            shift is better than two double-word shifts.  */
2139         goto do_shift_and;
2140     }
2141 
2142     if (TCG_TARGET_HAS_extract_i64
2143         && TCG_TARGET_extract_i64_valid(ofs, len)) {
2144         tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2145         return;
2146     }
2147 
2148     /* Assume that zero-extension, if available, is cheaper than a shift.  */
2149     switch (ofs + len) {
2150     case 32:
2151         if (TCG_TARGET_HAS_ext32u_i64) {
2152             tcg_gen_ext32u_i64(ret, arg);
2153             tcg_gen_shri_i64(ret, ret, ofs);
2154             return;
2155         }
2156         break;
2157     case 16:
2158         if (TCG_TARGET_HAS_ext16u_i64) {
2159             tcg_gen_ext16u_i64(ret, arg);
2160             tcg_gen_shri_i64(ret, ret, ofs);
2161             return;
2162         }
2163         break;
2164     case 8:
2165         if (TCG_TARGET_HAS_ext8u_i64) {
2166             tcg_gen_ext8u_i64(ret, arg);
2167             tcg_gen_shri_i64(ret, ret, ofs);
2168             return;
2169         }
2170         break;
2171     }
2172 
2173     /* ??? Ideally we'd know what values are available for immediate AND.
2174        Assume that 8 bits are available, plus the special cases of 16 and 32,
2175        so that we get ext8u, ext16u, and ext32u.  */
2176     switch (len) {
2177     case 1 ... 8: case 16: case 32:
2178     do_shift_and:
2179         tcg_gen_shri_i64(ret, arg, ofs);
2180         tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2181         break;
2182     default:
2183         tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2184         tcg_gen_shri_i64(ret, ret, 64 - len);
2185         break;
2186     }
2187 }
2188 
2189 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2190                           unsigned int ofs, unsigned int len)
2191 {
2192     tcg_debug_assert(ofs < 64);
2193     tcg_debug_assert(len > 0);
2194     tcg_debug_assert(len <= 64);
2195     tcg_debug_assert(ofs + len <= 64);
2196 
2197     /* Canonicalize certain special cases, even if sextract is supported.  */
2198     if (ofs + len == 64) {
2199         tcg_gen_sari_i64(ret, arg, 64 - len);
2200         return;
2201     }
2202     if (ofs == 0) {
2203         switch (len) {
2204         case 32:
2205             tcg_gen_ext32s_i64(ret, arg);
2206             return;
2207         case 16:
2208             tcg_gen_ext16s_i64(ret, arg);
2209             return;
2210         case 8:
2211             tcg_gen_ext8s_i64(ret, arg);
2212             return;
2213         }
2214     }
2215 
2216     if (TCG_TARGET_REG_BITS == 32) {
2217         /* Look for a 32-bit extract within one of the two words.  */
2218         if (ofs >= 32) {
2219             tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2220         } else if (ofs + len <= 32) {
2221             tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2222         } else if (ofs == 0) {
2223             tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2224             tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2225             return;
2226         } else if (len > 32) {
2227             TCGv_i32 t = tcg_temp_new_i32();
2228             /* Extract the bits for the high word normally.  */
2229             tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2230             /* Shift the field down for the low part.  */
2231             tcg_gen_shri_i64(ret, arg, ofs);
2232             /* Overwrite the shift into the high part.  */
2233             tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2234             tcg_temp_free_i32(t);
2235             return;
2236         } else {
2237             /* Shift the field down for the low part, such that the
2238                field sits at the MSB.  */
2239             tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2240             /* Shift the field down from the MSB, sign extending.  */
2241             tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2242         }
2243         /* Sign-extend the field from 32 bits.  */
2244         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2245         return;
2246     }
2247 
2248     if (TCG_TARGET_HAS_sextract_i64
2249         && TCG_TARGET_extract_i64_valid(ofs, len)) {
2250         tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2251         return;
2252     }
2253 
2254     /* Assume that sign-extension, if available, is cheaper than a shift.  */
2255     switch (ofs + len) {
2256     case 32:
2257         if (TCG_TARGET_HAS_ext32s_i64) {
2258             tcg_gen_ext32s_i64(ret, arg);
2259             tcg_gen_sari_i64(ret, ret, ofs);
2260             return;
2261         }
2262         break;
2263     case 16:
2264         if (TCG_TARGET_HAS_ext16s_i64) {
2265             tcg_gen_ext16s_i64(ret, arg);
2266             tcg_gen_sari_i64(ret, ret, ofs);
2267             return;
2268         }
2269         break;
2270     case 8:
2271         if (TCG_TARGET_HAS_ext8s_i64) {
2272             tcg_gen_ext8s_i64(ret, arg);
2273             tcg_gen_sari_i64(ret, ret, ofs);
2274             return;
2275         }
2276         break;
2277     }
2278     switch (len) {
2279     case 32:
2280         if (TCG_TARGET_HAS_ext32s_i64) {
2281             tcg_gen_shri_i64(ret, arg, ofs);
2282             tcg_gen_ext32s_i64(ret, ret);
2283             return;
2284         }
2285         break;
2286     case 16:
2287         if (TCG_TARGET_HAS_ext16s_i64) {
2288             tcg_gen_shri_i64(ret, arg, ofs);
2289             tcg_gen_ext16s_i64(ret, ret);
2290             return;
2291         }
2292         break;
2293     case 8:
2294         if (TCG_TARGET_HAS_ext8s_i64) {
2295             tcg_gen_shri_i64(ret, arg, ofs);
2296             tcg_gen_ext8s_i64(ret, ret);
2297             return;
2298         }
2299         break;
2300     }
2301     tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2302     tcg_gen_sari_i64(ret, ret, 64 - len);
2303 }
2304 
2305 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2306                          TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2307 {
2308     if (cond == TCG_COND_ALWAYS) {
2309         tcg_gen_mov_i64(ret, v1);
2310     } else if (cond == TCG_COND_NEVER) {
2311         tcg_gen_mov_i64(ret, v2);
2312     } else if (TCG_TARGET_REG_BITS == 32) {
2313         TCGv_i32 t0 = tcg_temp_new_i32();
2314         TCGv_i32 t1 = tcg_temp_new_i32();
2315         tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2316                          TCGV_LOW(c1), TCGV_HIGH(c1),
2317                          TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2318 
2319         if (TCG_TARGET_HAS_movcond_i32) {
2320             tcg_gen_movi_i32(t1, 0);
2321             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2322                                 TCGV_LOW(v1), TCGV_LOW(v2));
2323             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2324                                 TCGV_HIGH(v1), TCGV_HIGH(v2));
2325         } else {
2326             tcg_gen_neg_i32(t0, t0);
2327 
2328             tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2329             tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2330             tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
2331 
2332             tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2333             tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2334             tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2335         }
2336         tcg_temp_free_i32(t0);
2337         tcg_temp_free_i32(t1);
2338     } else if (TCG_TARGET_HAS_movcond_i64) {
2339         tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2340     } else {
2341         TCGv_i64 t0 = tcg_temp_new_i64();
2342         TCGv_i64 t1 = tcg_temp_new_i64();
2343         tcg_gen_setcond_i64(cond, t0, c1, c2);
2344         tcg_gen_neg_i64(t0, t0);
2345         tcg_gen_and_i64(t1, v1, t0);
2346         tcg_gen_andc_i64(ret, v2, t0);
2347         tcg_gen_or_i64(ret, ret, t1);
2348         tcg_temp_free_i64(t0);
2349         tcg_temp_free_i64(t1);
2350     }
2351 }
2352 
2353 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2354                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2355 {
2356     if (TCG_TARGET_HAS_add2_i64) {
2357         tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
2358     } else {
2359         TCGv_i64 t0 = tcg_temp_new_i64();
2360         TCGv_i64 t1 = tcg_temp_new_i64();
2361         tcg_gen_add_i64(t0, al, bl);
2362         tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2363         tcg_gen_add_i64(rh, ah, bh);
2364         tcg_gen_add_i64(rh, rh, t1);
2365         tcg_gen_mov_i64(rl, t0);
2366         tcg_temp_free_i64(t0);
2367         tcg_temp_free_i64(t1);
2368     }
2369 }
2370 
2371 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2372                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2373 {
2374     if (TCG_TARGET_HAS_sub2_i64) {
2375         tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
2376     } else {
2377         TCGv_i64 t0 = tcg_temp_new_i64();
2378         TCGv_i64 t1 = tcg_temp_new_i64();
2379         tcg_gen_sub_i64(t0, al, bl);
2380         tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2381         tcg_gen_sub_i64(rh, ah, bh);
2382         tcg_gen_sub_i64(rh, rh, t1);
2383         tcg_gen_mov_i64(rl, t0);
2384         tcg_temp_free_i64(t0);
2385         tcg_temp_free_i64(t1);
2386     }
2387 }
2388 
2389 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2390 {
2391     if (TCG_TARGET_HAS_mulu2_i64) {
2392         tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
2393     } else if (TCG_TARGET_HAS_muluh_i64) {
2394         TCGv_i64 t = tcg_temp_new_i64();
2395         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2396         tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2397         tcg_gen_mov_i64(rl, t);
2398         tcg_temp_free_i64(t);
2399     } else {
2400         TCGv_i64 t0 = tcg_temp_new_i64();
2401         tcg_gen_mul_i64(t0, arg1, arg2);
2402         gen_helper_muluh_i64(rh, arg1, arg2);
2403         tcg_gen_mov_i64(rl, t0);
2404         tcg_temp_free_i64(t0);
2405     }
2406 }
2407 
2408 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2409 {
2410     if (TCG_TARGET_HAS_muls2_i64) {
2411         tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
2412     } else if (TCG_TARGET_HAS_mulsh_i64) {
2413         TCGv_i64 t = tcg_temp_new_i64();
2414         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2415         tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2416         tcg_gen_mov_i64(rl, t);
2417         tcg_temp_free_i64(t);
2418     } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2419         TCGv_i64 t0 = tcg_temp_new_i64();
2420         TCGv_i64 t1 = tcg_temp_new_i64();
2421         TCGv_i64 t2 = tcg_temp_new_i64();
2422         TCGv_i64 t3 = tcg_temp_new_i64();
2423         tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2424         /* Adjust for negative inputs.  */
2425         tcg_gen_sari_i64(t2, arg1, 63);
2426         tcg_gen_sari_i64(t3, arg2, 63);
2427         tcg_gen_and_i64(t2, t2, arg2);
2428         tcg_gen_and_i64(t3, t3, arg1);
2429         tcg_gen_sub_i64(rh, t1, t2);
2430         tcg_gen_sub_i64(rh, rh, t3);
2431         tcg_gen_mov_i64(rl, t0);
2432         tcg_temp_free_i64(t0);
2433         tcg_temp_free_i64(t1);
2434         tcg_temp_free_i64(t2);
2435         tcg_temp_free_i64(t3);
2436     } else {
2437         TCGv_i64 t0 = tcg_temp_new_i64();
2438         tcg_gen_mul_i64(t0, arg1, arg2);
2439         gen_helper_mulsh_i64(rh, arg1, arg2);
2440         tcg_gen_mov_i64(rl, t0);
2441         tcg_temp_free_i64(t0);
2442     }
2443 }
2444 
2445 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2446 {
2447     TCGv_i64 t0 = tcg_temp_new_i64();
2448     TCGv_i64 t1 = tcg_temp_new_i64();
2449     TCGv_i64 t2 = tcg_temp_new_i64();
2450     tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2451     /* Adjust for negative input for the signed arg1.  */
2452     tcg_gen_sari_i64(t2, arg1, 63);
2453     tcg_gen_and_i64(t2, t2, arg2);
2454     tcg_gen_sub_i64(rh, t1, t2);
2455     tcg_gen_mov_i64(rl, t0);
2456     tcg_temp_free_i64(t0);
2457     tcg_temp_free_i64(t1);
2458     tcg_temp_free_i64(t2);
2459 }
2460 
2461 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2462 {
2463     tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2464 }
2465 
2466 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2467 {
2468     tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2469 }
2470 
2471 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2472 {
2473     tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2474 }
2475 
2476 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2477 {
2478     tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2479 }
2480 
2481 /* Size changing operations.  */
2482 
2483 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2484 {
2485     if (TCG_TARGET_REG_BITS == 32) {
2486         tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2487     } else if (TCG_TARGET_HAS_extrl_i64_i32) {
2488         tcg_gen_op2(INDEX_op_extrl_i64_i32,
2489                     tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2490     } else {
2491         tcg_gen_mov_i32(ret, (TCGv_i32)arg);
2492     }
2493 }
2494 
2495 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2496 {
2497     if (TCG_TARGET_REG_BITS == 32) {
2498         tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2499     } else if (TCG_TARGET_HAS_extrh_i64_i32) {
2500         tcg_gen_op2(INDEX_op_extrh_i64_i32,
2501                     tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2502     } else {
2503         TCGv_i64 t = tcg_temp_new_i64();
2504         tcg_gen_shri_i64(t, arg, 32);
2505         tcg_gen_mov_i32(ret, (TCGv_i32)t);
2506         tcg_temp_free_i64(t);
2507     }
2508 }
2509 
2510 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2511 {
2512     if (TCG_TARGET_REG_BITS == 32) {
2513         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2514         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2515     } else {
2516         tcg_gen_op2(INDEX_op_extu_i32_i64,
2517                     tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2518     }
2519 }
2520 
2521 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2522 {
2523     if (TCG_TARGET_REG_BITS == 32) {
2524         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2525         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2526     } else {
2527         tcg_gen_op2(INDEX_op_ext_i32_i64,
2528                     tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2529     }
2530 }
2531 
2532 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2533 {
2534     TCGv_i64 tmp;
2535 
2536     if (TCG_TARGET_REG_BITS == 32) {
2537         tcg_gen_mov_i32(TCGV_LOW(dest), low);
2538         tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2539         return;
2540     }
2541 
2542     tmp = tcg_temp_new_i64();
2543     /* These extensions are only needed for type correctness.
2544        We may be able to do better given target specific information.  */
2545     tcg_gen_extu_i32_i64(tmp, high);
2546     tcg_gen_extu_i32_i64(dest, low);
2547     /* If deposit is available, use it.  Otherwise use the extra
2548        knowledge that we have of the zero-extensions above.  */
2549     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2550         tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2551     } else {
2552         tcg_gen_shli_i64(tmp, tmp, 32);
2553         tcg_gen_or_i64(dest, dest, tmp);
2554     }
2555     tcg_temp_free_i64(tmp);
2556 }
2557 
2558 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2559 {
2560     if (TCG_TARGET_REG_BITS == 32) {
2561         tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2562         tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2563     } else {
2564         tcg_gen_extrl_i64_i32(lo, arg);
2565         tcg_gen_extrh_i64_i32(hi, arg);
2566     }
2567 }
2568 
2569 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2570 {
2571     tcg_gen_ext32u_i64(lo, arg);
2572     tcg_gen_shri_i64(hi, arg, 32);
2573 }
2574 
2575 /* QEMU specific operations.  */
2576 
2577 void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx)
2578 {
2579     uintptr_t val = (uintptr_t)tb + idx;
2580 
2581     if (tb == NULL) {
2582         tcg_debug_assert(idx == 0);
2583     } else if (idx <= TB_EXIT_IDXMAX) {
2584 #ifdef CONFIG_DEBUG_TCG
2585         /* This is an exit following a goto_tb.  Verify that we have
2586            seen this numbered exit before, via tcg_gen_goto_tb.  */
2587         tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2588 #endif
2589         /* When not chaining, exit without indicating a link.  */
2590         if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2591             val = 0;
2592         }
2593     } else {
2594         /* This is an exit via the exitreq label.  */
2595         tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2596     }
2597 
2598     tcg_gen_op1i(INDEX_op_exit_tb, val);
2599 }
2600 
2601 void tcg_gen_goto_tb(unsigned idx)
2602 {
2603     /* We only support two chained exits.  */
2604     tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
2605 #ifdef CONFIG_DEBUG_TCG
2606     /* Verify that we havn't seen this numbered exit before.  */
2607     tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2608     tcg_ctx->goto_tb_issue_mask |= 1 << idx;
2609 #endif
2610     /* When not chaining, we simply fall through to the "fallback" exit.  */
2611     if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2612         tcg_gen_op1i(INDEX_op_goto_tb, idx);
2613     }
2614 }
2615 
2616 void tcg_gen_lookup_and_goto_ptr(void)
2617 {
2618     if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2619         TCGv_ptr ptr = tcg_temp_new_ptr();
2620         gen_helper_lookup_tb_ptr(ptr, cpu_env);
2621         tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
2622         tcg_temp_free_ptr(ptr);
2623     } else {
2624         tcg_gen_exit_tb(NULL, 0);
2625     }
2626 }
2627 
2628 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)
2629 {
2630     /* Trigger the asserts within as early as possible.  */
2631     (void)get_alignment_bits(op);
2632 
2633     switch (op & MO_SIZE) {
2634     case MO_8:
2635         op &= ~MO_BSWAP;
2636         break;
2637     case MO_16:
2638         break;
2639     case MO_32:
2640         if (!is64) {
2641             op &= ~MO_SIGN;
2642         }
2643         break;
2644     case MO_64:
2645         if (!is64) {
2646             tcg_abort();
2647         }
2648         break;
2649     }
2650     if (st) {
2651         op &= ~MO_SIGN;
2652     }
2653     return op;
2654 }
2655 
2656 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2657                          TCGMemOp memop, TCGArg idx)
2658 {
2659     TCGMemOpIdx oi = make_memop_idx(memop, idx);
2660 #if TARGET_LONG_BITS == 32
2661     tcg_gen_op3i_i32(opc, val, addr, oi);
2662 #else
2663     if (TCG_TARGET_REG_BITS == 32) {
2664         tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2665     } else {
2666         tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
2667     }
2668 #endif
2669 }
2670 
2671 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2672                          TCGMemOp memop, TCGArg idx)
2673 {
2674     TCGMemOpIdx oi = make_memop_idx(memop, idx);
2675 #if TARGET_LONG_BITS == 32
2676     if (TCG_TARGET_REG_BITS == 32) {
2677         tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
2678     } else {
2679         tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
2680     }
2681 #else
2682     if (TCG_TARGET_REG_BITS == 32) {
2683         tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2684                          TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2685     } else {
2686         tcg_gen_op3i_i64(opc, val, addr, oi);
2687     }
2688 #endif
2689 }
2690 
2691 static void tcg_gen_req_mo(TCGBar type)
2692 {
2693 #ifdef TCG_GUEST_DEFAULT_MO
2694     type &= TCG_GUEST_DEFAULT_MO;
2695 #endif
2696     type &= ~TCG_TARGET_DEFAULT_MO;
2697     if (type) {
2698         tcg_gen_mb(type | TCG_BAR_SC);
2699     }
2700 }
2701 
2702 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2703 {
2704     tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2705     memop = tcg_canonicalize_memop(memop, 0, 0);
2706     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2707                                addr, trace_mem_get_info(memop, 0));
2708     gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
2709 }
2710 
2711 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2712 {
2713     tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2714     memop = tcg_canonicalize_memop(memop, 0, 1);
2715     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2716                                addr, trace_mem_get_info(memop, 1));
2717     gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
2718 }
2719 
2720 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2721 {
2722     tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2723     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2724         tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2725         if (memop & MO_SIGN) {
2726             tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2727         } else {
2728             tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2729         }
2730         return;
2731     }
2732 
2733     memop = tcg_canonicalize_memop(memop, 1, 0);
2734     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2735                                addr, trace_mem_get_info(memop, 0));
2736     gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
2737 }
2738 
2739 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2740 {
2741     tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2742     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2743         tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
2744         return;
2745     }
2746 
2747     memop = tcg_canonicalize_memop(memop, 1, 1);
2748     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2749                                addr, trace_mem_get_info(memop, 1));
2750     gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
2751 }
2752 
2753 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc)
2754 {
2755     switch (opc & MO_SSIZE) {
2756     case MO_SB:
2757         tcg_gen_ext8s_i32(ret, val);
2758         break;
2759     case MO_UB:
2760         tcg_gen_ext8u_i32(ret, val);
2761         break;
2762     case MO_SW:
2763         tcg_gen_ext16s_i32(ret, val);
2764         break;
2765     case MO_UW:
2766         tcg_gen_ext16u_i32(ret, val);
2767         break;
2768     default:
2769         tcg_gen_mov_i32(ret, val);
2770         break;
2771     }
2772 }
2773 
2774 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc)
2775 {
2776     switch (opc & MO_SSIZE) {
2777     case MO_SB:
2778         tcg_gen_ext8s_i64(ret, val);
2779         break;
2780     case MO_UB:
2781         tcg_gen_ext8u_i64(ret, val);
2782         break;
2783     case MO_SW:
2784         tcg_gen_ext16s_i64(ret, val);
2785         break;
2786     case MO_UW:
2787         tcg_gen_ext16u_i64(ret, val);
2788         break;
2789     case MO_SL:
2790         tcg_gen_ext32s_i64(ret, val);
2791         break;
2792     case MO_UL:
2793         tcg_gen_ext32u_i64(ret, val);
2794         break;
2795     default:
2796         tcg_gen_mov_i64(ret, val);
2797         break;
2798     }
2799 }
2800 
2801 #ifdef CONFIG_SOFTMMU
2802 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
2803                                   TCGv_i32, TCGv_i32, TCGv_i32);
2804 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
2805                                   TCGv_i64, TCGv_i64, TCGv_i32);
2806 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
2807                                   TCGv_i32, TCGv_i32);
2808 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
2809                                   TCGv_i64, TCGv_i32);
2810 #else
2811 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
2812 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
2813 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
2814 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
2815 #endif
2816 
2817 #ifdef CONFIG_ATOMIC64
2818 # define WITH_ATOMIC64(X) X,
2819 #else
2820 # define WITH_ATOMIC64(X)
2821 #endif
2822 
2823 static void * const table_cmpxchg[16] = {
2824     [MO_8] = gen_helper_atomic_cmpxchgb,
2825     [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
2826     [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
2827     [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
2828     [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
2829     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
2830     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
2831 };
2832 
2833 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
2834                                 TCGv_i32 newv, TCGArg idx, TCGMemOp memop)
2835 {
2836     memop = tcg_canonicalize_memop(memop, 0, 0);
2837 
2838     if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
2839         TCGv_i32 t1 = tcg_temp_new_i32();
2840         TCGv_i32 t2 = tcg_temp_new_i32();
2841 
2842         tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
2843 
2844         tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2845         tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
2846         tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2847         tcg_temp_free_i32(t2);
2848 
2849         if (memop & MO_SIGN) {
2850             tcg_gen_ext_i32(retv, t1, memop);
2851         } else {
2852             tcg_gen_mov_i32(retv, t1);
2853         }
2854         tcg_temp_free_i32(t1);
2855     } else {
2856         gen_atomic_cx_i32 gen;
2857 
2858         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2859         tcg_debug_assert(gen != NULL);
2860 
2861 #ifdef CONFIG_SOFTMMU
2862         {
2863             TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2864             gen(retv, cpu_env, addr, cmpv, newv, oi);
2865             tcg_temp_free_i32(oi);
2866         }
2867 #else
2868         gen(retv, cpu_env, addr, cmpv, newv);
2869 #endif
2870 
2871         if (memop & MO_SIGN) {
2872             tcg_gen_ext_i32(retv, retv, memop);
2873         }
2874     }
2875 }
2876 
2877 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
2878                                 TCGv_i64 newv, TCGArg idx, TCGMemOp memop)
2879 {
2880     memop = tcg_canonicalize_memop(memop, 1, 0);
2881 
2882     if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
2883         TCGv_i64 t1 = tcg_temp_new_i64();
2884         TCGv_i64 t2 = tcg_temp_new_i64();
2885 
2886         tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
2887 
2888         tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2889         tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
2890         tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2891         tcg_temp_free_i64(t2);
2892 
2893         if (memop & MO_SIGN) {
2894             tcg_gen_ext_i64(retv, t1, memop);
2895         } else {
2896             tcg_gen_mov_i64(retv, t1);
2897         }
2898         tcg_temp_free_i64(t1);
2899     } else if ((memop & MO_SIZE) == MO_64) {
2900 #ifdef CONFIG_ATOMIC64
2901         gen_atomic_cx_i64 gen;
2902 
2903         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2904         tcg_debug_assert(gen != NULL);
2905 
2906 #ifdef CONFIG_SOFTMMU
2907         {
2908             TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx));
2909             gen(retv, cpu_env, addr, cmpv, newv, oi);
2910             tcg_temp_free_i32(oi);
2911         }
2912 #else
2913         gen(retv, cpu_env, addr, cmpv, newv);
2914 #endif
2915 #else
2916         gen_helper_exit_atomic(cpu_env);
2917         /* Produce a result, so that we have a well-formed opcode stream
2918            with respect to uses of the result in the (dead) code following.  */
2919         tcg_gen_movi_i64(retv, 0);
2920 #endif /* CONFIG_ATOMIC64 */
2921     } else {
2922         TCGv_i32 c32 = tcg_temp_new_i32();
2923         TCGv_i32 n32 = tcg_temp_new_i32();
2924         TCGv_i32 r32 = tcg_temp_new_i32();
2925 
2926         tcg_gen_extrl_i64_i32(c32, cmpv);
2927         tcg_gen_extrl_i64_i32(n32, newv);
2928         tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
2929         tcg_temp_free_i32(c32);
2930         tcg_temp_free_i32(n32);
2931 
2932         tcg_gen_extu_i32_i64(retv, r32);
2933         tcg_temp_free_i32(r32);
2934 
2935         if (memop & MO_SIGN) {
2936             tcg_gen_ext_i64(retv, retv, memop);
2937         }
2938     }
2939 }
2940 
2941 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2942                                 TCGArg idx, TCGMemOp memop, bool new_val,
2943                                 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
2944 {
2945     TCGv_i32 t1 = tcg_temp_new_i32();
2946     TCGv_i32 t2 = tcg_temp_new_i32();
2947 
2948     memop = tcg_canonicalize_memop(memop, 0, 0);
2949 
2950     tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2951     gen(t2, t1, val);
2952     tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2953 
2954     tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
2955     tcg_temp_free_i32(t1);
2956     tcg_temp_free_i32(t2);
2957 }
2958 
2959 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2960                              TCGArg idx, TCGMemOp memop, void * const table[])
2961 {
2962     gen_atomic_op_i32 gen;
2963 
2964     memop = tcg_canonicalize_memop(memop, 0, 0);
2965 
2966     gen = table[memop & (MO_SIZE | MO_BSWAP)];
2967     tcg_debug_assert(gen != NULL);
2968 
2969 #ifdef CONFIG_SOFTMMU
2970     {
2971         TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2972         gen(ret, cpu_env, addr, val, oi);
2973         tcg_temp_free_i32(oi);
2974     }
2975 #else
2976     gen(ret, cpu_env, addr, val);
2977 #endif
2978 
2979     if (memop & MO_SIGN) {
2980         tcg_gen_ext_i32(ret, ret, memop);
2981     }
2982 }
2983 
2984 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
2985                                 TCGArg idx, TCGMemOp memop, bool new_val,
2986                                 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
2987 {
2988     TCGv_i64 t1 = tcg_temp_new_i64();
2989     TCGv_i64 t2 = tcg_temp_new_i64();
2990 
2991     memop = tcg_canonicalize_memop(memop, 1, 0);
2992 
2993     tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2994     gen(t2, t1, val);
2995     tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2996 
2997     tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
2998     tcg_temp_free_i64(t1);
2999     tcg_temp_free_i64(t2);
3000 }
3001 
3002 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3003                              TCGArg idx, TCGMemOp memop, void * const table[])
3004 {
3005     memop = tcg_canonicalize_memop(memop, 1, 0);
3006 
3007     if ((memop & MO_SIZE) == MO_64) {
3008 #ifdef CONFIG_ATOMIC64
3009         gen_atomic_op_i64 gen;
3010 
3011         gen = table[memop & (MO_SIZE | MO_BSWAP)];
3012         tcg_debug_assert(gen != NULL);
3013 
3014 #ifdef CONFIG_SOFTMMU
3015         {
3016             TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
3017             gen(ret, cpu_env, addr, val, oi);
3018             tcg_temp_free_i32(oi);
3019         }
3020 #else
3021         gen(ret, cpu_env, addr, val);
3022 #endif
3023 #else
3024         gen_helper_exit_atomic(cpu_env);
3025         /* Produce a result, so that we have a well-formed opcode stream
3026            with respect to uses of the result in the (dead) code following.  */
3027         tcg_gen_movi_i64(ret, 0);
3028 #endif /* CONFIG_ATOMIC64 */
3029     } else {
3030         TCGv_i32 v32 = tcg_temp_new_i32();
3031         TCGv_i32 r32 = tcg_temp_new_i32();
3032 
3033         tcg_gen_extrl_i64_i32(v32, val);
3034         do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3035         tcg_temp_free_i32(v32);
3036 
3037         tcg_gen_extu_i32_i64(ret, r32);
3038         tcg_temp_free_i32(r32);
3039 
3040         if (memop & MO_SIGN) {
3041             tcg_gen_ext_i64(ret, ret, memop);
3042         }
3043     }
3044 }
3045 
3046 #define GEN_ATOMIC_HELPER(NAME, OP, NEW)                                \
3047 static void * const table_##NAME[16] = {                                \
3048     [MO_8] = gen_helper_atomic_##NAME##b,                               \
3049     [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le,                   \
3050     [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be,                   \
3051     [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le,                   \
3052     [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be,                   \
3053     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le)     \
3054     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be)     \
3055 };                                                                      \
3056 void tcg_gen_atomic_##NAME##_i32                                        \
3057     (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
3058 {                                                                       \
3059     if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
3060         do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME);     \
3061     } else {                                                            \
3062         do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW,            \
3063                             tcg_gen_##OP##_i32);                        \
3064     }                                                                   \
3065 }                                                                       \
3066 void tcg_gen_atomic_##NAME##_i64                                        \
3067     (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \
3068 {                                                                       \
3069     if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
3070         do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME);     \
3071     } else {                                                            \
3072         do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW,            \
3073                             tcg_gen_##OP##_i64);                        \
3074     }                                                                   \
3075 }
3076 
3077 GEN_ATOMIC_HELPER(fetch_add, add, 0)
3078 GEN_ATOMIC_HELPER(fetch_and, and, 0)
3079 GEN_ATOMIC_HELPER(fetch_or, or, 0)
3080 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
3081 GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3082 GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3083 GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3084 GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
3085 
3086 GEN_ATOMIC_HELPER(add_fetch, add, 1)
3087 GEN_ATOMIC_HELPER(and_fetch, and, 1)
3088 GEN_ATOMIC_HELPER(or_fetch, or, 1)
3089 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
3090 GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3091 GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3092 GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3093 GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
3094 
3095 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3096 {
3097     tcg_gen_mov_i32(r, b);
3098 }
3099 
3100 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3101 {
3102     tcg_gen_mov_i64(r, b);
3103 }
3104 
3105 GEN_ATOMIC_HELPER(xchg, mov2, 0)
3106 
3107 #undef GEN_ATOMIC_HELPER
3108