1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "qemu-common.h" 27 #include "cpu.h" 28 #include "exec/exec-all.h" 29 #include "tcg.h" 30 #include "tcg-op.h" 31 #include "tcg-mo.h" 32 #include "trace-tcg.h" 33 #include "trace/mem.h" 34 35 /* Reduce the number of ifdefs below. This assumes that all uses of 36 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that 37 the compiler can eliminate. */ 38 #if TCG_TARGET_REG_BITS == 64 39 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); 40 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); 41 #define TCGV_LOW TCGV_LOW_link_error 42 #define TCGV_HIGH TCGV_HIGH_link_error 43 #endif 44 45 void tcg_gen_op1(TCGOpcode opc, TCGArg a1) 46 { 47 TCGOp *op = tcg_emit_op(opc); 48 op->args[0] = a1; 49 } 50 51 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2) 52 { 53 TCGOp *op = tcg_emit_op(opc); 54 op->args[0] = a1; 55 op->args[1] = a2; 56 } 57 58 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) 59 { 60 TCGOp *op = tcg_emit_op(opc); 61 op->args[0] = a1; 62 op->args[1] = a2; 63 op->args[2] = a3; 64 } 65 66 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4) 67 { 68 TCGOp *op = tcg_emit_op(opc); 69 op->args[0] = a1; 70 op->args[1] = a2; 71 op->args[2] = a3; 72 op->args[3] = a4; 73 } 74 75 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 76 TCGArg a4, TCGArg a5) 77 { 78 TCGOp *op = tcg_emit_op(opc); 79 op->args[0] = a1; 80 op->args[1] = a2; 81 op->args[2] = a3; 82 op->args[3] = a4; 83 op->args[4] = a5; 84 } 85 86 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 87 TCGArg a4, TCGArg a5, TCGArg a6) 88 { 89 TCGOp *op = tcg_emit_op(opc); 90 op->args[0] = a1; 91 op->args[1] = a2; 92 op->args[2] = a3; 93 op->args[3] = a4; 94 op->args[4] = a5; 95 op->args[5] = a6; 96 } 97 98 void tcg_gen_mb(TCGBar mb_type) 99 { 100 if (tcg_ctx->tb_cflags & CF_PARALLEL) { 101 tcg_gen_op1(INDEX_op_mb, mb_type); 102 } 103 } 104 105 /* 32 bit ops */ 106 107 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 108 { 109 /* some cases can be optimized here */ 110 if (arg2 == 0) { 111 tcg_gen_mov_i32(ret, arg1); 112 } else { 113 TCGv_i32 t0 = tcg_const_i32(arg2); 114 tcg_gen_add_i32(ret, arg1, t0); 115 tcg_temp_free_i32(t0); 116 } 117 } 118 119 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 120 { 121 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) { 122 /* Don't recurse with tcg_gen_neg_i32. */ 123 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2); 124 } else { 125 TCGv_i32 t0 = tcg_const_i32(arg1); 126 tcg_gen_sub_i32(ret, t0, arg2); 127 tcg_temp_free_i32(t0); 128 } 129 } 130 131 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 132 { 133 /* some cases can be optimized here */ 134 if (arg2 == 0) { 135 tcg_gen_mov_i32(ret, arg1); 136 } else { 137 TCGv_i32 t0 = tcg_const_i32(arg2); 138 tcg_gen_sub_i32(ret, arg1, t0); 139 tcg_temp_free_i32(t0); 140 } 141 } 142 143 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 144 { 145 TCGv_i32 t0; 146 /* Some cases can be optimized here. */ 147 switch (arg2) { 148 case 0: 149 tcg_gen_movi_i32(ret, 0); 150 return; 151 case -1: 152 tcg_gen_mov_i32(ret, arg1); 153 return; 154 case 0xff: 155 /* Don't recurse with tcg_gen_ext8u_i32. */ 156 if (TCG_TARGET_HAS_ext8u_i32) { 157 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1); 158 return; 159 } 160 break; 161 case 0xffff: 162 if (TCG_TARGET_HAS_ext16u_i32) { 163 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1); 164 return; 165 } 166 break; 167 } 168 t0 = tcg_const_i32(arg2); 169 tcg_gen_and_i32(ret, arg1, t0); 170 tcg_temp_free_i32(t0); 171 } 172 173 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 174 { 175 /* Some cases can be optimized here. */ 176 if (arg2 == -1) { 177 tcg_gen_movi_i32(ret, -1); 178 } else if (arg2 == 0) { 179 tcg_gen_mov_i32(ret, arg1); 180 } else { 181 TCGv_i32 t0 = tcg_const_i32(arg2); 182 tcg_gen_or_i32(ret, arg1, t0); 183 tcg_temp_free_i32(t0); 184 } 185 } 186 187 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 188 { 189 /* Some cases can be optimized here. */ 190 if (arg2 == 0) { 191 tcg_gen_mov_i32(ret, arg1); 192 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) { 193 /* Don't recurse with tcg_gen_not_i32. */ 194 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1); 195 } else { 196 TCGv_i32 t0 = tcg_const_i32(arg2); 197 tcg_gen_xor_i32(ret, arg1, t0); 198 tcg_temp_free_i32(t0); 199 } 200 } 201 202 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 203 { 204 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 205 if (arg2 == 0) { 206 tcg_gen_mov_i32(ret, arg1); 207 } else { 208 TCGv_i32 t0 = tcg_const_i32(arg2); 209 tcg_gen_shl_i32(ret, arg1, t0); 210 tcg_temp_free_i32(t0); 211 } 212 } 213 214 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 215 { 216 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 217 if (arg2 == 0) { 218 tcg_gen_mov_i32(ret, arg1); 219 } else { 220 TCGv_i32 t0 = tcg_const_i32(arg2); 221 tcg_gen_shr_i32(ret, arg1, t0); 222 tcg_temp_free_i32(t0); 223 } 224 } 225 226 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 227 { 228 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 229 if (arg2 == 0) { 230 tcg_gen_mov_i32(ret, arg1); 231 } else { 232 TCGv_i32 t0 = tcg_const_i32(arg2); 233 tcg_gen_sar_i32(ret, arg1, t0); 234 tcg_temp_free_i32(t0); 235 } 236 } 237 238 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 239 { 240 if (cond == TCG_COND_ALWAYS) { 241 tcg_gen_br(l); 242 } else if (cond != TCG_COND_NEVER) { 243 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l)); 244 } 245 } 246 247 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 248 { 249 if (cond == TCG_COND_ALWAYS) { 250 tcg_gen_br(l); 251 } else if (cond != TCG_COND_NEVER) { 252 TCGv_i32 t0 = tcg_const_i32(arg2); 253 tcg_gen_brcond_i32(cond, arg1, t0, l); 254 tcg_temp_free_i32(t0); 255 } 256 } 257 258 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 259 TCGv_i32 arg1, TCGv_i32 arg2) 260 { 261 if (cond == TCG_COND_ALWAYS) { 262 tcg_gen_movi_i32(ret, 1); 263 } else if (cond == TCG_COND_NEVER) { 264 tcg_gen_movi_i32(ret, 0); 265 } else { 266 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 267 } 268 } 269 270 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 271 TCGv_i32 arg1, int32_t arg2) 272 { 273 TCGv_i32 t0 = tcg_const_i32(arg2); 274 tcg_gen_setcond_i32(cond, ret, arg1, t0); 275 tcg_temp_free_i32(t0); 276 } 277 278 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 279 { 280 if (arg2 == 0) { 281 tcg_gen_movi_i32(ret, 0); 282 } else if (is_power_of_2(arg2)) { 283 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 284 } else { 285 TCGv_i32 t0 = tcg_const_i32(arg2); 286 tcg_gen_mul_i32(ret, arg1, t0); 287 tcg_temp_free_i32(t0); 288 } 289 } 290 291 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 292 { 293 if (TCG_TARGET_HAS_div_i32) { 294 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2); 295 } else if (TCG_TARGET_HAS_div2_i32) { 296 TCGv_i32 t0 = tcg_temp_new_i32(); 297 tcg_gen_sari_i32(t0, arg1, 31); 298 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2); 299 tcg_temp_free_i32(t0); 300 } else { 301 gen_helper_div_i32(ret, arg1, arg2); 302 } 303 } 304 305 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 306 { 307 if (TCG_TARGET_HAS_rem_i32) { 308 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2); 309 } else if (TCG_TARGET_HAS_div_i32) { 310 TCGv_i32 t0 = tcg_temp_new_i32(); 311 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2); 312 tcg_gen_mul_i32(t0, t0, arg2); 313 tcg_gen_sub_i32(ret, arg1, t0); 314 tcg_temp_free_i32(t0); 315 } else if (TCG_TARGET_HAS_div2_i32) { 316 TCGv_i32 t0 = tcg_temp_new_i32(); 317 tcg_gen_sari_i32(t0, arg1, 31); 318 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2); 319 tcg_temp_free_i32(t0); 320 } else { 321 gen_helper_rem_i32(ret, arg1, arg2); 322 } 323 } 324 325 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 326 { 327 if (TCG_TARGET_HAS_div_i32) { 328 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2); 329 } else if (TCG_TARGET_HAS_div2_i32) { 330 TCGv_i32 t0 = tcg_temp_new_i32(); 331 tcg_gen_movi_i32(t0, 0); 332 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2); 333 tcg_temp_free_i32(t0); 334 } else { 335 gen_helper_divu_i32(ret, arg1, arg2); 336 } 337 } 338 339 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 340 { 341 if (TCG_TARGET_HAS_rem_i32) { 342 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2); 343 } else if (TCG_TARGET_HAS_div_i32) { 344 TCGv_i32 t0 = tcg_temp_new_i32(); 345 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2); 346 tcg_gen_mul_i32(t0, t0, arg2); 347 tcg_gen_sub_i32(ret, arg1, t0); 348 tcg_temp_free_i32(t0); 349 } else if (TCG_TARGET_HAS_div2_i32) { 350 TCGv_i32 t0 = tcg_temp_new_i32(); 351 tcg_gen_movi_i32(t0, 0); 352 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2); 353 tcg_temp_free_i32(t0); 354 } else { 355 gen_helper_remu_i32(ret, arg1, arg2); 356 } 357 } 358 359 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 360 { 361 if (TCG_TARGET_HAS_andc_i32) { 362 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2); 363 } else { 364 TCGv_i32 t0 = tcg_temp_new_i32(); 365 tcg_gen_not_i32(t0, arg2); 366 tcg_gen_and_i32(ret, arg1, t0); 367 tcg_temp_free_i32(t0); 368 } 369 } 370 371 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 372 { 373 if (TCG_TARGET_HAS_eqv_i32) { 374 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2); 375 } else { 376 tcg_gen_xor_i32(ret, arg1, arg2); 377 tcg_gen_not_i32(ret, ret); 378 } 379 } 380 381 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 382 { 383 if (TCG_TARGET_HAS_nand_i32) { 384 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2); 385 } else { 386 tcg_gen_and_i32(ret, arg1, arg2); 387 tcg_gen_not_i32(ret, ret); 388 } 389 } 390 391 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 392 { 393 if (TCG_TARGET_HAS_nor_i32) { 394 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2); 395 } else { 396 tcg_gen_or_i32(ret, arg1, arg2); 397 tcg_gen_not_i32(ret, ret); 398 } 399 } 400 401 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 402 { 403 if (TCG_TARGET_HAS_orc_i32) { 404 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2); 405 } else { 406 TCGv_i32 t0 = tcg_temp_new_i32(); 407 tcg_gen_not_i32(t0, arg2); 408 tcg_gen_or_i32(ret, arg1, t0); 409 tcg_temp_free_i32(t0); 410 } 411 } 412 413 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 414 { 415 if (TCG_TARGET_HAS_clz_i32) { 416 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2); 417 } else if (TCG_TARGET_HAS_clz_i64) { 418 TCGv_i64 t1 = tcg_temp_new_i64(); 419 TCGv_i64 t2 = tcg_temp_new_i64(); 420 tcg_gen_extu_i32_i64(t1, arg1); 421 tcg_gen_extu_i32_i64(t2, arg2); 422 tcg_gen_addi_i64(t2, t2, 32); 423 tcg_gen_clz_i64(t1, t1, t2); 424 tcg_gen_extrl_i64_i32(ret, t1); 425 tcg_temp_free_i64(t1); 426 tcg_temp_free_i64(t2); 427 tcg_gen_subi_i32(ret, ret, 32); 428 } else { 429 gen_helper_clz_i32(ret, arg1, arg2); 430 } 431 } 432 433 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 434 { 435 TCGv_i32 t = tcg_const_i32(arg2); 436 tcg_gen_clz_i32(ret, arg1, t); 437 tcg_temp_free_i32(t); 438 } 439 440 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 441 { 442 if (TCG_TARGET_HAS_ctz_i32) { 443 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); 444 } else if (TCG_TARGET_HAS_ctz_i64) { 445 TCGv_i64 t1 = tcg_temp_new_i64(); 446 TCGv_i64 t2 = tcg_temp_new_i64(); 447 tcg_gen_extu_i32_i64(t1, arg1); 448 tcg_gen_extu_i32_i64(t2, arg2); 449 tcg_gen_ctz_i64(t1, t1, t2); 450 tcg_gen_extrl_i64_i32(ret, t1); 451 tcg_temp_free_i64(t1); 452 tcg_temp_free_i64(t2); 453 } else if (TCG_TARGET_HAS_ctpop_i32 454 || TCG_TARGET_HAS_ctpop_i64 455 || TCG_TARGET_HAS_clz_i32 456 || TCG_TARGET_HAS_clz_i64) { 457 TCGv_i32 z, t = tcg_temp_new_i32(); 458 459 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 460 tcg_gen_subi_i32(t, arg1, 1); 461 tcg_gen_andc_i32(t, t, arg1); 462 tcg_gen_ctpop_i32(t, t); 463 } else { 464 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */ 465 tcg_gen_neg_i32(t, arg1); 466 tcg_gen_and_i32(t, t, arg1); 467 tcg_gen_clzi_i32(t, t, 32); 468 tcg_gen_xori_i32(t, t, 31); 469 } 470 z = tcg_const_i32(0); 471 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 472 tcg_temp_free_i32(t); 473 tcg_temp_free_i32(z); 474 } else { 475 gen_helper_ctz_i32(ret, arg1, arg2); 476 } 477 } 478 479 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 480 { 481 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 482 /* This equivalence has the advantage of not requiring a fixup. */ 483 TCGv_i32 t = tcg_temp_new_i32(); 484 tcg_gen_subi_i32(t, arg1, 1); 485 tcg_gen_andc_i32(t, t, arg1); 486 tcg_gen_ctpop_i32(ret, t); 487 tcg_temp_free_i32(t); 488 } else { 489 TCGv_i32 t = tcg_const_i32(arg2); 490 tcg_gen_ctz_i32(ret, arg1, t); 491 tcg_temp_free_i32(t); 492 } 493 } 494 495 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 496 { 497 if (TCG_TARGET_HAS_clz_i32) { 498 TCGv_i32 t = tcg_temp_new_i32(); 499 tcg_gen_sari_i32(t, arg, 31); 500 tcg_gen_xor_i32(t, t, arg); 501 tcg_gen_clzi_i32(t, t, 32); 502 tcg_gen_subi_i32(ret, t, 1); 503 tcg_temp_free_i32(t); 504 } else { 505 gen_helper_clrsb_i32(ret, arg); 506 } 507 } 508 509 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 510 { 511 if (TCG_TARGET_HAS_ctpop_i32) { 512 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 513 } else if (TCG_TARGET_HAS_ctpop_i64) { 514 TCGv_i64 t = tcg_temp_new_i64(); 515 tcg_gen_extu_i32_i64(t, arg1); 516 tcg_gen_ctpop_i64(t, t); 517 tcg_gen_extrl_i64_i32(ret, t); 518 tcg_temp_free_i64(t); 519 } else { 520 gen_helper_ctpop_i32(ret, arg1); 521 } 522 } 523 524 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 525 { 526 if (TCG_TARGET_HAS_rot_i32) { 527 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2); 528 } else { 529 TCGv_i32 t0, t1; 530 531 t0 = tcg_temp_new_i32(); 532 t1 = tcg_temp_new_i32(); 533 tcg_gen_shl_i32(t0, arg1, arg2); 534 tcg_gen_subfi_i32(t1, 32, arg2); 535 tcg_gen_shr_i32(t1, arg1, t1); 536 tcg_gen_or_i32(ret, t0, t1); 537 tcg_temp_free_i32(t0); 538 tcg_temp_free_i32(t1); 539 } 540 } 541 542 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 543 { 544 tcg_debug_assert(arg2 < 32); 545 /* some cases can be optimized here */ 546 if (arg2 == 0) { 547 tcg_gen_mov_i32(ret, arg1); 548 } else if (TCG_TARGET_HAS_rot_i32) { 549 TCGv_i32 t0 = tcg_const_i32(arg2); 550 tcg_gen_rotl_i32(ret, arg1, t0); 551 tcg_temp_free_i32(t0); 552 } else { 553 TCGv_i32 t0, t1; 554 t0 = tcg_temp_new_i32(); 555 t1 = tcg_temp_new_i32(); 556 tcg_gen_shli_i32(t0, arg1, arg2); 557 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 558 tcg_gen_or_i32(ret, t0, t1); 559 tcg_temp_free_i32(t0); 560 tcg_temp_free_i32(t1); 561 } 562 } 563 564 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 565 { 566 if (TCG_TARGET_HAS_rot_i32) { 567 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2); 568 } else { 569 TCGv_i32 t0, t1; 570 571 t0 = tcg_temp_new_i32(); 572 t1 = tcg_temp_new_i32(); 573 tcg_gen_shr_i32(t0, arg1, arg2); 574 tcg_gen_subfi_i32(t1, 32, arg2); 575 tcg_gen_shl_i32(t1, arg1, t1); 576 tcg_gen_or_i32(ret, t0, t1); 577 tcg_temp_free_i32(t0); 578 tcg_temp_free_i32(t1); 579 } 580 } 581 582 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 583 { 584 tcg_debug_assert(arg2 < 32); 585 /* some cases can be optimized here */ 586 if (arg2 == 0) { 587 tcg_gen_mov_i32(ret, arg1); 588 } else { 589 tcg_gen_rotli_i32(ret, arg1, 32 - arg2); 590 } 591 } 592 593 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 594 unsigned int ofs, unsigned int len) 595 { 596 uint32_t mask; 597 TCGv_i32 t1; 598 599 tcg_debug_assert(ofs < 32); 600 tcg_debug_assert(len > 0); 601 tcg_debug_assert(len <= 32); 602 tcg_debug_assert(ofs + len <= 32); 603 604 if (len == 32) { 605 tcg_gen_mov_i32(ret, arg2); 606 return; 607 } 608 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 609 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 610 return; 611 } 612 613 mask = (1u << len) - 1; 614 t1 = tcg_temp_new_i32(); 615 616 if (ofs + len < 32) { 617 tcg_gen_andi_i32(t1, arg2, mask); 618 tcg_gen_shli_i32(t1, t1, ofs); 619 } else { 620 tcg_gen_shli_i32(t1, arg2, ofs); 621 } 622 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 623 tcg_gen_or_i32(ret, ret, t1); 624 625 tcg_temp_free_i32(t1); 626 } 627 628 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 629 unsigned int ofs, unsigned int len) 630 { 631 tcg_debug_assert(ofs < 32); 632 tcg_debug_assert(len > 0); 633 tcg_debug_assert(len <= 32); 634 tcg_debug_assert(ofs + len <= 32); 635 636 if (ofs + len == 32) { 637 tcg_gen_shli_i32(ret, arg, ofs); 638 } else if (ofs == 0) { 639 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 640 } else if (TCG_TARGET_HAS_deposit_i32 641 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 642 TCGv_i32 zero = tcg_const_i32(0); 643 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 644 tcg_temp_free_i32(zero); 645 } else { 646 /* To help two-operand hosts we prefer to zero-extend first, 647 which allows ARG to stay live. */ 648 switch (len) { 649 case 16: 650 if (TCG_TARGET_HAS_ext16u_i32) { 651 tcg_gen_ext16u_i32(ret, arg); 652 tcg_gen_shli_i32(ret, ret, ofs); 653 return; 654 } 655 break; 656 case 8: 657 if (TCG_TARGET_HAS_ext8u_i32) { 658 tcg_gen_ext8u_i32(ret, arg); 659 tcg_gen_shli_i32(ret, ret, ofs); 660 return; 661 } 662 break; 663 } 664 /* Otherwise prefer zero-extension over AND for code size. */ 665 switch (ofs + len) { 666 case 16: 667 if (TCG_TARGET_HAS_ext16u_i32) { 668 tcg_gen_shli_i32(ret, arg, ofs); 669 tcg_gen_ext16u_i32(ret, ret); 670 return; 671 } 672 break; 673 case 8: 674 if (TCG_TARGET_HAS_ext8u_i32) { 675 tcg_gen_shli_i32(ret, arg, ofs); 676 tcg_gen_ext8u_i32(ret, ret); 677 return; 678 } 679 break; 680 } 681 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 682 tcg_gen_shli_i32(ret, ret, ofs); 683 } 684 } 685 686 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 687 unsigned int ofs, unsigned int len) 688 { 689 tcg_debug_assert(ofs < 32); 690 tcg_debug_assert(len > 0); 691 tcg_debug_assert(len <= 32); 692 tcg_debug_assert(ofs + len <= 32); 693 694 /* Canonicalize certain special cases, even if extract is supported. */ 695 if (ofs + len == 32) { 696 tcg_gen_shri_i32(ret, arg, 32 - len); 697 return; 698 } 699 if (ofs == 0) { 700 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 701 return; 702 } 703 704 if (TCG_TARGET_HAS_extract_i32 705 && TCG_TARGET_extract_i32_valid(ofs, len)) { 706 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 707 return; 708 } 709 710 /* Assume that zero-extension, if available, is cheaper than a shift. */ 711 switch (ofs + len) { 712 case 16: 713 if (TCG_TARGET_HAS_ext16u_i32) { 714 tcg_gen_ext16u_i32(ret, arg); 715 tcg_gen_shri_i32(ret, ret, ofs); 716 return; 717 } 718 break; 719 case 8: 720 if (TCG_TARGET_HAS_ext8u_i32) { 721 tcg_gen_ext8u_i32(ret, arg); 722 tcg_gen_shri_i32(ret, ret, ofs); 723 return; 724 } 725 break; 726 } 727 728 /* ??? Ideally we'd know what values are available for immediate AND. 729 Assume that 8 bits are available, plus the special case of 16, 730 so that we get ext8u, ext16u. */ 731 switch (len) { 732 case 1 ... 8: case 16: 733 tcg_gen_shri_i32(ret, arg, ofs); 734 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 735 break; 736 default: 737 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 738 tcg_gen_shri_i32(ret, ret, 32 - len); 739 break; 740 } 741 } 742 743 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 744 unsigned int ofs, unsigned int len) 745 { 746 tcg_debug_assert(ofs < 32); 747 tcg_debug_assert(len > 0); 748 tcg_debug_assert(len <= 32); 749 tcg_debug_assert(ofs + len <= 32); 750 751 /* Canonicalize certain special cases, even if extract is supported. */ 752 if (ofs + len == 32) { 753 tcg_gen_sari_i32(ret, arg, 32 - len); 754 return; 755 } 756 if (ofs == 0) { 757 switch (len) { 758 case 16: 759 tcg_gen_ext16s_i32(ret, arg); 760 return; 761 case 8: 762 tcg_gen_ext8s_i32(ret, arg); 763 return; 764 } 765 } 766 767 if (TCG_TARGET_HAS_sextract_i32 768 && TCG_TARGET_extract_i32_valid(ofs, len)) { 769 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 770 return; 771 } 772 773 /* Assume that sign-extension, if available, is cheaper than a shift. */ 774 switch (ofs + len) { 775 case 16: 776 if (TCG_TARGET_HAS_ext16s_i32) { 777 tcg_gen_ext16s_i32(ret, arg); 778 tcg_gen_sari_i32(ret, ret, ofs); 779 return; 780 } 781 break; 782 case 8: 783 if (TCG_TARGET_HAS_ext8s_i32) { 784 tcg_gen_ext8s_i32(ret, arg); 785 tcg_gen_sari_i32(ret, ret, ofs); 786 return; 787 } 788 break; 789 } 790 switch (len) { 791 case 16: 792 if (TCG_TARGET_HAS_ext16s_i32) { 793 tcg_gen_shri_i32(ret, arg, ofs); 794 tcg_gen_ext16s_i32(ret, ret); 795 return; 796 } 797 break; 798 case 8: 799 if (TCG_TARGET_HAS_ext8s_i32) { 800 tcg_gen_shri_i32(ret, arg, ofs); 801 tcg_gen_ext8s_i32(ret, ret); 802 return; 803 } 804 break; 805 } 806 807 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 808 tcg_gen_sari_i32(ret, ret, 32 - len); 809 } 810 811 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 812 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 813 { 814 if (cond == TCG_COND_ALWAYS) { 815 tcg_gen_mov_i32(ret, v1); 816 } else if (cond == TCG_COND_NEVER) { 817 tcg_gen_mov_i32(ret, v2); 818 } else if (TCG_TARGET_HAS_movcond_i32) { 819 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 820 } else { 821 TCGv_i32 t0 = tcg_temp_new_i32(); 822 TCGv_i32 t1 = tcg_temp_new_i32(); 823 tcg_gen_setcond_i32(cond, t0, c1, c2); 824 tcg_gen_neg_i32(t0, t0); 825 tcg_gen_and_i32(t1, v1, t0); 826 tcg_gen_andc_i32(ret, v2, t0); 827 tcg_gen_or_i32(ret, ret, t1); 828 tcg_temp_free_i32(t0); 829 tcg_temp_free_i32(t1); 830 } 831 } 832 833 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 834 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 835 { 836 if (TCG_TARGET_HAS_add2_i32) { 837 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 838 } else { 839 TCGv_i64 t0 = tcg_temp_new_i64(); 840 TCGv_i64 t1 = tcg_temp_new_i64(); 841 tcg_gen_concat_i32_i64(t0, al, ah); 842 tcg_gen_concat_i32_i64(t1, bl, bh); 843 tcg_gen_add_i64(t0, t0, t1); 844 tcg_gen_extr_i64_i32(rl, rh, t0); 845 tcg_temp_free_i64(t0); 846 tcg_temp_free_i64(t1); 847 } 848 } 849 850 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 851 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 852 { 853 if (TCG_TARGET_HAS_sub2_i32) { 854 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 855 } else { 856 TCGv_i64 t0 = tcg_temp_new_i64(); 857 TCGv_i64 t1 = tcg_temp_new_i64(); 858 tcg_gen_concat_i32_i64(t0, al, ah); 859 tcg_gen_concat_i32_i64(t1, bl, bh); 860 tcg_gen_sub_i64(t0, t0, t1); 861 tcg_gen_extr_i64_i32(rl, rh, t0); 862 tcg_temp_free_i64(t0); 863 tcg_temp_free_i64(t1); 864 } 865 } 866 867 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 868 { 869 if (TCG_TARGET_HAS_mulu2_i32) { 870 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 871 } else if (TCG_TARGET_HAS_muluh_i32) { 872 TCGv_i32 t = tcg_temp_new_i32(); 873 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 874 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2); 875 tcg_gen_mov_i32(rl, t); 876 tcg_temp_free_i32(t); 877 } else { 878 TCGv_i64 t0 = tcg_temp_new_i64(); 879 TCGv_i64 t1 = tcg_temp_new_i64(); 880 tcg_gen_extu_i32_i64(t0, arg1); 881 tcg_gen_extu_i32_i64(t1, arg2); 882 tcg_gen_mul_i64(t0, t0, t1); 883 tcg_gen_extr_i64_i32(rl, rh, t0); 884 tcg_temp_free_i64(t0); 885 tcg_temp_free_i64(t1); 886 } 887 } 888 889 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 890 { 891 if (TCG_TARGET_HAS_muls2_i32) { 892 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 893 } else if (TCG_TARGET_HAS_mulsh_i32) { 894 TCGv_i32 t = tcg_temp_new_i32(); 895 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 896 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2); 897 tcg_gen_mov_i32(rl, t); 898 tcg_temp_free_i32(t); 899 } else if (TCG_TARGET_REG_BITS == 32) { 900 TCGv_i32 t0 = tcg_temp_new_i32(); 901 TCGv_i32 t1 = tcg_temp_new_i32(); 902 TCGv_i32 t2 = tcg_temp_new_i32(); 903 TCGv_i32 t3 = tcg_temp_new_i32(); 904 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 905 /* Adjust for negative inputs. */ 906 tcg_gen_sari_i32(t2, arg1, 31); 907 tcg_gen_sari_i32(t3, arg2, 31); 908 tcg_gen_and_i32(t2, t2, arg2); 909 tcg_gen_and_i32(t3, t3, arg1); 910 tcg_gen_sub_i32(rh, t1, t2); 911 tcg_gen_sub_i32(rh, rh, t3); 912 tcg_gen_mov_i32(rl, t0); 913 tcg_temp_free_i32(t0); 914 tcg_temp_free_i32(t1); 915 tcg_temp_free_i32(t2); 916 tcg_temp_free_i32(t3); 917 } else { 918 TCGv_i64 t0 = tcg_temp_new_i64(); 919 TCGv_i64 t1 = tcg_temp_new_i64(); 920 tcg_gen_ext_i32_i64(t0, arg1); 921 tcg_gen_ext_i32_i64(t1, arg2); 922 tcg_gen_mul_i64(t0, t0, t1); 923 tcg_gen_extr_i64_i32(rl, rh, t0); 924 tcg_temp_free_i64(t0); 925 tcg_temp_free_i64(t1); 926 } 927 } 928 929 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 930 { 931 if (TCG_TARGET_REG_BITS == 32) { 932 TCGv_i32 t0 = tcg_temp_new_i32(); 933 TCGv_i32 t1 = tcg_temp_new_i32(); 934 TCGv_i32 t2 = tcg_temp_new_i32(); 935 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 936 /* Adjust for negative input for the signed arg1. */ 937 tcg_gen_sari_i32(t2, arg1, 31); 938 tcg_gen_and_i32(t2, t2, arg2); 939 tcg_gen_sub_i32(rh, t1, t2); 940 tcg_gen_mov_i32(rl, t0); 941 tcg_temp_free_i32(t0); 942 tcg_temp_free_i32(t1); 943 tcg_temp_free_i32(t2); 944 } else { 945 TCGv_i64 t0 = tcg_temp_new_i64(); 946 TCGv_i64 t1 = tcg_temp_new_i64(); 947 tcg_gen_ext_i32_i64(t0, arg1); 948 tcg_gen_extu_i32_i64(t1, arg2); 949 tcg_gen_mul_i64(t0, t0, t1); 950 tcg_gen_extr_i64_i32(rl, rh, t0); 951 tcg_temp_free_i64(t0); 952 tcg_temp_free_i64(t1); 953 } 954 } 955 956 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 957 { 958 if (TCG_TARGET_HAS_ext8s_i32) { 959 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg); 960 } else { 961 tcg_gen_shli_i32(ret, arg, 24); 962 tcg_gen_sari_i32(ret, ret, 24); 963 } 964 } 965 966 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 967 { 968 if (TCG_TARGET_HAS_ext16s_i32) { 969 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg); 970 } else { 971 tcg_gen_shli_i32(ret, arg, 16); 972 tcg_gen_sari_i32(ret, ret, 16); 973 } 974 } 975 976 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 977 { 978 if (TCG_TARGET_HAS_ext8u_i32) { 979 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg); 980 } else { 981 tcg_gen_andi_i32(ret, arg, 0xffu); 982 } 983 } 984 985 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 986 { 987 if (TCG_TARGET_HAS_ext16u_i32) { 988 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg); 989 } else { 990 tcg_gen_andi_i32(ret, arg, 0xffffu); 991 } 992 } 993 994 /* Note: we assume the two high bytes are set to zero */ 995 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg) 996 { 997 if (TCG_TARGET_HAS_bswap16_i32) { 998 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg); 999 } else { 1000 TCGv_i32 t0 = tcg_temp_new_i32(); 1001 1002 tcg_gen_ext8u_i32(t0, arg); 1003 tcg_gen_shli_i32(t0, t0, 8); 1004 tcg_gen_shri_i32(ret, arg, 8); 1005 tcg_gen_or_i32(ret, ret, t0); 1006 tcg_temp_free_i32(t0); 1007 } 1008 } 1009 1010 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1011 { 1012 if (TCG_TARGET_HAS_bswap32_i32) { 1013 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg); 1014 } else { 1015 TCGv_i32 t0, t1; 1016 t0 = tcg_temp_new_i32(); 1017 t1 = tcg_temp_new_i32(); 1018 1019 tcg_gen_shli_i32(t0, arg, 24); 1020 1021 tcg_gen_andi_i32(t1, arg, 0x0000ff00); 1022 tcg_gen_shli_i32(t1, t1, 8); 1023 tcg_gen_or_i32(t0, t0, t1); 1024 1025 tcg_gen_shri_i32(t1, arg, 8); 1026 tcg_gen_andi_i32(t1, t1, 0x0000ff00); 1027 tcg_gen_or_i32(t0, t0, t1); 1028 1029 tcg_gen_shri_i32(t1, arg, 24); 1030 tcg_gen_or_i32(ret, t0, t1); 1031 tcg_temp_free_i32(t0); 1032 tcg_temp_free_i32(t1); 1033 } 1034 } 1035 1036 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1037 { 1038 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1039 } 1040 1041 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1042 { 1043 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1044 } 1045 1046 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1047 { 1048 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1049 } 1050 1051 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1052 { 1053 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1054 } 1055 1056 /* 64-bit ops */ 1057 1058 #if TCG_TARGET_REG_BITS == 32 1059 /* These are all inline for TCG_TARGET_REG_BITS == 64. */ 1060 1061 void tcg_gen_discard_i64(TCGv_i64 arg) 1062 { 1063 tcg_gen_discard_i32(TCGV_LOW(arg)); 1064 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1065 } 1066 1067 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1068 { 1069 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1070 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1071 } 1072 1073 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1074 { 1075 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1076 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1077 } 1078 1079 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1080 { 1081 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1082 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1083 } 1084 1085 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1086 { 1087 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1088 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1089 } 1090 1091 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1092 { 1093 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1094 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1095 } 1096 1097 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1098 { 1099 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1100 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1101 } 1102 1103 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1104 { 1105 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1106 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1107 } 1108 1109 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1110 { 1111 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1112 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1113 } 1114 1115 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1116 { 1117 /* Since arg2 and ret have different types, 1118 they cannot be the same temporary */ 1119 #ifdef HOST_WORDS_BIGENDIAN 1120 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1121 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1122 #else 1123 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1124 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1125 #endif 1126 } 1127 1128 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1129 { 1130 #ifdef HOST_WORDS_BIGENDIAN 1131 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1132 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1133 #else 1134 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1135 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1136 #endif 1137 } 1138 1139 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1140 { 1141 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1142 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1143 } 1144 1145 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1146 { 1147 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1148 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1149 } 1150 1151 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1152 { 1153 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1154 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1155 } 1156 1157 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1158 { 1159 gen_helper_shl_i64(ret, arg1, arg2); 1160 } 1161 1162 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1163 { 1164 gen_helper_shr_i64(ret, arg1, arg2); 1165 } 1166 1167 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1168 { 1169 gen_helper_sar_i64(ret, arg1, arg2); 1170 } 1171 1172 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1173 { 1174 TCGv_i64 t0; 1175 TCGv_i32 t1; 1176 1177 t0 = tcg_temp_new_i64(); 1178 t1 = tcg_temp_new_i32(); 1179 1180 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1181 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1182 1183 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1184 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1185 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1186 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1187 1188 tcg_gen_mov_i64(ret, t0); 1189 tcg_temp_free_i64(t0); 1190 tcg_temp_free_i32(t1); 1191 } 1192 #endif /* TCG_TARGET_REG_SIZE == 32 */ 1193 1194 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1195 { 1196 /* some cases can be optimized here */ 1197 if (arg2 == 0) { 1198 tcg_gen_mov_i64(ret, arg1); 1199 } else { 1200 TCGv_i64 t0 = tcg_const_i64(arg2); 1201 tcg_gen_add_i64(ret, arg1, t0); 1202 tcg_temp_free_i64(t0); 1203 } 1204 } 1205 1206 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1207 { 1208 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) { 1209 /* Don't recurse with tcg_gen_neg_i64. */ 1210 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2); 1211 } else { 1212 TCGv_i64 t0 = tcg_const_i64(arg1); 1213 tcg_gen_sub_i64(ret, t0, arg2); 1214 tcg_temp_free_i64(t0); 1215 } 1216 } 1217 1218 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1219 { 1220 /* some cases can be optimized here */ 1221 if (arg2 == 0) { 1222 tcg_gen_mov_i64(ret, arg1); 1223 } else { 1224 TCGv_i64 t0 = tcg_const_i64(arg2); 1225 tcg_gen_sub_i64(ret, arg1, t0); 1226 tcg_temp_free_i64(t0); 1227 } 1228 } 1229 1230 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1231 { 1232 TCGv_i64 t0; 1233 1234 if (TCG_TARGET_REG_BITS == 32) { 1235 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1236 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1237 return; 1238 } 1239 1240 /* Some cases can be optimized here. */ 1241 switch (arg2) { 1242 case 0: 1243 tcg_gen_movi_i64(ret, 0); 1244 return; 1245 case -1: 1246 tcg_gen_mov_i64(ret, arg1); 1247 return; 1248 case 0xff: 1249 /* Don't recurse with tcg_gen_ext8u_i64. */ 1250 if (TCG_TARGET_HAS_ext8u_i64) { 1251 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1); 1252 return; 1253 } 1254 break; 1255 case 0xffff: 1256 if (TCG_TARGET_HAS_ext16u_i64) { 1257 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1); 1258 return; 1259 } 1260 break; 1261 case 0xffffffffu: 1262 if (TCG_TARGET_HAS_ext32u_i64) { 1263 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1); 1264 return; 1265 } 1266 break; 1267 } 1268 t0 = tcg_const_i64(arg2); 1269 tcg_gen_and_i64(ret, arg1, t0); 1270 tcg_temp_free_i64(t0); 1271 } 1272 1273 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1274 { 1275 if (TCG_TARGET_REG_BITS == 32) { 1276 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1277 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1278 return; 1279 } 1280 /* Some cases can be optimized here. */ 1281 if (arg2 == -1) { 1282 tcg_gen_movi_i64(ret, -1); 1283 } else if (arg2 == 0) { 1284 tcg_gen_mov_i64(ret, arg1); 1285 } else { 1286 TCGv_i64 t0 = tcg_const_i64(arg2); 1287 tcg_gen_or_i64(ret, arg1, t0); 1288 tcg_temp_free_i64(t0); 1289 } 1290 } 1291 1292 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1293 { 1294 if (TCG_TARGET_REG_BITS == 32) { 1295 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1296 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1297 return; 1298 } 1299 /* Some cases can be optimized here. */ 1300 if (arg2 == 0) { 1301 tcg_gen_mov_i64(ret, arg1); 1302 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) { 1303 /* Don't recurse with tcg_gen_not_i64. */ 1304 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1); 1305 } else { 1306 TCGv_i64 t0 = tcg_const_i64(arg2); 1307 tcg_gen_xor_i64(ret, arg1, t0); 1308 tcg_temp_free_i64(t0); 1309 } 1310 } 1311 1312 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1313 unsigned c, bool right, bool arith) 1314 { 1315 tcg_debug_assert(c < 64); 1316 if (c == 0) { 1317 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1318 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1319 } else if (c >= 32) { 1320 c -= 32; 1321 if (right) { 1322 if (arith) { 1323 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1324 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1325 } else { 1326 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1327 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1328 } 1329 } else { 1330 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1331 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1332 } 1333 } else { 1334 TCGv_i32 t0, t1; 1335 1336 t0 = tcg_temp_new_i32(); 1337 t1 = tcg_temp_new_i32(); 1338 if (right) { 1339 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c); 1340 if (arith) { 1341 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c); 1342 } else { 1343 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c); 1344 } 1345 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1346 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0); 1347 tcg_gen_mov_i32(TCGV_HIGH(ret), t1); 1348 } else { 1349 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1350 /* Note: ret can be the same as arg1, so we use t1 */ 1351 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c); 1352 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1353 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0); 1354 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1355 } 1356 tcg_temp_free_i32(t0); 1357 tcg_temp_free_i32(t1); 1358 } 1359 } 1360 1361 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1362 { 1363 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1364 if (TCG_TARGET_REG_BITS == 32) { 1365 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1366 } else if (arg2 == 0) { 1367 tcg_gen_mov_i64(ret, arg1); 1368 } else { 1369 TCGv_i64 t0 = tcg_const_i64(arg2); 1370 tcg_gen_shl_i64(ret, arg1, t0); 1371 tcg_temp_free_i64(t0); 1372 } 1373 } 1374 1375 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1376 { 1377 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1378 if (TCG_TARGET_REG_BITS == 32) { 1379 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1380 } else if (arg2 == 0) { 1381 tcg_gen_mov_i64(ret, arg1); 1382 } else { 1383 TCGv_i64 t0 = tcg_const_i64(arg2); 1384 tcg_gen_shr_i64(ret, arg1, t0); 1385 tcg_temp_free_i64(t0); 1386 } 1387 } 1388 1389 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1390 { 1391 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1392 if (TCG_TARGET_REG_BITS == 32) { 1393 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1394 } else if (arg2 == 0) { 1395 tcg_gen_mov_i64(ret, arg1); 1396 } else { 1397 TCGv_i64 t0 = tcg_const_i64(arg2); 1398 tcg_gen_sar_i64(ret, arg1, t0); 1399 tcg_temp_free_i64(t0); 1400 } 1401 } 1402 1403 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1404 { 1405 if (cond == TCG_COND_ALWAYS) { 1406 tcg_gen_br(l); 1407 } else if (cond != TCG_COND_NEVER) { 1408 if (TCG_TARGET_REG_BITS == 32) { 1409 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1410 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1411 TCGV_HIGH(arg2), cond, label_arg(l)); 1412 } else { 1413 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1414 label_arg(l)); 1415 } 1416 } 1417 } 1418 1419 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1420 { 1421 if (cond == TCG_COND_ALWAYS) { 1422 tcg_gen_br(l); 1423 } else if (cond != TCG_COND_NEVER) { 1424 TCGv_i64 t0 = tcg_const_i64(arg2); 1425 tcg_gen_brcond_i64(cond, arg1, t0, l); 1426 tcg_temp_free_i64(t0); 1427 } 1428 } 1429 1430 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1431 TCGv_i64 arg1, TCGv_i64 arg2) 1432 { 1433 if (cond == TCG_COND_ALWAYS) { 1434 tcg_gen_movi_i64(ret, 1); 1435 } else if (cond == TCG_COND_NEVER) { 1436 tcg_gen_movi_i64(ret, 0); 1437 } else { 1438 if (TCG_TARGET_REG_BITS == 32) { 1439 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1440 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1441 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1442 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1443 } else { 1444 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1445 } 1446 } 1447 } 1448 1449 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1450 TCGv_i64 arg1, int64_t arg2) 1451 { 1452 TCGv_i64 t0 = tcg_const_i64(arg2); 1453 tcg_gen_setcond_i64(cond, ret, arg1, t0); 1454 tcg_temp_free_i64(t0); 1455 } 1456 1457 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1458 { 1459 if (arg2 == 0) { 1460 tcg_gen_movi_i64(ret, 0); 1461 } else if (is_power_of_2(arg2)) { 1462 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 1463 } else { 1464 TCGv_i64 t0 = tcg_const_i64(arg2); 1465 tcg_gen_mul_i64(ret, arg1, t0); 1466 tcg_temp_free_i64(t0); 1467 } 1468 } 1469 1470 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1471 { 1472 if (TCG_TARGET_HAS_div_i64) { 1473 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2); 1474 } else if (TCG_TARGET_HAS_div2_i64) { 1475 TCGv_i64 t0 = tcg_temp_new_i64(); 1476 tcg_gen_sari_i64(t0, arg1, 63); 1477 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2); 1478 tcg_temp_free_i64(t0); 1479 } else { 1480 gen_helper_div_i64(ret, arg1, arg2); 1481 } 1482 } 1483 1484 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1485 { 1486 if (TCG_TARGET_HAS_rem_i64) { 1487 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2); 1488 } else if (TCG_TARGET_HAS_div_i64) { 1489 TCGv_i64 t0 = tcg_temp_new_i64(); 1490 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2); 1491 tcg_gen_mul_i64(t0, t0, arg2); 1492 tcg_gen_sub_i64(ret, arg1, t0); 1493 tcg_temp_free_i64(t0); 1494 } else if (TCG_TARGET_HAS_div2_i64) { 1495 TCGv_i64 t0 = tcg_temp_new_i64(); 1496 tcg_gen_sari_i64(t0, arg1, 63); 1497 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2); 1498 tcg_temp_free_i64(t0); 1499 } else { 1500 gen_helper_rem_i64(ret, arg1, arg2); 1501 } 1502 } 1503 1504 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1505 { 1506 if (TCG_TARGET_HAS_div_i64) { 1507 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2); 1508 } else if (TCG_TARGET_HAS_div2_i64) { 1509 TCGv_i64 t0 = tcg_temp_new_i64(); 1510 tcg_gen_movi_i64(t0, 0); 1511 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2); 1512 tcg_temp_free_i64(t0); 1513 } else { 1514 gen_helper_divu_i64(ret, arg1, arg2); 1515 } 1516 } 1517 1518 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1519 { 1520 if (TCG_TARGET_HAS_rem_i64) { 1521 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2); 1522 } else if (TCG_TARGET_HAS_div_i64) { 1523 TCGv_i64 t0 = tcg_temp_new_i64(); 1524 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2); 1525 tcg_gen_mul_i64(t0, t0, arg2); 1526 tcg_gen_sub_i64(ret, arg1, t0); 1527 tcg_temp_free_i64(t0); 1528 } else if (TCG_TARGET_HAS_div2_i64) { 1529 TCGv_i64 t0 = tcg_temp_new_i64(); 1530 tcg_gen_movi_i64(t0, 0); 1531 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2); 1532 tcg_temp_free_i64(t0); 1533 } else { 1534 gen_helper_remu_i64(ret, arg1, arg2); 1535 } 1536 } 1537 1538 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 1539 { 1540 if (TCG_TARGET_REG_BITS == 32) { 1541 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1542 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1543 } else if (TCG_TARGET_HAS_ext8s_i64) { 1544 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg); 1545 } else { 1546 tcg_gen_shli_i64(ret, arg, 56); 1547 tcg_gen_sari_i64(ret, ret, 56); 1548 } 1549 } 1550 1551 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 1552 { 1553 if (TCG_TARGET_REG_BITS == 32) { 1554 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1555 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1556 } else if (TCG_TARGET_HAS_ext16s_i64) { 1557 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg); 1558 } else { 1559 tcg_gen_shli_i64(ret, arg, 48); 1560 tcg_gen_sari_i64(ret, ret, 48); 1561 } 1562 } 1563 1564 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 1565 { 1566 if (TCG_TARGET_REG_BITS == 32) { 1567 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1568 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1569 } else if (TCG_TARGET_HAS_ext32s_i64) { 1570 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg); 1571 } else { 1572 tcg_gen_shli_i64(ret, arg, 32); 1573 tcg_gen_sari_i64(ret, ret, 32); 1574 } 1575 } 1576 1577 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 1578 { 1579 if (TCG_TARGET_REG_BITS == 32) { 1580 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1581 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1582 } else if (TCG_TARGET_HAS_ext8u_i64) { 1583 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg); 1584 } else { 1585 tcg_gen_andi_i64(ret, arg, 0xffu); 1586 } 1587 } 1588 1589 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 1590 { 1591 if (TCG_TARGET_REG_BITS == 32) { 1592 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1593 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1594 } else if (TCG_TARGET_HAS_ext16u_i64) { 1595 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg); 1596 } else { 1597 tcg_gen_andi_i64(ret, arg, 0xffffu); 1598 } 1599 } 1600 1601 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 1602 { 1603 if (TCG_TARGET_REG_BITS == 32) { 1604 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1605 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1606 } else if (TCG_TARGET_HAS_ext32u_i64) { 1607 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg); 1608 } else { 1609 tcg_gen_andi_i64(ret, arg, 0xffffffffu); 1610 } 1611 } 1612 1613 /* Note: we assume the six high bytes are set to zero */ 1614 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg) 1615 { 1616 if (TCG_TARGET_REG_BITS == 32) { 1617 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1618 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1619 } else if (TCG_TARGET_HAS_bswap16_i64) { 1620 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg); 1621 } else { 1622 TCGv_i64 t0 = tcg_temp_new_i64(); 1623 1624 tcg_gen_ext8u_i64(t0, arg); 1625 tcg_gen_shli_i64(t0, t0, 8); 1626 tcg_gen_shri_i64(ret, arg, 8); 1627 tcg_gen_or_i64(ret, ret, t0); 1628 tcg_temp_free_i64(t0); 1629 } 1630 } 1631 1632 /* Note: we assume the four high bytes are set to zero */ 1633 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg) 1634 { 1635 if (TCG_TARGET_REG_BITS == 32) { 1636 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1637 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1638 } else if (TCG_TARGET_HAS_bswap32_i64) { 1639 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg); 1640 } else { 1641 TCGv_i64 t0, t1; 1642 t0 = tcg_temp_new_i64(); 1643 t1 = tcg_temp_new_i64(); 1644 1645 tcg_gen_shli_i64(t0, arg, 24); 1646 tcg_gen_ext32u_i64(t0, t0); 1647 1648 tcg_gen_andi_i64(t1, arg, 0x0000ff00); 1649 tcg_gen_shli_i64(t1, t1, 8); 1650 tcg_gen_or_i64(t0, t0, t1); 1651 1652 tcg_gen_shri_i64(t1, arg, 8); 1653 tcg_gen_andi_i64(t1, t1, 0x0000ff00); 1654 tcg_gen_or_i64(t0, t0, t1); 1655 1656 tcg_gen_shri_i64(t1, arg, 24); 1657 tcg_gen_or_i64(ret, t0, t1); 1658 tcg_temp_free_i64(t0); 1659 tcg_temp_free_i64(t1); 1660 } 1661 } 1662 1663 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 1664 { 1665 if (TCG_TARGET_REG_BITS == 32) { 1666 TCGv_i32 t0, t1; 1667 t0 = tcg_temp_new_i32(); 1668 t1 = tcg_temp_new_i32(); 1669 1670 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 1671 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 1672 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1673 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 1674 tcg_temp_free_i32(t0); 1675 tcg_temp_free_i32(t1); 1676 } else if (TCG_TARGET_HAS_bswap64_i64) { 1677 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg); 1678 } else { 1679 TCGv_i64 t0 = tcg_temp_new_i64(); 1680 TCGv_i64 t1 = tcg_temp_new_i64(); 1681 1682 tcg_gen_shli_i64(t0, arg, 56); 1683 1684 tcg_gen_andi_i64(t1, arg, 0x0000ff00); 1685 tcg_gen_shli_i64(t1, t1, 40); 1686 tcg_gen_or_i64(t0, t0, t1); 1687 1688 tcg_gen_andi_i64(t1, arg, 0x00ff0000); 1689 tcg_gen_shli_i64(t1, t1, 24); 1690 tcg_gen_or_i64(t0, t0, t1); 1691 1692 tcg_gen_andi_i64(t1, arg, 0xff000000); 1693 tcg_gen_shli_i64(t1, t1, 8); 1694 tcg_gen_or_i64(t0, t0, t1); 1695 1696 tcg_gen_shri_i64(t1, arg, 8); 1697 tcg_gen_andi_i64(t1, t1, 0xff000000); 1698 tcg_gen_or_i64(t0, t0, t1); 1699 1700 tcg_gen_shri_i64(t1, arg, 24); 1701 tcg_gen_andi_i64(t1, t1, 0x00ff0000); 1702 tcg_gen_or_i64(t0, t0, t1); 1703 1704 tcg_gen_shri_i64(t1, arg, 40); 1705 tcg_gen_andi_i64(t1, t1, 0x0000ff00); 1706 tcg_gen_or_i64(t0, t0, t1); 1707 1708 tcg_gen_shri_i64(t1, arg, 56); 1709 tcg_gen_or_i64(ret, t0, t1); 1710 tcg_temp_free_i64(t0); 1711 tcg_temp_free_i64(t1); 1712 } 1713 } 1714 1715 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 1716 { 1717 if (TCG_TARGET_REG_BITS == 32) { 1718 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1719 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1720 } else if (TCG_TARGET_HAS_not_i64) { 1721 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg); 1722 } else { 1723 tcg_gen_xori_i64(ret, arg, -1); 1724 } 1725 } 1726 1727 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1728 { 1729 if (TCG_TARGET_REG_BITS == 32) { 1730 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1731 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1732 } else if (TCG_TARGET_HAS_andc_i64) { 1733 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2); 1734 } else { 1735 TCGv_i64 t0 = tcg_temp_new_i64(); 1736 tcg_gen_not_i64(t0, arg2); 1737 tcg_gen_and_i64(ret, arg1, t0); 1738 tcg_temp_free_i64(t0); 1739 } 1740 } 1741 1742 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1743 { 1744 if (TCG_TARGET_REG_BITS == 32) { 1745 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1746 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1747 } else if (TCG_TARGET_HAS_eqv_i64) { 1748 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2); 1749 } else { 1750 tcg_gen_xor_i64(ret, arg1, arg2); 1751 tcg_gen_not_i64(ret, ret); 1752 } 1753 } 1754 1755 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1756 { 1757 if (TCG_TARGET_REG_BITS == 32) { 1758 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1759 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1760 } else if (TCG_TARGET_HAS_nand_i64) { 1761 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2); 1762 } else { 1763 tcg_gen_and_i64(ret, arg1, arg2); 1764 tcg_gen_not_i64(ret, ret); 1765 } 1766 } 1767 1768 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1769 { 1770 if (TCG_TARGET_REG_BITS == 32) { 1771 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1772 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1773 } else if (TCG_TARGET_HAS_nor_i64) { 1774 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2); 1775 } else { 1776 tcg_gen_or_i64(ret, arg1, arg2); 1777 tcg_gen_not_i64(ret, ret); 1778 } 1779 } 1780 1781 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1782 { 1783 if (TCG_TARGET_REG_BITS == 32) { 1784 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1785 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1786 } else if (TCG_TARGET_HAS_orc_i64) { 1787 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2); 1788 } else { 1789 TCGv_i64 t0 = tcg_temp_new_i64(); 1790 tcg_gen_not_i64(t0, arg2); 1791 tcg_gen_or_i64(ret, arg1, t0); 1792 tcg_temp_free_i64(t0); 1793 } 1794 } 1795 1796 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1797 { 1798 if (TCG_TARGET_HAS_clz_i64) { 1799 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2); 1800 } else { 1801 gen_helper_clz_i64(ret, arg1, arg2); 1802 } 1803 } 1804 1805 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1806 { 1807 if (TCG_TARGET_REG_BITS == 32 1808 && TCG_TARGET_HAS_clz_i32 1809 && arg2 <= 0xffffffffu) { 1810 TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32); 1811 tcg_gen_clz_i32(t, TCGV_LOW(arg1), t); 1812 tcg_gen_addi_i32(t, t, 32); 1813 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 1814 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1815 tcg_temp_free_i32(t); 1816 } else { 1817 TCGv_i64 t = tcg_const_i64(arg2); 1818 tcg_gen_clz_i64(ret, arg1, t); 1819 tcg_temp_free_i64(t); 1820 } 1821 } 1822 1823 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1824 { 1825 if (TCG_TARGET_HAS_ctz_i64) { 1826 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); 1827 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) { 1828 TCGv_i64 z, t = tcg_temp_new_i64(); 1829 1830 if (TCG_TARGET_HAS_ctpop_i64) { 1831 tcg_gen_subi_i64(t, arg1, 1); 1832 tcg_gen_andc_i64(t, t, arg1); 1833 tcg_gen_ctpop_i64(t, t); 1834 } else { 1835 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */ 1836 tcg_gen_neg_i64(t, arg1); 1837 tcg_gen_and_i64(t, t, arg1); 1838 tcg_gen_clzi_i64(t, t, 64); 1839 tcg_gen_xori_i64(t, t, 63); 1840 } 1841 z = tcg_const_i64(0); 1842 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 1843 tcg_temp_free_i64(t); 1844 tcg_temp_free_i64(z); 1845 } else { 1846 gen_helper_ctz_i64(ret, arg1, arg2); 1847 } 1848 } 1849 1850 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1851 { 1852 if (TCG_TARGET_REG_BITS == 32 1853 && TCG_TARGET_HAS_ctz_i32 1854 && arg2 <= 0xffffffffu) { 1855 TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32); 1856 tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32); 1857 tcg_gen_addi_i32(t32, t32, 32); 1858 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 1859 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1860 tcg_temp_free_i32(t32); 1861 } else if (!TCG_TARGET_HAS_ctz_i64 1862 && TCG_TARGET_HAS_ctpop_i64 1863 && arg2 == 64) { 1864 /* This equivalence has the advantage of not requiring a fixup. */ 1865 TCGv_i64 t = tcg_temp_new_i64(); 1866 tcg_gen_subi_i64(t, arg1, 1); 1867 tcg_gen_andc_i64(t, t, arg1); 1868 tcg_gen_ctpop_i64(ret, t); 1869 tcg_temp_free_i64(t); 1870 } else { 1871 TCGv_i64 t64 = tcg_const_i64(arg2); 1872 tcg_gen_ctz_i64(ret, arg1, t64); 1873 tcg_temp_free_i64(t64); 1874 } 1875 } 1876 1877 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 1878 { 1879 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) { 1880 TCGv_i64 t = tcg_temp_new_i64(); 1881 tcg_gen_sari_i64(t, arg, 63); 1882 tcg_gen_xor_i64(t, t, arg); 1883 tcg_gen_clzi_i64(t, t, 64); 1884 tcg_gen_subi_i64(ret, t, 1); 1885 tcg_temp_free_i64(t); 1886 } else { 1887 gen_helper_clrsb_i64(ret, arg); 1888 } 1889 } 1890 1891 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 1892 { 1893 if (TCG_TARGET_HAS_ctpop_i64) { 1894 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 1895 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 1896 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1897 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1898 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 1899 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1900 } else { 1901 gen_helper_ctpop_i64(ret, arg1); 1902 } 1903 } 1904 1905 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1906 { 1907 if (TCG_TARGET_HAS_rot_i64) { 1908 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2); 1909 } else { 1910 TCGv_i64 t0, t1; 1911 t0 = tcg_temp_new_i64(); 1912 t1 = tcg_temp_new_i64(); 1913 tcg_gen_shl_i64(t0, arg1, arg2); 1914 tcg_gen_subfi_i64(t1, 64, arg2); 1915 tcg_gen_shr_i64(t1, arg1, t1); 1916 tcg_gen_or_i64(ret, t0, t1); 1917 tcg_temp_free_i64(t0); 1918 tcg_temp_free_i64(t1); 1919 } 1920 } 1921 1922 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1923 { 1924 tcg_debug_assert(arg2 < 64); 1925 /* some cases can be optimized here */ 1926 if (arg2 == 0) { 1927 tcg_gen_mov_i64(ret, arg1); 1928 } else if (TCG_TARGET_HAS_rot_i64) { 1929 TCGv_i64 t0 = tcg_const_i64(arg2); 1930 tcg_gen_rotl_i64(ret, arg1, t0); 1931 tcg_temp_free_i64(t0); 1932 } else { 1933 TCGv_i64 t0, t1; 1934 t0 = tcg_temp_new_i64(); 1935 t1 = tcg_temp_new_i64(); 1936 tcg_gen_shli_i64(t0, arg1, arg2); 1937 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 1938 tcg_gen_or_i64(ret, t0, t1); 1939 tcg_temp_free_i64(t0); 1940 tcg_temp_free_i64(t1); 1941 } 1942 } 1943 1944 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1945 { 1946 if (TCG_TARGET_HAS_rot_i64) { 1947 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2); 1948 } else { 1949 TCGv_i64 t0, t1; 1950 t0 = tcg_temp_new_i64(); 1951 t1 = tcg_temp_new_i64(); 1952 tcg_gen_shr_i64(t0, arg1, arg2); 1953 tcg_gen_subfi_i64(t1, 64, arg2); 1954 tcg_gen_shl_i64(t1, arg1, t1); 1955 tcg_gen_or_i64(ret, t0, t1); 1956 tcg_temp_free_i64(t0); 1957 tcg_temp_free_i64(t1); 1958 } 1959 } 1960 1961 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1962 { 1963 tcg_debug_assert(arg2 < 64); 1964 /* some cases can be optimized here */ 1965 if (arg2 == 0) { 1966 tcg_gen_mov_i64(ret, arg1); 1967 } else { 1968 tcg_gen_rotli_i64(ret, arg1, 64 - arg2); 1969 } 1970 } 1971 1972 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 1973 unsigned int ofs, unsigned int len) 1974 { 1975 uint64_t mask; 1976 TCGv_i64 t1; 1977 1978 tcg_debug_assert(ofs < 64); 1979 tcg_debug_assert(len > 0); 1980 tcg_debug_assert(len <= 64); 1981 tcg_debug_assert(ofs + len <= 64); 1982 1983 if (len == 64) { 1984 tcg_gen_mov_i64(ret, arg2); 1985 return; 1986 } 1987 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 1988 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 1989 return; 1990 } 1991 1992 if (TCG_TARGET_REG_BITS == 32) { 1993 if (ofs >= 32) { 1994 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 1995 TCGV_LOW(arg2), ofs - 32, len); 1996 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1997 return; 1998 } 1999 if (ofs + len <= 32) { 2000 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 2001 TCGV_LOW(arg2), ofs, len); 2002 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2003 return; 2004 } 2005 } 2006 2007 mask = (1ull << len) - 1; 2008 t1 = tcg_temp_new_i64(); 2009 2010 if (ofs + len < 64) { 2011 tcg_gen_andi_i64(t1, arg2, mask); 2012 tcg_gen_shli_i64(t1, t1, ofs); 2013 } else { 2014 tcg_gen_shli_i64(t1, arg2, ofs); 2015 } 2016 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2017 tcg_gen_or_i64(ret, ret, t1); 2018 2019 tcg_temp_free_i64(t1); 2020 } 2021 2022 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2023 unsigned int ofs, unsigned int len) 2024 { 2025 tcg_debug_assert(ofs < 64); 2026 tcg_debug_assert(len > 0); 2027 tcg_debug_assert(len <= 64); 2028 tcg_debug_assert(ofs + len <= 64); 2029 2030 if (ofs + len == 64) { 2031 tcg_gen_shli_i64(ret, arg, ofs); 2032 } else if (ofs == 0) { 2033 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2034 } else if (TCG_TARGET_HAS_deposit_i64 2035 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2036 TCGv_i64 zero = tcg_const_i64(0); 2037 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2038 tcg_temp_free_i64(zero); 2039 } else { 2040 if (TCG_TARGET_REG_BITS == 32) { 2041 if (ofs >= 32) { 2042 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2043 ofs - 32, len); 2044 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2045 return; 2046 } 2047 if (ofs + len <= 32) { 2048 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2049 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2050 return; 2051 } 2052 } 2053 /* To help two-operand hosts we prefer to zero-extend first, 2054 which allows ARG to stay live. */ 2055 switch (len) { 2056 case 32: 2057 if (TCG_TARGET_HAS_ext32u_i64) { 2058 tcg_gen_ext32u_i64(ret, arg); 2059 tcg_gen_shli_i64(ret, ret, ofs); 2060 return; 2061 } 2062 break; 2063 case 16: 2064 if (TCG_TARGET_HAS_ext16u_i64) { 2065 tcg_gen_ext16u_i64(ret, arg); 2066 tcg_gen_shli_i64(ret, ret, ofs); 2067 return; 2068 } 2069 break; 2070 case 8: 2071 if (TCG_TARGET_HAS_ext8u_i64) { 2072 tcg_gen_ext8u_i64(ret, arg); 2073 tcg_gen_shli_i64(ret, ret, ofs); 2074 return; 2075 } 2076 break; 2077 } 2078 /* Otherwise prefer zero-extension over AND for code size. */ 2079 switch (ofs + len) { 2080 case 32: 2081 if (TCG_TARGET_HAS_ext32u_i64) { 2082 tcg_gen_shli_i64(ret, arg, ofs); 2083 tcg_gen_ext32u_i64(ret, ret); 2084 return; 2085 } 2086 break; 2087 case 16: 2088 if (TCG_TARGET_HAS_ext16u_i64) { 2089 tcg_gen_shli_i64(ret, arg, ofs); 2090 tcg_gen_ext16u_i64(ret, ret); 2091 return; 2092 } 2093 break; 2094 case 8: 2095 if (TCG_TARGET_HAS_ext8u_i64) { 2096 tcg_gen_shli_i64(ret, arg, ofs); 2097 tcg_gen_ext8u_i64(ret, ret); 2098 return; 2099 } 2100 break; 2101 } 2102 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2103 tcg_gen_shli_i64(ret, ret, ofs); 2104 } 2105 } 2106 2107 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2108 unsigned int ofs, unsigned int len) 2109 { 2110 tcg_debug_assert(ofs < 64); 2111 tcg_debug_assert(len > 0); 2112 tcg_debug_assert(len <= 64); 2113 tcg_debug_assert(ofs + len <= 64); 2114 2115 /* Canonicalize certain special cases, even if extract is supported. */ 2116 if (ofs + len == 64) { 2117 tcg_gen_shri_i64(ret, arg, 64 - len); 2118 return; 2119 } 2120 if (ofs == 0) { 2121 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2122 return; 2123 } 2124 2125 if (TCG_TARGET_REG_BITS == 32) { 2126 /* Look for a 32-bit extract within one of the two words. */ 2127 if (ofs >= 32) { 2128 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2129 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2130 return; 2131 } 2132 if (ofs + len <= 32) { 2133 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2134 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2135 return; 2136 } 2137 /* The field is split across two words. One double-word 2138 shift is better than two double-word shifts. */ 2139 goto do_shift_and; 2140 } 2141 2142 if (TCG_TARGET_HAS_extract_i64 2143 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2144 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2145 return; 2146 } 2147 2148 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2149 switch (ofs + len) { 2150 case 32: 2151 if (TCG_TARGET_HAS_ext32u_i64) { 2152 tcg_gen_ext32u_i64(ret, arg); 2153 tcg_gen_shri_i64(ret, ret, ofs); 2154 return; 2155 } 2156 break; 2157 case 16: 2158 if (TCG_TARGET_HAS_ext16u_i64) { 2159 tcg_gen_ext16u_i64(ret, arg); 2160 tcg_gen_shri_i64(ret, ret, ofs); 2161 return; 2162 } 2163 break; 2164 case 8: 2165 if (TCG_TARGET_HAS_ext8u_i64) { 2166 tcg_gen_ext8u_i64(ret, arg); 2167 tcg_gen_shri_i64(ret, ret, ofs); 2168 return; 2169 } 2170 break; 2171 } 2172 2173 /* ??? Ideally we'd know what values are available for immediate AND. 2174 Assume that 8 bits are available, plus the special cases of 16 and 32, 2175 so that we get ext8u, ext16u, and ext32u. */ 2176 switch (len) { 2177 case 1 ... 8: case 16: case 32: 2178 do_shift_and: 2179 tcg_gen_shri_i64(ret, arg, ofs); 2180 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2181 break; 2182 default: 2183 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2184 tcg_gen_shri_i64(ret, ret, 64 - len); 2185 break; 2186 } 2187 } 2188 2189 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2190 unsigned int ofs, unsigned int len) 2191 { 2192 tcg_debug_assert(ofs < 64); 2193 tcg_debug_assert(len > 0); 2194 tcg_debug_assert(len <= 64); 2195 tcg_debug_assert(ofs + len <= 64); 2196 2197 /* Canonicalize certain special cases, even if sextract is supported. */ 2198 if (ofs + len == 64) { 2199 tcg_gen_sari_i64(ret, arg, 64 - len); 2200 return; 2201 } 2202 if (ofs == 0) { 2203 switch (len) { 2204 case 32: 2205 tcg_gen_ext32s_i64(ret, arg); 2206 return; 2207 case 16: 2208 tcg_gen_ext16s_i64(ret, arg); 2209 return; 2210 case 8: 2211 tcg_gen_ext8s_i64(ret, arg); 2212 return; 2213 } 2214 } 2215 2216 if (TCG_TARGET_REG_BITS == 32) { 2217 /* Look for a 32-bit extract within one of the two words. */ 2218 if (ofs >= 32) { 2219 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2220 } else if (ofs + len <= 32) { 2221 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2222 } else if (ofs == 0) { 2223 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2224 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2225 return; 2226 } else if (len > 32) { 2227 TCGv_i32 t = tcg_temp_new_i32(); 2228 /* Extract the bits for the high word normally. */ 2229 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2230 /* Shift the field down for the low part. */ 2231 tcg_gen_shri_i64(ret, arg, ofs); 2232 /* Overwrite the shift into the high part. */ 2233 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2234 tcg_temp_free_i32(t); 2235 return; 2236 } else { 2237 /* Shift the field down for the low part, such that the 2238 field sits at the MSB. */ 2239 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2240 /* Shift the field down from the MSB, sign extending. */ 2241 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2242 } 2243 /* Sign-extend the field from 32 bits. */ 2244 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2245 return; 2246 } 2247 2248 if (TCG_TARGET_HAS_sextract_i64 2249 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2250 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2251 return; 2252 } 2253 2254 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2255 switch (ofs + len) { 2256 case 32: 2257 if (TCG_TARGET_HAS_ext32s_i64) { 2258 tcg_gen_ext32s_i64(ret, arg); 2259 tcg_gen_sari_i64(ret, ret, ofs); 2260 return; 2261 } 2262 break; 2263 case 16: 2264 if (TCG_TARGET_HAS_ext16s_i64) { 2265 tcg_gen_ext16s_i64(ret, arg); 2266 tcg_gen_sari_i64(ret, ret, ofs); 2267 return; 2268 } 2269 break; 2270 case 8: 2271 if (TCG_TARGET_HAS_ext8s_i64) { 2272 tcg_gen_ext8s_i64(ret, arg); 2273 tcg_gen_sari_i64(ret, ret, ofs); 2274 return; 2275 } 2276 break; 2277 } 2278 switch (len) { 2279 case 32: 2280 if (TCG_TARGET_HAS_ext32s_i64) { 2281 tcg_gen_shri_i64(ret, arg, ofs); 2282 tcg_gen_ext32s_i64(ret, ret); 2283 return; 2284 } 2285 break; 2286 case 16: 2287 if (TCG_TARGET_HAS_ext16s_i64) { 2288 tcg_gen_shri_i64(ret, arg, ofs); 2289 tcg_gen_ext16s_i64(ret, ret); 2290 return; 2291 } 2292 break; 2293 case 8: 2294 if (TCG_TARGET_HAS_ext8s_i64) { 2295 tcg_gen_shri_i64(ret, arg, ofs); 2296 tcg_gen_ext8s_i64(ret, ret); 2297 return; 2298 } 2299 break; 2300 } 2301 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2302 tcg_gen_sari_i64(ret, ret, 64 - len); 2303 } 2304 2305 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2306 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2307 { 2308 if (cond == TCG_COND_ALWAYS) { 2309 tcg_gen_mov_i64(ret, v1); 2310 } else if (cond == TCG_COND_NEVER) { 2311 tcg_gen_mov_i64(ret, v2); 2312 } else if (TCG_TARGET_REG_BITS == 32) { 2313 TCGv_i32 t0 = tcg_temp_new_i32(); 2314 TCGv_i32 t1 = tcg_temp_new_i32(); 2315 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2316 TCGV_LOW(c1), TCGV_HIGH(c1), 2317 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2318 2319 if (TCG_TARGET_HAS_movcond_i32) { 2320 tcg_gen_movi_i32(t1, 0); 2321 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1, 2322 TCGV_LOW(v1), TCGV_LOW(v2)); 2323 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1, 2324 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2325 } else { 2326 tcg_gen_neg_i32(t0, t0); 2327 2328 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0); 2329 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0); 2330 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1); 2331 2332 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0); 2333 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0); 2334 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1); 2335 } 2336 tcg_temp_free_i32(t0); 2337 tcg_temp_free_i32(t1); 2338 } else if (TCG_TARGET_HAS_movcond_i64) { 2339 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2340 } else { 2341 TCGv_i64 t0 = tcg_temp_new_i64(); 2342 TCGv_i64 t1 = tcg_temp_new_i64(); 2343 tcg_gen_setcond_i64(cond, t0, c1, c2); 2344 tcg_gen_neg_i64(t0, t0); 2345 tcg_gen_and_i64(t1, v1, t0); 2346 tcg_gen_andc_i64(ret, v2, t0); 2347 tcg_gen_or_i64(ret, ret, t1); 2348 tcg_temp_free_i64(t0); 2349 tcg_temp_free_i64(t1); 2350 } 2351 } 2352 2353 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2354 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2355 { 2356 if (TCG_TARGET_HAS_add2_i64) { 2357 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2358 } else { 2359 TCGv_i64 t0 = tcg_temp_new_i64(); 2360 TCGv_i64 t1 = tcg_temp_new_i64(); 2361 tcg_gen_add_i64(t0, al, bl); 2362 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2363 tcg_gen_add_i64(rh, ah, bh); 2364 tcg_gen_add_i64(rh, rh, t1); 2365 tcg_gen_mov_i64(rl, t0); 2366 tcg_temp_free_i64(t0); 2367 tcg_temp_free_i64(t1); 2368 } 2369 } 2370 2371 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2372 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2373 { 2374 if (TCG_TARGET_HAS_sub2_i64) { 2375 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2376 } else { 2377 TCGv_i64 t0 = tcg_temp_new_i64(); 2378 TCGv_i64 t1 = tcg_temp_new_i64(); 2379 tcg_gen_sub_i64(t0, al, bl); 2380 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2381 tcg_gen_sub_i64(rh, ah, bh); 2382 tcg_gen_sub_i64(rh, rh, t1); 2383 tcg_gen_mov_i64(rl, t0); 2384 tcg_temp_free_i64(t0); 2385 tcg_temp_free_i64(t1); 2386 } 2387 } 2388 2389 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2390 { 2391 if (TCG_TARGET_HAS_mulu2_i64) { 2392 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2393 } else if (TCG_TARGET_HAS_muluh_i64) { 2394 TCGv_i64 t = tcg_temp_new_i64(); 2395 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2396 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2); 2397 tcg_gen_mov_i64(rl, t); 2398 tcg_temp_free_i64(t); 2399 } else { 2400 TCGv_i64 t0 = tcg_temp_new_i64(); 2401 tcg_gen_mul_i64(t0, arg1, arg2); 2402 gen_helper_muluh_i64(rh, arg1, arg2); 2403 tcg_gen_mov_i64(rl, t0); 2404 tcg_temp_free_i64(t0); 2405 } 2406 } 2407 2408 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2409 { 2410 if (TCG_TARGET_HAS_muls2_i64) { 2411 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 2412 } else if (TCG_TARGET_HAS_mulsh_i64) { 2413 TCGv_i64 t = tcg_temp_new_i64(); 2414 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2415 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2); 2416 tcg_gen_mov_i64(rl, t); 2417 tcg_temp_free_i64(t); 2418 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) { 2419 TCGv_i64 t0 = tcg_temp_new_i64(); 2420 TCGv_i64 t1 = tcg_temp_new_i64(); 2421 TCGv_i64 t2 = tcg_temp_new_i64(); 2422 TCGv_i64 t3 = tcg_temp_new_i64(); 2423 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2424 /* Adjust for negative inputs. */ 2425 tcg_gen_sari_i64(t2, arg1, 63); 2426 tcg_gen_sari_i64(t3, arg2, 63); 2427 tcg_gen_and_i64(t2, t2, arg2); 2428 tcg_gen_and_i64(t3, t3, arg1); 2429 tcg_gen_sub_i64(rh, t1, t2); 2430 tcg_gen_sub_i64(rh, rh, t3); 2431 tcg_gen_mov_i64(rl, t0); 2432 tcg_temp_free_i64(t0); 2433 tcg_temp_free_i64(t1); 2434 tcg_temp_free_i64(t2); 2435 tcg_temp_free_i64(t3); 2436 } else { 2437 TCGv_i64 t0 = tcg_temp_new_i64(); 2438 tcg_gen_mul_i64(t0, arg1, arg2); 2439 gen_helper_mulsh_i64(rh, arg1, arg2); 2440 tcg_gen_mov_i64(rl, t0); 2441 tcg_temp_free_i64(t0); 2442 } 2443 } 2444 2445 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2446 { 2447 TCGv_i64 t0 = tcg_temp_new_i64(); 2448 TCGv_i64 t1 = tcg_temp_new_i64(); 2449 TCGv_i64 t2 = tcg_temp_new_i64(); 2450 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2451 /* Adjust for negative input for the signed arg1. */ 2452 tcg_gen_sari_i64(t2, arg1, 63); 2453 tcg_gen_and_i64(t2, t2, arg2); 2454 tcg_gen_sub_i64(rh, t1, t2); 2455 tcg_gen_mov_i64(rl, t0); 2456 tcg_temp_free_i64(t0); 2457 tcg_temp_free_i64(t1); 2458 tcg_temp_free_i64(t2); 2459 } 2460 2461 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2462 { 2463 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 2464 } 2465 2466 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2467 { 2468 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 2469 } 2470 2471 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2472 { 2473 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 2474 } 2475 2476 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2477 { 2478 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 2479 } 2480 2481 /* Size changing operations. */ 2482 2483 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2484 { 2485 if (TCG_TARGET_REG_BITS == 32) { 2486 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2487 } else if (TCG_TARGET_HAS_extrl_i64_i32) { 2488 tcg_gen_op2(INDEX_op_extrl_i64_i32, 2489 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2490 } else { 2491 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2492 } 2493 } 2494 2495 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2496 { 2497 if (TCG_TARGET_REG_BITS == 32) { 2498 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2499 } else if (TCG_TARGET_HAS_extrh_i64_i32) { 2500 tcg_gen_op2(INDEX_op_extrh_i64_i32, 2501 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2502 } else { 2503 TCGv_i64 t = tcg_temp_new_i64(); 2504 tcg_gen_shri_i64(t, arg, 32); 2505 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2506 tcg_temp_free_i64(t); 2507 } 2508 } 2509 2510 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2511 { 2512 if (TCG_TARGET_REG_BITS == 32) { 2513 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2514 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2515 } else { 2516 tcg_gen_op2(INDEX_op_extu_i32_i64, 2517 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2518 } 2519 } 2520 2521 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2522 { 2523 if (TCG_TARGET_REG_BITS == 32) { 2524 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2525 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2526 } else { 2527 tcg_gen_op2(INDEX_op_ext_i32_i64, 2528 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2529 } 2530 } 2531 2532 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 2533 { 2534 TCGv_i64 tmp; 2535 2536 if (TCG_TARGET_REG_BITS == 32) { 2537 tcg_gen_mov_i32(TCGV_LOW(dest), low); 2538 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 2539 return; 2540 } 2541 2542 tmp = tcg_temp_new_i64(); 2543 /* These extensions are only needed for type correctness. 2544 We may be able to do better given target specific information. */ 2545 tcg_gen_extu_i32_i64(tmp, high); 2546 tcg_gen_extu_i32_i64(dest, low); 2547 /* If deposit is available, use it. Otherwise use the extra 2548 knowledge that we have of the zero-extensions above. */ 2549 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) { 2550 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 2551 } else { 2552 tcg_gen_shli_i64(tmp, tmp, 32); 2553 tcg_gen_or_i64(dest, dest, tmp); 2554 } 2555 tcg_temp_free_i64(tmp); 2556 } 2557 2558 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 2559 { 2560 if (TCG_TARGET_REG_BITS == 32) { 2561 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 2562 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 2563 } else { 2564 tcg_gen_extrl_i64_i32(lo, arg); 2565 tcg_gen_extrh_i64_i32(hi, arg); 2566 } 2567 } 2568 2569 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 2570 { 2571 tcg_gen_ext32u_i64(lo, arg); 2572 tcg_gen_shri_i64(hi, arg, 32); 2573 } 2574 2575 /* QEMU specific operations. */ 2576 2577 void tcg_gen_goto_tb(unsigned idx) 2578 { 2579 /* We only support two chained exits. */ 2580 tcg_debug_assert(idx <= 1); 2581 #ifdef CONFIG_DEBUG_TCG 2582 /* Verify that we havn't seen this numbered exit before. */ 2583 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 2584 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 2585 #endif 2586 tcg_gen_op1i(INDEX_op_goto_tb, idx); 2587 } 2588 2589 void tcg_gen_lookup_and_goto_ptr(void) 2590 { 2591 if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2592 TCGv_ptr ptr = tcg_temp_new_ptr(); 2593 gen_helper_lookup_tb_ptr(ptr, cpu_env); 2594 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr)); 2595 tcg_temp_free_ptr(ptr); 2596 } else { 2597 tcg_gen_exit_tb(0); 2598 } 2599 } 2600 2601 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st) 2602 { 2603 /* Trigger the asserts within as early as possible. */ 2604 (void)get_alignment_bits(op); 2605 2606 switch (op & MO_SIZE) { 2607 case MO_8: 2608 op &= ~MO_BSWAP; 2609 break; 2610 case MO_16: 2611 break; 2612 case MO_32: 2613 if (!is64) { 2614 op &= ~MO_SIGN; 2615 } 2616 break; 2617 case MO_64: 2618 if (!is64) { 2619 tcg_abort(); 2620 } 2621 break; 2622 } 2623 if (st) { 2624 op &= ~MO_SIGN; 2625 } 2626 return op; 2627 } 2628 2629 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr, 2630 TCGMemOp memop, TCGArg idx) 2631 { 2632 TCGMemOpIdx oi = make_memop_idx(memop, idx); 2633 #if TARGET_LONG_BITS == 32 2634 tcg_gen_op3i_i32(opc, val, addr, oi); 2635 #else 2636 if (TCG_TARGET_REG_BITS == 32) { 2637 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2638 } else { 2639 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi); 2640 } 2641 #endif 2642 } 2643 2644 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr, 2645 TCGMemOp memop, TCGArg idx) 2646 { 2647 TCGMemOpIdx oi = make_memop_idx(memop, idx); 2648 #if TARGET_LONG_BITS == 32 2649 if (TCG_TARGET_REG_BITS == 32) { 2650 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi); 2651 } else { 2652 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi); 2653 } 2654 #else 2655 if (TCG_TARGET_REG_BITS == 32) { 2656 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), 2657 TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2658 } else { 2659 tcg_gen_op3i_i64(opc, val, addr, oi); 2660 } 2661 #endif 2662 } 2663 2664 static void tcg_gen_req_mo(TCGBar type) 2665 { 2666 #ifdef TCG_GUEST_DEFAULT_MO 2667 type &= TCG_GUEST_DEFAULT_MO; 2668 #endif 2669 type &= ~TCG_TARGET_DEFAULT_MO; 2670 if (type) { 2671 tcg_gen_mb(type | TCG_BAR_SC); 2672 } 2673 } 2674 2675 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2676 { 2677 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2678 memop = tcg_canonicalize_memop(memop, 0, 0); 2679 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2680 addr, trace_mem_get_info(memop, 0)); 2681 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx); 2682 } 2683 2684 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2685 { 2686 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2687 memop = tcg_canonicalize_memop(memop, 0, 1); 2688 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2689 addr, trace_mem_get_info(memop, 1)); 2690 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); 2691 } 2692 2693 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2694 { 2695 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2696 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 2697 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); 2698 if (memop & MO_SIGN) { 2699 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 2700 } else { 2701 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 2702 } 2703 return; 2704 } 2705 2706 memop = tcg_canonicalize_memop(memop, 1, 0); 2707 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2708 addr, trace_mem_get_info(memop, 0)); 2709 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx); 2710 } 2711 2712 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2713 { 2714 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2715 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 2716 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); 2717 return; 2718 } 2719 2720 memop = tcg_canonicalize_memop(memop, 1, 1); 2721 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2722 addr, trace_mem_get_info(memop, 1)); 2723 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx); 2724 } 2725 2726 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc) 2727 { 2728 switch (opc & MO_SSIZE) { 2729 case MO_SB: 2730 tcg_gen_ext8s_i32(ret, val); 2731 break; 2732 case MO_UB: 2733 tcg_gen_ext8u_i32(ret, val); 2734 break; 2735 case MO_SW: 2736 tcg_gen_ext16s_i32(ret, val); 2737 break; 2738 case MO_UW: 2739 tcg_gen_ext16u_i32(ret, val); 2740 break; 2741 default: 2742 tcg_gen_mov_i32(ret, val); 2743 break; 2744 } 2745 } 2746 2747 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc) 2748 { 2749 switch (opc & MO_SSIZE) { 2750 case MO_SB: 2751 tcg_gen_ext8s_i64(ret, val); 2752 break; 2753 case MO_UB: 2754 tcg_gen_ext8u_i64(ret, val); 2755 break; 2756 case MO_SW: 2757 tcg_gen_ext16s_i64(ret, val); 2758 break; 2759 case MO_UW: 2760 tcg_gen_ext16u_i64(ret, val); 2761 break; 2762 case MO_SL: 2763 tcg_gen_ext32s_i64(ret, val); 2764 break; 2765 case MO_UL: 2766 tcg_gen_ext32u_i64(ret, val); 2767 break; 2768 default: 2769 tcg_gen_mov_i64(ret, val); 2770 break; 2771 } 2772 } 2773 2774 #ifdef CONFIG_SOFTMMU 2775 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, 2776 TCGv_i32, TCGv_i32, TCGv_i32); 2777 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, 2778 TCGv_i64, TCGv_i64, TCGv_i32); 2779 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, 2780 TCGv_i32, TCGv_i32); 2781 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, 2782 TCGv_i64, TCGv_i32); 2783 #else 2784 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32); 2785 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64); 2786 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32); 2787 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64); 2788 #endif 2789 2790 #ifdef CONFIG_ATOMIC64 2791 # define WITH_ATOMIC64(X) X, 2792 #else 2793 # define WITH_ATOMIC64(X) 2794 #endif 2795 2796 static void * const table_cmpxchg[16] = { 2797 [MO_8] = gen_helper_atomic_cmpxchgb, 2798 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, 2799 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, 2800 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, 2801 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, 2802 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) 2803 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) 2804 }; 2805 2806 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, 2807 TCGv_i32 newv, TCGArg idx, TCGMemOp memop) 2808 { 2809 memop = tcg_canonicalize_memop(memop, 0, 0); 2810 2811 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { 2812 TCGv_i32 t1 = tcg_temp_new_i32(); 2813 TCGv_i32 t2 = tcg_temp_new_i32(); 2814 2815 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); 2816 2817 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 2818 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); 2819 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 2820 tcg_temp_free_i32(t2); 2821 2822 if (memop & MO_SIGN) { 2823 tcg_gen_ext_i32(retv, t1, memop); 2824 } else { 2825 tcg_gen_mov_i32(retv, t1); 2826 } 2827 tcg_temp_free_i32(t1); 2828 } else { 2829 gen_atomic_cx_i32 gen; 2830 2831 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 2832 tcg_debug_assert(gen != NULL); 2833 2834 #ifdef CONFIG_SOFTMMU 2835 { 2836 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 2837 gen(retv, cpu_env, addr, cmpv, newv, oi); 2838 tcg_temp_free_i32(oi); 2839 } 2840 #else 2841 gen(retv, cpu_env, addr, cmpv, newv); 2842 #endif 2843 2844 if (memop & MO_SIGN) { 2845 tcg_gen_ext_i32(retv, retv, memop); 2846 } 2847 } 2848 } 2849 2850 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, 2851 TCGv_i64 newv, TCGArg idx, TCGMemOp memop) 2852 { 2853 memop = tcg_canonicalize_memop(memop, 1, 0); 2854 2855 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { 2856 TCGv_i64 t1 = tcg_temp_new_i64(); 2857 TCGv_i64 t2 = tcg_temp_new_i64(); 2858 2859 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); 2860 2861 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 2862 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); 2863 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 2864 tcg_temp_free_i64(t2); 2865 2866 if (memop & MO_SIGN) { 2867 tcg_gen_ext_i64(retv, t1, memop); 2868 } else { 2869 tcg_gen_mov_i64(retv, t1); 2870 } 2871 tcg_temp_free_i64(t1); 2872 } else if ((memop & MO_SIZE) == MO_64) { 2873 #ifdef CONFIG_ATOMIC64 2874 gen_atomic_cx_i64 gen; 2875 2876 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 2877 tcg_debug_assert(gen != NULL); 2878 2879 #ifdef CONFIG_SOFTMMU 2880 { 2881 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx)); 2882 gen(retv, cpu_env, addr, cmpv, newv, oi); 2883 tcg_temp_free_i32(oi); 2884 } 2885 #else 2886 gen(retv, cpu_env, addr, cmpv, newv); 2887 #endif 2888 #else 2889 gen_helper_exit_atomic(cpu_env); 2890 /* Produce a result, so that we have a well-formed opcode stream 2891 with respect to uses of the result in the (dead) code following. */ 2892 tcg_gen_movi_i64(retv, 0); 2893 #endif /* CONFIG_ATOMIC64 */ 2894 } else { 2895 TCGv_i32 c32 = tcg_temp_new_i32(); 2896 TCGv_i32 n32 = tcg_temp_new_i32(); 2897 TCGv_i32 r32 = tcg_temp_new_i32(); 2898 2899 tcg_gen_extrl_i64_i32(c32, cmpv); 2900 tcg_gen_extrl_i64_i32(n32, newv); 2901 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN); 2902 tcg_temp_free_i32(c32); 2903 tcg_temp_free_i32(n32); 2904 2905 tcg_gen_extu_i32_i64(retv, r32); 2906 tcg_temp_free_i32(r32); 2907 2908 if (memop & MO_SIGN) { 2909 tcg_gen_ext_i64(retv, retv, memop); 2910 } 2911 } 2912 } 2913 2914 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 2915 TCGArg idx, TCGMemOp memop, bool new_val, 2916 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) 2917 { 2918 TCGv_i32 t1 = tcg_temp_new_i32(); 2919 TCGv_i32 t2 = tcg_temp_new_i32(); 2920 2921 memop = tcg_canonicalize_memop(memop, 0, 0); 2922 2923 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 2924 gen(t2, t1, val); 2925 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 2926 2927 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); 2928 tcg_temp_free_i32(t1); 2929 tcg_temp_free_i32(t2); 2930 } 2931 2932 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 2933 TCGArg idx, TCGMemOp memop, void * const table[]) 2934 { 2935 gen_atomic_op_i32 gen; 2936 2937 memop = tcg_canonicalize_memop(memop, 0, 0); 2938 2939 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 2940 tcg_debug_assert(gen != NULL); 2941 2942 #ifdef CONFIG_SOFTMMU 2943 { 2944 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 2945 gen(ret, cpu_env, addr, val, oi); 2946 tcg_temp_free_i32(oi); 2947 } 2948 #else 2949 gen(ret, cpu_env, addr, val); 2950 #endif 2951 2952 if (memop & MO_SIGN) { 2953 tcg_gen_ext_i32(ret, ret, memop); 2954 } 2955 } 2956 2957 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 2958 TCGArg idx, TCGMemOp memop, bool new_val, 2959 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) 2960 { 2961 TCGv_i64 t1 = tcg_temp_new_i64(); 2962 TCGv_i64 t2 = tcg_temp_new_i64(); 2963 2964 memop = tcg_canonicalize_memop(memop, 1, 0); 2965 2966 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 2967 gen(t2, t1, val); 2968 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 2969 2970 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); 2971 tcg_temp_free_i64(t1); 2972 tcg_temp_free_i64(t2); 2973 } 2974 2975 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 2976 TCGArg idx, TCGMemOp memop, void * const table[]) 2977 { 2978 memop = tcg_canonicalize_memop(memop, 1, 0); 2979 2980 if ((memop & MO_SIZE) == MO_64) { 2981 #ifdef CONFIG_ATOMIC64 2982 gen_atomic_op_i64 gen; 2983 2984 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 2985 tcg_debug_assert(gen != NULL); 2986 2987 #ifdef CONFIG_SOFTMMU 2988 { 2989 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 2990 gen(ret, cpu_env, addr, val, oi); 2991 tcg_temp_free_i32(oi); 2992 } 2993 #else 2994 gen(ret, cpu_env, addr, val); 2995 #endif 2996 #else 2997 gen_helper_exit_atomic(cpu_env); 2998 /* Produce a result, so that we have a well-formed opcode stream 2999 with respect to uses of the result in the (dead) code following. */ 3000 tcg_gen_movi_i64(ret, 0); 3001 #endif /* CONFIG_ATOMIC64 */ 3002 } else { 3003 TCGv_i32 v32 = tcg_temp_new_i32(); 3004 TCGv_i32 r32 = tcg_temp_new_i32(); 3005 3006 tcg_gen_extrl_i64_i32(v32, val); 3007 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); 3008 tcg_temp_free_i32(v32); 3009 3010 tcg_gen_extu_i32_i64(ret, r32); 3011 tcg_temp_free_i32(r32); 3012 3013 if (memop & MO_SIGN) { 3014 tcg_gen_ext_i64(ret, ret, memop); 3015 } 3016 } 3017 } 3018 3019 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ 3020 static void * const table_##NAME[16] = { \ 3021 [MO_8] = gen_helper_atomic_##NAME##b, \ 3022 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ 3023 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ 3024 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ 3025 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ 3026 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ 3027 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ 3028 }; \ 3029 void tcg_gen_atomic_##NAME##_i32 \ 3030 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \ 3031 { \ 3032 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ 3033 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ 3034 } else { \ 3035 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ 3036 tcg_gen_##OP##_i32); \ 3037 } \ 3038 } \ 3039 void tcg_gen_atomic_##NAME##_i64 \ 3040 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \ 3041 { \ 3042 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ 3043 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ 3044 } else { \ 3045 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ 3046 tcg_gen_##OP##_i64); \ 3047 } \ 3048 } 3049 3050 GEN_ATOMIC_HELPER(fetch_add, add, 0) 3051 GEN_ATOMIC_HELPER(fetch_and, and, 0) 3052 GEN_ATOMIC_HELPER(fetch_or, or, 0) 3053 GEN_ATOMIC_HELPER(fetch_xor, xor, 0) 3054 GEN_ATOMIC_HELPER(fetch_smin, smin, 0) 3055 GEN_ATOMIC_HELPER(fetch_umin, umin, 0) 3056 GEN_ATOMIC_HELPER(fetch_smax, smax, 0) 3057 GEN_ATOMIC_HELPER(fetch_umax, umax, 0) 3058 3059 GEN_ATOMIC_HELPER(add_fetch, add, 1) 3060 GEN_ATOMIC_HELPER(and_fetch, and, 1) 3061 GEN_ATOMIC_HELPER(or_fetch, or, 1) 3062 GEN_ATOMIC_HELPER(xor_fetch, xor, 1) 3063 GEN_ATOMIC_HELPER(smin_fetch, smin, 1) 3064 GEN_ATOMIC_HELPER(umin_fetch, umin, 1) 3065 GEN_ATOMIC_HELPER(smax_fetch, smax, 1) 3066 GEN_ATOMIC_HELPER(umax_fetch, umax, 1) 3067 3068 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) 3069 { 3070 tcg_gen_mov_i32(r, b); 3071 } 3072 3073 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) 3074 { 3075 tcg_gen_mov_i64(r, b); 3076 } 3077 3078 GEN_ATOMIC_HELPER(xchg, mov2, 0) 3079 3080 #undef GEN_ATOMIC_HELPER 3081