1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "qemu-common.h" 27 #include "cpu.h" 28 #include "exec/exec-all.h" 29 #include "tcg.h" 30 #include "tcg-op.h" 31 #include "tcg-mo.h" 32 #include "trace-tcg.h" 33 #include "trace/mem.h" 34 35 /* Reduce the number of ifdefs below. This assumes that all uses of 36 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that 37 the compiler can eliminate. */ 38 #if TCG_TARGET_REG_BITS == 64 39 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); 40 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); 41 #define TCGV_LOW TCGV_LOW_link_error 42 #define TCGV_HIGH TCGV_HIGH_link_error 43 #endif 44 45 void tcg_gen_op1(TCGOpcode opc, TCGArg a1) 46 { 47 TCGOp *op = tcg_emit_op(opc); 48 op->args[0] = a1; 49 } 50 51 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2) 52 { 53 TCGOp *op = tcg_emit_op(opc); 54 op->args[0] = a1; 55 op->args[1] = a2; 56 } 57 58 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) 59 { 60 TCGOp *op = tcg_emit_op(opc); 61 op->args[0] = a1; 62 op->args[1] = a2; 63 op->args[2] = a3; 64 } 65 66 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4) 67 { 68 TCGOp *op = tcg_emit_op(opc); 69 op->args[0] = a1; 70 op->args[1] = a2; 71 op->args[2] = a3; 72 op->args[3] = a4; 73 } 74 75 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 76 TCGArg a4, TCGArg a5) 77 { 78 TCGOp *op = tcg_emit_op(opc); 79 op->args[0] = a1; 80 op->args[1] = a2; 81 op->args[2] = a3; 82 op->args[3] = a4; 83 op->args[4] = a5; 84 } 85 86 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 87 TCGArg a4, TCGArg a5, TCGArg a6) 88 { 89 TCGOp *op = tcg_emit_op(opc); 90 op->args[0] = a1; 91 op->args[1] = a2; 92 op->args[2] = a3; 93 op->args[3] = a4; 94 op->args[4] = a5; 95 op->args[5] = a6; 96 } 97 98 void tcg_gen_mb(TCGBar mb_type) 99 { 100 if (tcg_ctx->tb_cflags & CF_PARALLEL) { 101 tcg_gen_op1(INDEX_op_mb, mb_type); 102 } 103 } 104 105 /* 32 bit ops */ 106 107 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 108 { 109 /* some cases can be optimized here */ 110 if (arg2 == 0) { 111 tcg_gen_mov_i32(ret, arg1); 112 } else { 113 TCGv_i32 t0 = tcg_const_i32(arg2); 114 tcg_gen_add_i32(ret, arg1, t0); 115 tcg_temp_free_i32(t0); 116 } 117 } 118 119 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 120 { 121 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) { 122 /* Don't recurse with tcg_gen_neg_i32. */ 123 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2); 124 } else { 125 TCGv_i32 t0 = tcg_const_i32(arg1); 126 tcg_gen_sub_i32(ret, t0, arg2); 127 tcg_temp_free_i32(t0); 128 } 129 } 130 131 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 132 { 133 /* some cases can be optimized here */ 134 if (arg2 == 0) { 135 tcg_gen_mov_i32(ret, arg1); 136 } else { 137 TCGv_i32 t0 = tcg_const_i32(arg2); 138 tcg_gen_sub_i32(ret, arg1, t0); 139 tcg_temp_free_i32(t0); 140 } 141 } 142 143 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 144 { 145 TCGv_i32 t0; 146 /* Some cases can be optimized here. */ 147 switch (arg2) { 148 case 0: 149 tcg_gen_movi_i32(ret, 0); 150 return; 151 case -1: 152 tcg_gen_mov_i32(ret, arg1); 153 return; 154 case 0xff: 155 /* Don't recurse with tcg_gen_ext8u_i32. */ 156 if (TCG_TARGET_HAS_ext8u_i32) { 157 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1); 158 return; 159 } 160 break; 161 case 0xffff: 162 if (TCG_TARGET_HAS_ext16u_i32) { 163 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1); 164 return; 165 } 166 break; 167 } 168 t0 = tcg_const_i32(arg2); 169 tcg_gen_and_i32(ret, arg1, t0); 170 tcg_temp_free_i32(t0); 171 } 172 173 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 174 { 175 /* Some cases can be optimized here. */ 176 if (arg2 == -1) { 177 tcg_gen_movi_i32(ret, -1); 178 } else if (arg2 == 0) { 179 tcg_gen_mov_i32(ret, arg1); 180 } else { 181 TCGv_i32 t0 = tcg_const_i32(arg2); 182 tcg_gen_or_i32(ret, arg1, t0); 183 tcg_temp_free_i32(t0); 184 } 185 } 186 187 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 188 { 189 /* Some cases can be optimized here. */ 190 if (arg2 == 0) { 191 tcg_gen_mov_i32(ret, arg1); 192 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) { 193 /* Don't recurse with tcg_gen_not_i32. */ 194 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1); 195 } else { 196 TCGv_i32 t0 = tcg_const_i32(arg2); 197 tcg_gen_xor_i32(ret, arg1, t0); 198 tcg_temp_free_i32(t0); 199 } 200 } 201 202 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 203 { 204 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 205 if (arg2 == 0) { 206 tcg_gen_mov_i32(ret, arg1); 207 } else { 208 TCGv_i32 t0 = tcg_const_i32(arg2); 209 tcg_gen_shl_i32(ret, arg1, t0); 210 tcg_temp_free_i32(t0); 211 } 212 } 213 214 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 215 { 216 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 217 if (arg2 == 0) { 218 tcg_gen_mov_i32(ret, arg1); 219 } else { 220 TCGv_i32 t0 = tcg_const_i32(arg2); 221 tcg_gen_shr_i32(ret, arg1, t0); 222 tcg_temp_free_i32(t0); 223 } 224 } 225 226 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 227 { 228 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 229 if (arg2 == 0) { 230 tcg_gen_mov_i32(ret, arg1); 231 } else { 232 TCGv_i32 t0 = tcg_const_i32(arg2); 233 tcg_gen_sar_i32(ret, arg1, t0); 234 tcg_temp_free_i32(t0); 235 } 236 } 237 238 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 239 { 240 if (cond == TCG_COND_ALWAYS) { 241 tcg_gen_br(l); 242 } else if (cond != TCG_COND_NEVER) { 243 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l)); 244 } 245 } 246 247 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 248 { 249 if (cond == TCG_COND_ALWAYS) { 250 tcg_gen_br(l); 251 } else if (cond != TCG_COND_NEVER) { 252 TCGv_i32 t0 = tcg_const_i32(arg2); 253 tcg_gen_brcond_i32(cond, arg1, t0, l); 254 tcg_temp_free_i32(t0); 255 } 256 } 257 258 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 259 TCGv_i32 arg1, TCGv_i32 arg2) 260 { 261 if (cond == TCG_COND_ALWAYS) { 262 tcg_gen_movi_i32(ret, 1); 263 } else if (cond == TCG_COND_NEVER) { 264 tcg_gen_movi_i32(ret, 0); 265 } else { 266 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 267 } 268 } 269 270 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 271 TCGv_i32 arg1, int32_t arg2) 272 { 273 TCGv_i32 t0 = tcg_const_i32(arg2); 274 tcg_gen_setcond_i32(cond, ret, arg1, t0); 275 tcg_temp_free_i32(t0); 276 } 277 278 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 279 { 280 if (arg2 == 0) { 281 tcg_gen_movi_i32(ret, 0); 282 } else if (is_power_of_2(arg2)) { 283 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 284 } else { 285 TCGv_i32 t0 = tcg_const_i32(arg2); 286 tcg_gen_mul_i32(ret, arg1, t0); 287 tcg_temp_free_i32(t0); 288 } 289 } 290 291 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 292 { 293 if (TCG_TARGET_HAS_div_i32) { 294 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2); 295 } else if (TCG_TARGET_HAS_div2_i32) { 296 TCGv_i32 t0 = tcg_temp_new_i32(); 297 tcg_gen_sari_i32(t0, arg1, 31); 298 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2); 299 tcg_temp_free_i32(t0); 300 } else { 301 gen_helper_div_i32(ret, arg1, arg2); 302 } 303 } 304 305 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 306 { 307 if (TCG_TARGET_HAS_rem_i32) { 308 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2); 309 } else if (TCG_TARGET_HAS_div_i32) { 310 TCGv_i32 t0 = tcg_temp_new_i32(); 311 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2); 312 tcg_gen_mul_i32(t0, t0, arg2); 313 tcg_gen_sub_i32(ret, arg1, t0); 314 tcg_temp_free_i32(t0); 315 } else if (TCG_TARGET_HAS_div2_i32) { 316 TCGv_i32 t0 = tcg_temp_new_i32(); 317 tcg_gen_sari_i32(t0, arg1, 31); 318 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2); 319 tcg_temp_free_i32(t0); 320 } else { 321 gen_helper_rem_i32(ret, arg1, arg2); 322 } 323 } 324 325 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 326 { 327 if (TCG_TARGET_HAS_div_i32) { 328 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2); 329 } else if (TCG_TARGET_HAS_div2_i32) { 330 TCGv_i32 t0 = tcg_temp_new_i32(); 331 tcg_gen_movi_i32(t0, 0); 332 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2); 333 tcg_temp_free_i32(t0); 334 } else { 335 gen_helper_divu_i32(ret, arg1, arg2); 336 } 337 } 338 339 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 340 { 341 if (TCG_TARGET_HAS_rem_i32) { 342 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2); 343 } else if (TCG_TARGET_HAS_div_i32) { 344 TCGv_i32 t0 = tcg_temp_new_i32(); 345 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2); 346 tcg_gen_mul_i32(t0, t0, arg2); 347 tcg_gen_sub_i32(ret, arg1, t0); 348 tcg_temp_free_i32(t0); 349 } else if (TCG_TARGET_HAS_div2_i32) { 350 TCGv_i32 t0 = tcg_temp_new_i32(); 351 tcg_gen_movi_i32(t0, 0); 352 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2); 353 tcg_temp_free_i32(t0); 354 } else { 355 gen_helper_remu_i32(ret, arg1, arg2); 356 } 357 } 358 359 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 360 { 361 if (TCG_TARGET_HAS_andc_i32) { 362 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2); 363 } else { 364 TCGv_i32 t0 = tcg_temp_new_i32(); 365 tcg_gen_not_i32(t0, arg2); 366 tcg_gen_and_i32(ret, arg1, t0); 367 tcg_temp_free_i32(t0); 368 } 369 } 370 371 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 372 { 373 if (TCG_TARGET_HAS_eqv_i32) { 374 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2); 375 } else { 376 tcg_gen_xor_i32(ret, arg1, arg2); 377 tcg_gen_not_i32(ret, ret); 378 } 379 } 380 381 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 382 { 383 if (TCG_TARGET_HAS_nand_i32) { 384 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2); 385 } else { 386 tcg_gen_and_i32(ret, arg1, arg2); 387 tcg_gen_not_i32(ret, ret); 388 } 389 } 390 391 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 392 { 393 if (TCG_TARGET_HAS_nor_i32) { 394 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2); 395 } else { 396 tcg_gen_or_i32(ret, arg1, arg2); 397 tcg_gen_not_i32(ret, ret); 398 } 399 } 400 401 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 402 { 403 if (TCG_TARGET_HAS_orc_i32) { 404 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2); 405 } else { 406 TCGv_i32 t0 = tcg_temp_new_i32(); 407 tcg_gen_not_i32(t0, arg2); 408 tcg_gen_or_i32(ret, arg1, t0); 409 tcg_temp_free_i32(t0); 410 } 411 } 412 413 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 414 { 415 if (TCG_TARGET_HAS_clz_i32) { 416 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2); 417 } else if (TCG_TARGET_HAS_clz_i64) { 418 TCGv_i64 t1 = tcg_temp_new_i64(); 419 TCGv_i64 t2 = tcg_temp_new_i64(); 420 tcg_gen_extu_i32_i64(t1, arg1); 421 tcg_gen_extu_i32_i64(t2, arg2); 422 tcg_gen_addi_i64(t2, t2, 32); 423 tcg_gen_clz_i64(t1, t1, t2); 424 tcg_gen_extrl_i64_i32(ret, t1); 425 tcg_temp_free_i64(t1); 426 tcg_temp_free_i64(t2); 427 tcg_gen_subi_i32(ret, ret, 32); 428 } else { 429 gen_helper_clz_i32(ret, arg1, arg2); 430 } 431 } 432 433 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 434 { 435 TCGv_i32 t = tcg_const_i32(arg2); 436 tcg_gen_clz_i32(ret, arg1, t); 437 tcg_temp_free_i32(t); 438 } 439 440 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 441 { 442 if (TCG_TARGET_HAS_ctz_i32) { 443 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); 444 } else if (TCG_TARGET_HAS_ctz_i64) { 445 TCGv_i64 t1 = tcg_temp_new_i64(); 446 TCGv_i64 t2 = tcg_temp_new_i64(); 447 tcg_gen_extu_i32_i64(t1, arg1); 448 tcg_gen_extu_i32_i64(t2, arg2); 449 tcg_gen_ctz_i64(t1, t1, t2); 450 tcg_gen_extrl_i64_i32(ret, t1); 451 tcg_temp_free_i64(t1); 452 tcg_temp_free_i64(t2); 453 } else if (TCG_TARGET_HAS_ctpop_i32 454 || TCG_TARGET_HAS_ctpop_i64 455 || TCG_TARGET_HAS_clz_i32 456 || TCG_TARGET_HAS_clz_i64) { 457 TCGv_i32 z, t = tcg_temp_new_i32(); 458 459 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 460 tcg_gen_subi_i32(t, arg1, 1); 461 tcg_gen_andc_i32(t, t, arg1); 462 tcg_gen_ctpop_i32(t, t); 463 } else { 464 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */ 465 tcg_gen_neg_i32(t, arg1); 466 tcg_gen_and_i32(t, t, arg1); 467 tcg_gen_clzi_i32(t, t, 32); 468 tcg_gen_xori_i32(t, t, 31); 469 } 470 z = tcg_const_i32(0); 471 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 472 tcg_temp_free_i32(t); 473 tcg_temp_free_i32(z); 474 } else { 475 gen_helper_ctz_i32(ret, arg1, arg2); 476 } 477 } 478 479 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 480 { 481 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 482 /* This equivalence has the advantage of not requiring a fixup. */ 483 TCGv_i32 t = tcg_temp_new_i32(); 484 tcg_gen_subi_i32(t, arg1, 1); 485 tcg_gen_andc_i32(t, t, arg1); 486 tcg_gen_ctpop_i32(ret, t); 487 tcg_temp_free_i32(t); 488 } else { 489 TCGv_i32 t = tcg_const_i32(arg2); 490 tcg_gen_ctz_i32(ret, arg1, t); 491 tcg_temp_free_i32(t); 492 } 493 } 494 495 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 496 { 497 if (TCG_TARGET_HAS_clz_i32) { 498 TCGv_i32 t = tcg_temp_new_i32(); 499 tcg_gen_sari_i32(t, arg, 31); 500 tcg_gen_xor_i32(t, t, arg); 501 tcg_gen_clzi_i32(t, t, 32); 502 tcg_gen_subi_i32(ret, t, 1); 503 tcg_temp_free_i32(t); 504 } else { 505 gen_helper_clrsb_i32(ret, arg); 506 } 507 } 508 509 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 510 { 511 if (TCG_TARGET_HAS_ctpop_i32) { 512 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 513 } else if (TCG_TARGET_HAS_ctpop_i64) { 514 TCGv_i64 t = tcg_temp_new_i64(); 515 tcg_gen_extu_i32_i64(t, arg1); 516 tcg_gen_ctpop_i64(t, t); 517 tcg_gen_extrl_i64_i32(ret, t); 518 tcg_temp_free_i64(t); 519 } else { 520 gen_helper_ctpop_i32(ret, arg1); 521 } 522 } 523 524 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 525 { 526 if (TCG_TARGET_HAS_rot_i32) { 527 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2); 528 } else { 529 TCGv_i32 t0, t1; 530 531 t0 = tcg_temp_new_i32(); 532 t1 = tcg_temp_new_i32(); 533 tcg_gen_shl_i32(t0, arg1, arg2); 534 tcg_gen_subfi_i32(t1, 32, arg2); 535 tcg_gen_shr_i32(t1, arg1, t1); 536 tcg_gen_or_i32(ret, t0, t1); 537 tcg_temp_free_i32(t0); 538 tcg_temp_free_i32(t1); 539 } 540 } 541 542 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 543 { 544 tcg_debug_assert(arg2 < 32); 545 /* some cases can be optimized here */ 546 if (arg2 == 0) { 547 tcg_gen_mov_i32(ret, arg1); 548 } else if (TCG_TARGET_HAS_rot_i32) { 549 TCGv_i32 t0 = tcg_const_i32(arg2); 550 tcg_gen_rotl_i32(ret, arg1, t0); 551 tcg_temp_free_i32(t0); 552 } else { 553 TCGv_i32 t0, t1; 554 t0 = tcg_temp_new_i32(); 555 t1 = tcg_temp_new_i32(); 556 tcg_gen_shli_i32(t0, arg1, arg2); 557 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 558 tcg_gen_or_i32(ret, t0, t1); 559 tcg_temp_free_i32(t0); 560 tcg_temp_free_i32(t1); 561 } 562 } 563 564 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 565 { 566 if (TCG_TARGET_HAS_rot_i32) { 567 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2); 568 } else { 569 TCGv_i32 t0, t1; 570 571 t0 = tcg_temp_new_i32(); 572 t1 = tcg_temp_new_i32(); 573 tcg_gen_shr_i32(t0, arg1, arg2); 574 tcg_gen_subfi_i32(t1, 32, arg2); 575 tcg_gen_shl_i32(t1, arg1, t1); 576 tcg_gen_or_i32(ret, t0, t1); 577 tcg_temp_free_i32(t0); 578 tcg_temp_free_i32(t1); 579 } 580 } 581 582 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 583 { 584 tcg_debug_assert(arg2 < 32); 585 /* some cases can be optimized here */ 586 if (arg2 == 0) { 587 tcg_gen_mov_i32(ret, arg1); 588 } else { 589 tcg_gen_rotli_i32(ret, arg1, 32 - arg2); 590 } 591 } 592 593 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 594 unsigned int ofs, unsigned int len) 595 { 596 uint32_t mask; 597 TCGv_i32 t1; 598 599 tcg_debug_assert(ofs < 32); 600 tcg_debug_assert(len > 0); 601 tcg_debug_assert(len <= 32); 602 tcg_debug_assert(ofs + len <= 32); 603 604 if (len == 32) { 605 tcg_gen_mov_i32(ret, arg2); 606 return; 607 } 608 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 609 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 610 return; 611 } 612 613 mask = (1u << len) - 1; 614 t1 = tcg_temp_new_i32(); 615 616 if (ofs + len < 32) { 617 tcg_gen_andi_i32(t1, arg2, mask); 618 tcg_gen_shli_i32(t1, t1, ofs); 619 } else { 620 tcg_gen_shli_i32(t1, arg2, ofs); 621 } 622 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 623 tcg_gen_or_i32(ret, ret, t1); 624 625 tcg_temp_free_i32(t1); 626 } 627 628 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 629 unsigned int ofs, unsigned int len) 630 { 631 tcg_debug_assert(ofs < 32); 632 tcg_debug_assert(len > 0); 633 tcg_debug_assert(len <= 32); 634 tcg_debug_assert(ofs + len <= 32); 635 636 if (ofs + len == 32) { 637 tcg_gen_shli_i32(ret, arg, ofs); 638 } else if (ofs == 0) { 639 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 640 } else if (TCG_TARGET_HAS_deposit_i32 641 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 642 TCGv_i32 zero = tcg_const_i32(0); 643 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 644 tcg_temp_free_i32(zero); 645 } else { 646 /* To help two-operand hosts we prefer to zero-extend first, 647 which allows ARG to stay live. */ 648 switch (len) { 649 case 16: 650 if (TCG_TARGET_HAS_ext16u_i32) { 651 tcg_gen_ext16u_i32(ret, arg); 652 tcg_gen_shli_i32(ret, ret, ofs); 653 return; 654 } 655 break; 656 case 8: 657 if (TCG_TARGET_HAS_ext8u_i32) { 658 tcg_gen_ext8u_i32(ret, arg); 659 tcg_gen_shli_i32(ret, ret, ofs); 660 return; 661 } 662 break; 663 } 664 /* Otherwise prefer zero-extension over AND for code size. */ 665 switch (ofs + len) { 666 case 16: 667 if (TCG_TARGET_HAS_ext16u_i32) { 668 tcg_gen_shli_i32(ret, arg, ofs); 669 tcg_gen_ext16u_i32(ret, ret); 670 return; 671 } 672 break; 673 case 8: 674 if (TCG_TARGET_HAS_ext8u_i32) { 675 tcg_gen_shli_i32(ret, arg, ofs); 676 tcg_gen_ext8u_i32(ret, ret); 677 return; 678 } 679 break; 680 } 681 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 682 tcg_gen_shli_i32(ret, ret, ofs); 683 } 684 } 685 686 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 687 unsigned int ofs, unsigned int len) 688 { 689 tcg_debug_assert(ofs < 32); 690 tcg_debug_assert(len > 0); 691 tcg_debug_assert(len <= 32); 692 tcg_debug_assert(ofs + len <= 32); 693 694 /* Canonicalize certain special cases, even if extract is supported. */ 695 if (ofs + len == 32) { 696 tcg_gen_shri_i32(ret, arg, 32 - len); 697 return; 698 } 699 if (ofs == 0) { 700 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 701 return; 702 } 703 704 if (TCG_TARGET_HAS_extract_i32 705 && TCG_TARGET_extract_i32_valid(ofs, len)) { 706 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 707 return; 708 } 709 710 /* Assume that zero-extension, if available, is cheaper than a shift. */ 711 switch (ofs + len) { 712 case 16: 713 if (TCG_TARGET_HAS_ext16u_i32) { 714 tcg_gen_ext16u_i32(ret, arg); 715 tcg_gen_shri_i32(ret, ret, ofs); 716 return; 717 } 718 break; 719 case 8: 720 if (TCG_TARGET_HAS_ext8u_i32) { 721 tcg_gen_ext8u_i32(ret, arg); 722 tcg_gen_shri_i32(ret, ret, ofs); 723 return; 724 } 725 break; 726 } 727 728 /* ??? Ideally we'd know what values are available for immediate AND. 729 Assume that 8 bits are available, plus the special case of 16, 730 so that we get ext8u, ext16u. */ 731 switch (len) { 732 case 1 ... 8: case 16: 733 tcg_gen_shri_i32(ret, arg, ofs); 734 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 735 break; 736 default: 737 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 738 tcg_gen_shri_i32(ret, ret, 32 - len); 739 break; 740 } 741 } 742 743 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 744 unsigned int ofs, unsigned int len) 745 { 746 tcg_debug_assert(ofs < 32); 747 tcg_debug_assert(len > 0); 748 tcg_debug_assert(len <= 32); 749 tcg_debug_assert(ofs + len <= 32); 750 751 /* Canonicalize certain special cases, even if extract is supported. */ 752 if (ofs + len == 32) { 753 tcg_gen_sari_i32(ret, arg, 32 - len); 754 return; 755 } 756 if (ofs == 0) { 757 switch (len) { 758 case 16: 759 tcg_gen_ext16s_i32(ret, arg); 760 return; 761 case 8: 762 tcg_gen_ext8s_i32(ret, arg); 763 return; 764 } 765 } 766 767 if (TCG_TARGET_HAS_sextract_i32 768 && TCG_TARGET_extract_i32_valid(ofs, len)) { 769 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 770 return; 771 } 772 773 /* Assume that sign-extension, if available, is cheaper than a shift. */ 774 switch (ofs + len) { 775 case 16: 776 if (TCG_TARGET_HAS_ext16s_i32) { 777 tcg_gen_ext16s_i32(ret, arg); 778 tcg_gen_sari_i32(ret, ret, ofs); 779 return; 780 } 781 break; 782 case 8: 783 if (TCG_TARGET_HAS_ext8s_i32) { 784 tcg_gen_ext8s_i32(ret, arg); 785 tcg_gen_sari_i32(ret, ret, ofs); 786 return; 787 } 788 break; 789 } 790 switch (len) { 791 case 16: 792 if (TCG_TARGET_HAS_ext16s_i32) { 793 tcg_gen_shri_i32(ret, arg, ofs); 794 tcg_gen_ext16s_i32(ret, ret); 795 return; 796 } 797 break; 798 case 8: 799 if (TCG_TARGET_HAS_ext8s_i32) { 800 tcg_gen_shri_i32(ret, arg, ofs); 801 tcg_gen_ext8s_i32(ret, ret); 802 return; 803 } 804 break; 805 } 806 807 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 808 tcg_gen_sari_i32(ret, ret, 32 - len); 809 } 810 811 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 812 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 813 { 814 if (cond == TCG_COND_ALWAYS) { 815 tcg_gen_mov_i32(ret, v1); 816 } else if (cond == TCG_COND_NEVER) { 817 tcg_gen_mov_i32(ret, v2); 818 } else if (TCG_TARGET_HAS_movcond_i32) { 819 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 820 } else { 821 TCGv_i32 t0 = tcg_temp_new_i32(); 822 TCGv_i32 t1 = tcg_temp_new_i32(); 823 tcg_gen_setcond_i32(cond, t0, c1, c2); 824 tcg_gen_neg_i32(t0, t0); 825 tcg_gen_and_i32(t1, v1, t0); 826 tcg_gen_andc_i32(ret, v2, t0); 827 tcg_gen_or_i32(ret, ret, t1); 828 tcg_temp_free_i32(t0); 829 tcg_temp_free_i32(t1); 830 } 831 } 832 833 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 834 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 835 { 836 if (TCG_TARGET_HAS_add2_i32) { 837 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 838 } else { 839 TCGv_i64 t0 = tcg_temp_new_i64(); 840 TCGv_i64 t1 = tcg_temp_new_i64(); 841 tcg_gen_concat_i32_i64(t0, al, ah); 842 tcg_gen_concat_i32_i64(t1, bl, bh); 843 tcg_gen_add_i64(t0, t0, t1); 844 tcg_gen_extr_i64_i32(rl, rh, t0); 845 tcg_temp_free_i64(t0); 846 tcg_temp_free_i64(t1); 847 } 848 } 849 850 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 851 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 852 { 853 if (TCG_TARGET_HAS_sub2_i32) { 854 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 855 } else { 856 TCGv_i64 t0 = tcg_temp_new_i64(); 857 TCGv_i64 t1 = tcg_temp_new_i64(); 858 tcg_gen_concat_i32_i64(t0, al, ah); 859 tcg_gen_concat_i32_i64(t1, bl, bh); 860 tcg_gen_sub_i64(t0, t0, t1); 861 tcg_gen_extr_i64_i32(rl, rh, t0); 862 tcg_temp_free_i64(t0); 863 tcg_temp_free_i64(t1); 864 } 865 } 866 867 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 868 { 869 if (TCG_TARGET_HAS_mulu2_i32) { 870 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 871 } else if (TCG_TARGET_HAS_muluh_i32) { 872 TCGv_i32 t = tcg_temp_new_i32(); 873 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 874 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2); 875 tcg_gen_mov_i32(rl, t); 876 tcg_temp_free_i32(t); 877 } else { 878 TCGv_i64 t0 = tcg_temp_new_i64(); 879 TCGv_i64 t1 = tcg_temp_new_i64(); 880 tcg_gen_extu_i32_i64(t0, arg1); 881 tcg_gen_extu_i32_i64(t1, arg2); 882 tcg_gen_mul_i64(t0, t0, t1); 883 tcg_gen_extr_i64_i32(rl, rh, t0); 884 tcg_temp_free_i64(t0); 885 tcg_temp_free_i64(t1); 886 } 887 } 888 889 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 890 { 891 if (TCG_TARGET_HAS_muls2_i32) { 892 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 893 } else if (TCG_TARGET_HAS_mulsh_i32) { 894 TCGv_i32 t = tcg_temp_new_i32(); 895 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 896 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2); 897 tcg_gen_mov_i32(rl, t); 898 tcg_temp_free_i32(t); 899 } else if (TCG_TARGET_REG_BITS == 32) { 900 TCGv_i32 t0 = tcg_temp_new_i32(); 901 TCGv_i32 t1 = tcg_temp_new_i32(); 902 TCGv_i32 t2 = tcg_temp_new_i32(); 903 TCGv_i32 t3 = tcg_temp_new_i32(); 904 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 905 /* Adjust for negative inputs. */ 906 tcg_gen_sari_i32(t2, arg1, 31); 907 tcg_gen_sari_i32(t3, arg2, 31); 908 tcg_gen_and_i32(t2, t2, arg2); 909 tcg_gen_and_i32(t3, t3, arg1); 910 tcg_gen_sub_i32(rh, t1, t2); 911 tcg_gen_sub_i32(rh, rh, t3); 912 tcg_gen_mov_i32(rl, t0); 913 tcg_temp_free_i32(t0); 914 tcg_temp_free_i32(t1); 915 tcg_temp_free_i32(t2); 916 tcg_temp_free_i32(t3); 917 } else { 918 TCGv_i64 t0 = tcg_temp_new_i64(); 919 TCGv_i64 t1 = tcg_temp_new_i64(); 920 tcg_gen_ext_i32_i64(t0, arg1); 921 tcg_gen_ext_i32_i64(t1, arg2); 922 tcg_gen_mul_i64(t0, t0, t1); 923 tcg_gen_extr_i64_i32(rl, rh, t0); 924 tcg_temp_free_i64(t0); 925 tcg_temp_free_i64(t1); 926 } 927 } 928 929 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 930 { 931 if (TCG_TARGET_REG_BITS == 32) { 932 TCGv_i32 t0 = tcg_temp_new_i32(); 933 TCGv_i32 t1 = tcg_temp_new_i32(); 934 TCGv_i32 t2 = tcg_temp_new_i32(); 935 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 936 /* Adjust for negative input for the signed arg1. */ 937 tcg_gen_sari_i32(t2, arg1, 31); 938 tcg_gen_and_i32(t2, t2, arg2); 939 tcg_gen_sub_i32(rh, t1, t2); 940 tcg_gen_mov_i32(rl, t0); 941 tcg_temp_free_i32(t0); 942 tcg_temp_free_i32(t1); 943 tcg_temp_free_i32(t2); 944 } else { 945 TCGv_i64 t0 = tcg_temp_new_i64(); 946 TCGv_i64 t1 = tcg_temp_new_i64(); 947 tcg_gen_ext_i32_i64(t0, arg1); 948 tcg_gen_extu_i32_i64(t1, arg2); 949 tcg_gen_mul_i64(t0, t0, t1); 950 tcg_gen_extr_i64_i32(rl, rh, t0); 951 tcg_temp_free_i64(t0); 952 tcg_temp_free_i64(t1); 953 } 954 } 955 956 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 957 { 958 if (TCG_TARGET_HAS_ext8s_i32) { 959 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg); 960 } else { 961 tcg_gen_shli_i32(ret, arg, 24); 962 tcg_gen_sari_i32(ret, ret, 24); 963 } 964 } 965 966 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 967 { 968 if (TCG_TARGET_HAS_ext16s_i32) { 969 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg); 970 } else { 971 tcg_gen_shli_i32(ret, arg, 16); 972 tcg_gen_sari_i32(ret, ret, 16); 973 } 974 } 975 976 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 977 { 978 if (TCG_TARGET_HAS_ext8u_i32) { 979 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg); 980 } else { 981 tcg_gen_andi_i32(ret, arg, 0xffu); 982 } 983 } 984 985 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 986 { 987 if (TCG_TARGET_HAS_ext16u_i32) { 988 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg); 989 } else { 990 tcg_gen_andi_i32(ret, arg, 0xffffu); 991 } 992 } 993 994 /* Note: we assume the two high bytes are set to zero */ 995 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg) 996 { 997 if (TCG_TARGET_HAS_bswap16_i32) { 998 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg); 999 } else { 1000 TCGv_i32 t0 = tcg_temp_new_i32(); 1001 1002 tcg_gen_ext8u_i32(t0, arg); 1003 tcg_gen_shli_i32(t0, t0, 8); 1004 tcg_gen_shri_i32(ret, arg, 8); 1005 tcg_gen_or_i32(ret, ret, t0); 1006 tcg_temp_free_i32(t0); 1007 } 1008 } 1009 1010 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1011 { 1012 if (TCG_TARGET_HAS_bswap32_i32) { 1013 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg); 1014 } else { 1015 TCGv_i32 t0, t1; 1016 t0 = tcg_temp_new_i32(); 1017 t1 = tcg_temp_new_i32(); 1018 1019 tcg_gen_shli_i32(t0, arg, 24); 1020 1021 tcg_gen_andi_i32(t1, arg, 0x0000ff00); 1022 tcg_gen_shli_i32(t1, t1, 8); 1023 tcg_gen_or_i32(t0, t0, t1); 1024 1025 tcg_gen_shri_i32(t1, arg, 8); 1026 tcg_gen_andi_i32(t1, t1, 0x0000ff00); 1027 tcg_gen_or_i32(t0, t0, t1); 1028 1029 tcg_gen_shri_i32(t1, arg, 24); 1030 tcg_gen_or_i32(ret, t0, t1); 1031 tcg_temp_free_i32(t0); 1032 tcg_temp_free_i32(t1); 1033 } 1034 } 1035 1036 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1037 { 1038 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1039 } 1040 1041 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1042 { 1043 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1044 } 1045 1046 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1047 { 1048 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1049 } 1050 1051 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1052 { 1053 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1054 } 1055 1056 /* 64-bit ops */ 1057 1058 #if TCG_TARGET_REG_BITS == 32 1059 /* These are all inline for TCG_TARGET_REG_BITS == 64. */ 1060 1061 void tcg_gen_discard_i64(TCGv_i64 arg) 1062 { 1063 tcg_gen_discard_i32(TCGV_LOW(arg)); 1064 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1065 } 1066 1067 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1068 { 1069 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1070 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1071 } 1072 1073 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1074 { 1075 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1076 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1077 } 1078 1079 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1080 { 1081 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1082 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1083 } 1084 1085 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1086 { 1087 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1088 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1089 } 1090 1091 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1092 { 1093 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1094 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1095 } 1096 1097 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1098 { 1099 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1100 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1101 } 1102 1103 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1104 { 1105 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1106 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1107 } 1108 1109 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1110 { 1111 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1112 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1113 } 1114 1115 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1116 { 1117 /* Since arg2 and ret have different types, 1118 they cannot be the same temporary */ 1119 #ifdef HOST_WORDS_BIGENDIAN 1120 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1121 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1122 #else 1123 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1124 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1125 #endif 1126 } 1127 1128 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1129 { 1130 #ifdef HOST_WORDS_BIGENDIAN 1131 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1132 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1133 #else 1134 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1135 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1136 #endif 1137 } 1138 1139 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1140 { 1141 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1142 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1143 } 1144 1145 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1146 { 1147 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1148 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1149 } 1150 1151 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1152 { 1153 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1154 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1155 } 1156 1157 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1158 { 1159 gen_helper_shl_i64(ret, arg1, arg2); 1160 } 1161 1162 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1163 { 1164 gen_helper_shr_i64(ret, arg1, arg2); 1165 } 1166 1167 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1168 { 1169 gen_helper_sar_i64(ret, arg1, arg2); 1170 } 1171 1172 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1173 { 1174 TCGv_i64 t0; 1175 TCGv_i32 t1; 1176 1177 t0 = tcg_temp_new_i64(); 1178 t1 = tcg_temp_new_i32(); 1179 1180 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1181 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1182 1183 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1184 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1185 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1186 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1187 1188 tcg_gen_mov_i64(ret, t0); 1189 tcg_temp_free_i64(t0); 1190 tcg_temp_free_i32(t1); 1191 } 1192 #endif /* TCG_TARGET_REG_SIZE == 32 */ 1193 1194 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1195 { 1196 /* some cases can be optimized here */ 1197 if (arg2 == 0) { 1198 tcg_gen_mov_i64(ret, arg1); 1199 } else { 1200 TCGv_i64 t0 = tcg_const_i64(arg2); 1201 tcg_gen_add_i64(ret, arg1, t0); 1202 tcg_temp_free_i64(t0); 1203 } 1204 } 1205 1206 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1207 { 1208 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) { 1209 /* Don't recurse with tcg_gen_neg_i64. */ 1210 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2); 1211 } else { 1212 TCGv_i64 t0 = tcg_const_i64(arg1); 1213 tcg_gen_sub_i64(ret, t0, arg2); 1214 tcg_temp_free_i64(t0); 1215 } 1216 } 1217 1218 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1219 { 1220 /* some cases can be optimized here */ 1221 if (arg2 == 0) { 1222 tcg_gen_mov_i64(ret, arg1); 1223 } else { 1224 TCGv_i64 t0 = tcg_const_i64(arg2); 1225 tcg_gen_sub_i64(ret, arg1, t0); 1226 tcg_temp_free_i64(t0); 1227 } 1228 } 1229 1230 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1231 { 1232 TCGv_i64 t0; 1233 1234 if (TCG_TARGET_REG_BITS == 32) { 1235 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1236 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1237 return; 1238 } 1239 1240 /* Some cases can be optimized here. */ 1241 switch (arg2) { 1242 case 0: 1243 tcg_gen_movi_i64(ret, 0); 1244 return; 1245 case -1: 1246 tcg_gen_mov_i64(ret, arg1); 1247 return; 1248 case 0xff: 1249 /* Don't recurse with tcg_gen_ext8u_i64. */ 1250 if (TCG_TARGET_HAS_ext8u_i64) { 1251 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1); 1252 return; 1253 } 1254 break; 1255 case 0xffff: 1256 if (TCG_TARGET_HAS_ext16u_i64) { 1257 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1); 1258 return; 1259 } 1260 break; 1261 case 0xffffffffu: 1262 if (TCG_TARGET_HAS_ext32u_i64) { 1263 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1); 1264 return; 1265 } 1266 break; 1267 } 1268 t0 = tcg_const_i64(arg2); 1269 tcg_gen_and_i64(ret, arg1, t0); 1270 tcg_temp_free_i64(t0); 1271 } 1272 1273 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1274 { 1275 if (TCG_TARGET_REG_BITS == 32) { 1276 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1277 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1278 return; 1279 } 1280 /* Some cases can be optimized here. */ 1281 if (arg2 == -1) { 1282 tcg_gen_movi_i64(ret, -1); 1283 } else if (arg2 == 0) { 1284 tcg_gen_mov_i64(ret, arg1); 1285 } else { 1286 TCGv_i64 t0 = tcg_const_i64(arg2); 1287 tcg_gen_or_i64(ret, arg1, t0); 1288 tcg_temp_free_i64(t0); 1289 } 1290 } 1291 1292 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1293 { 1294 if (TCG_TARGET_REG_BITS == 32) { 1295 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1296 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1297 return; 1298 } 1299 /* Some cases can be optimized here. */ 1300 if (arg2 == 0) { 1301 tcg_gen_mov_i64(ret, arg1); 1302 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) { 1303 /* Don't recurse with tcg_gen_not_i64. */ 1304 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1); 1305 } else { 1306 TCGv_i64 t0 = tcg_const_i64(arg2); 1307 tcg_gen_xor_i64(ret, arg1, t0); 1308 tcg_temp_free_i64(t0); 1309 } 1310 } 1311 1312 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1313 unsigned c, bool right, bool arith) 1314 { 1315 tcg_debug_assert(c < 64); 1316 if (c == 0) { 1317 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1318 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1319 } else if (c >= 32) { 1320 c -= 32; 1321 if (right) { 1322 if (arith) { 1323 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1324 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1325 } else { 1326 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1327 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1328 } 1329 } else { 1330 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1331 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1332 } 1333 } else { 1334 TCGv_i32 t0, t1; 1335 1336 t0 = tcg_temp_new_i32(); 1337 t1 = tcg_temp_new_i32(); 1338 if (right) { 1339 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c); 1340 if (arith) { 1341 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c); 1342 } else { 1343 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c); 1344 } 1345 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1346 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0); 1347 tcg_gen_mov_i32(TCGV_HIGH(ret), t1); 1348 } else { 1349 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1350 /* Note: ret can be the same as arg1, so we use t1 */ 1351 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c); 1352 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1353 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0); 1354 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1355 } 1356 tcg_temp_free_i32(t0); 1357 tcg_temp_free_i32(t1); 1358 } 1359 } 1360 1361 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1362 { 1363 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1364 if (TCG_TARGET_REG_BITS == 32) { 1365 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1366 } else if (arg2 == 0) { 1367 tcg_gen_mov_i64(ret, arg1); 1368 } else { 1369 TCGv_i64 t0 = tcg_const_i64(arg2); 1370 tcg_gen_shl_i64(ret, arg1, t0); 1371 tcg_temp_free_i64(t0); 1372 } 1373 } 1374 1375 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1376 { 1377 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1378 if (TCG_TARGET_REG_BITS == 32) { 1379 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1380 } else if (arg2 == 0) { 1381 tcg_gen_mov_i64(ret, arg1); 1382 } else { 1383 TCGv_i64 t0 = tcg_const_i64(arg2); 1384 tcg_gen_shr_i64(ret, arg1, t0); 1385 tcg_temp_free_i64(t0); 1386 } 1387 } 1388 1389 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1390 { 1391 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1392 if (TCG_TARGET_REG_BITS == 32) { 1393 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1394 } else if (arg2 == 0) { 1395 tcg_gen_mov_i64(ret, arg1); 1396 } else { 1397 TCGv_i64 t0 = tcg_const_i64(arg2); 1398 tcg_gen_sar_i64(ret, arg1, t0); 1399 tcg_temp_free_i64(t0); 1400 } 1401 } 1402 1403 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1404 { 1405 if (cond == TCG_COND_ALWAYS) { 1406 tcg_gen_br(l); 1407 } else if (cond != TCG_COND_NEVER) { 1408 if (TCG_TARGET_REG_BITS == 32) { 1409 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1410 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1411 TCGV_HIGH(arg2), cond, label_arg(l)); 1412 } else { 1413 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1414 label_arg(l)); 1415 } 1416 } 1417 } 1418 1419 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1420 { 1421 if (cond == TCG_COND_ALWAYS) { 1422 tcg_gen_br(l); 1423 } else if (cond != TCG_COND_NEVER) { 1424 TCGv_i64 t0 = tcg_const_i64(arg2); 1425 tcg_gen_brcond_i64(cond, arg1, t0, l); 1426 tcg_temp_free_i64(t0); 1427 } 1428 } 1429 1430 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1431 TCGv_i64 arg1, TCGv_i64 arg2) 1432 { 1433 if (cond == TCG_COND_ALWAYS) { 1434 tcg_gen_movi_i64(ret, 1); 1435 } else if (cond == TCG_COND_NEVER) { 1436 tcg_gen_movi_i64(ret, 0); 1437 } else { 1438 if (TCG_TARGET_REG_BITS == 32) { 1439 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1440 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1441 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1442 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1443 } else { 1444 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1445 } 1446 } 1447 } 1448 1449 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1450 TCGv_i64 arg1, int64_t arg2) 1451 { 1452 TCGv_i64 t0 = tcg_const_i64(arg2); 1453 tcg_gen_setcond_i64(cond, ret, arg1, t0); 1454 tcg_temp_free_i64(t0); 1455 } 1456 1457 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1458 { 1459 if (arg2 == 0) { 1460 tcg_gen_movi_i64(ret, 0); 1461 } else if (is_power_of_2(arg2)) { 1462 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 1463 } else { 1464 TCGv_i64 t0 = tcg_const_i64(arg2); 1465 tcg_gen_mul_i64(ret, arg1, t0); 1466 tcg_temp_free_i64(t0); 1467 } 1468 } 1469 1470 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1471 { 1472 if (TCG_TARGET_HAS_div_i64) { 1473 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2); 1474 } else if (TCG_TARGET_HAS_div2_i64) { 1475 TCGv_i64 t0 = tcg_temp_new_i64(); 1476 tcg_gen_sari_i64(t0, arg1, 63); 1477 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2); 1478 tcg_temp_free_i64(t0); 1479 } else { 1480 gen_helper_div_i64(ret, arg1, arg2); 1481 } 1482 } 1483 1484 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1485 { 1486 if (TCG_TARGET_HAS_rem_i64) { 1487 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2); 1488 } else if (TCG_TARGET_HAS_div_i64) { 1489 TCGv_i64 t0 = tcg_temp_new_i64(); 1490 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2); 1491 tcg_gen_mul_i64(t0, t0, arg2); 1492 tcg_gen_sub_i64(ret, arg1, t0); 1493 tcg_temp_free_i64(t0); 1494 } else if (TCG_TARGET_HAS_div2_i64) { 1495 TCGv_i64 t0 = tcg_temp_new_i64(); 1496 tcg_gen_sari_i64(t0, arg1, 63); 1497 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2); 1498 tcg_temp_free_i64(t0); 1499 } else { 1500 gen_helper_rem_i64(ret, arg1, arg2); 1501 } 1502 } 1503 1504 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1505 { 1506 if (TCG_TARGET_HAS_div_i64) { 1507 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2); 1508 } else if (TCG_TARGET_HAS_div2_i64) { 1509 TCGv_i64 t0 = tcg_temp_new_i64(); 1510 tcg_gen_movi_i64(t0, 0); 1511 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2); 1512 tcg_temp_free_i64(t0); 1513 } else { 1514 gen_helper_divu_i64(ret, arg1, arg2); 1515 } 1516 } 1517 1518 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1519 { 1520 if (TCG_TARGET_HAS_rem_i64) { 1521 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2); 1522 } else if (TCG_TARGET_HAS_div_i64) { 1523 TCGv_i64 t0 = tcg_temp_new_i64(); 1524 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2); 1525 tcg_gen_mul_i64(t0, t0, arg2); 1526 tcg_gen_sub_i64(ret, arg1, t0); 1527 tcg_temp_free_i64(t0); 1528 } else if (TCG_TARGET_HAS_div2_i64) { 1529 TCGv_i64 t0 = tcg_temp_new_i64(); 1530 tcg_gen_movi_i64(t0, 0); 1531 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2); 1532 tcg_temp_free_i64(t0); 1533 } else { 1534 gen_helper_remu_i64(ret, arg1, arg2); 1535 } 1536 } 1537 1538 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 1539 { 1540 if (TCG_TARGET_REG_BITS == 32) { 1541 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1542 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1543 } else if (TCG_TARGET_HAS_ext8s_i64) { 1544 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg); 1545 } else { 1546 tcg_gen_shli_i64(ret, arg, 56); 1547 tcg_gen_sari_i64(ret, ret, 56); 1548 } 1549 } 1550 1551 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 1552 { 1553 if (TCG_TARGET_REG_BITS == 32) { 1554 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1555 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1556 } else if (TCG_TARGET_HAS_ext16s_i64) { 1557 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg); 1558 } else { 1559 tcg_gen_shli_i64(ret, arg, 48); 1560 tcg_gen_sari_i64(ret, ret, 48); 1561 } 1562 } 1563 1564 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 1565 { 1566 if (TCG_TARGET_REG_BITS == 32) { 1567 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1568 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1569 } else if (TCG_TARGET_HAS_ext32s_i64) { 1570 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg); 1571 } else { 1572 tcg_gen_shli_i64(ret, arg, 32); 1573 tcg_gen_sari_i64(ret, ret, 32); 1574 } 1575 } 1576 1577 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 1578 { 1579 if (TCG_TARGET_REG_BITS == 32) { 1580 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1581 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1582 } else if (TCG_TARGET_HAS_ext8u_i64) { 1583 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg); 1584 } else { 1585 tcg_gen_andi_i64(ret, arg, 0xffu); 1586 } 1587 } 1588 1589 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 1590 { 1591 if (TCG_TARGET_REG_BITS == 32) { 1592 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1593 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1594 } else if (TCG_TARGET_HAS_ext16u_i64) { 1595 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg); 1596 } else { 1597 tcg_gen_andi_i64(ret, arg, 0xffffu); 1598 } 1599 } 1600 1601 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 1602 { 1603 if (TCG_TARGET_REG_BITS == 32) { 1604 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1605 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1606 } else if (TCG_TARGET_HAS_ext32u_i64) { 1607 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg); 1608 } else { 1609 tcg_gen_andi_i64(ret, arg, 0xffffffffu); 1610 } 1611 } 1612 1613 /* Note: we assume the six high bytes are set to zero */ 1614 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg) 1615 { 1616 if (TCG_TARGET_REG_BITS == 32) { 1617 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1618 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1619 } else if (TCG_TARGET_HAS_bswap16_i64) { 1620 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg); 1621 } else { 1622 TCGv_i64 t0 = tcg_temp_new_i64(); 1623 1624 tcg_gen_ext8u_i64(t0, arg); 1625 tcg_gen_shli_i64(t0, t0, 8); 1626 tcg_gen_shri_i64(ret, arg, 8); 1627 tcg_gen_or_i64(ret, ret, t0); 1628 tcg_temp_free_i64(t0); 1629 } 1630 } 1631 1632 /* Note: we assume the four high bytes are set to zero */ 1633 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg) 1634 { 1635 if (TCG_TARGET_REG_BITS == 32) { 1636 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1637 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1638 } else if (TCG_TARGET_HAS_bswap32_i64) { 1639 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg); 1640 } else { 1641 TCGv_i64 t0, t1; 1642 t0 = tcg_temp_new_i64(); 1643 t1 = tcg_temp_new_i64(); 1644 1645 tcg_gen_shli_i64(t0, arg, 24); 1646 tcg_gen_ext32u_i64(t0, t0); 1647 1648 tcg_gen_andi_i64(t1, arg, 0x0000ff00); 1649 tcg_gen_shli_i64(t1, t1, 8); 1650 tcg_gen_or_i64(t0, t0, t1); 1651 1652 tcg_gen_shri_i64(t1, arg, 8); 1653 tcg_gen_andi_i64(t1, t1, 0x0000ff00); 1654 tcg_gen_or_i64(t0, t0, t1); 1655 1656 tcg_gen_shri_i64(t1, arg, 24); 1657 tcg_gen_or_i64(ret, t0, t1); 1658 tcg_temp_free_i64(t0); 1659 tcg_temp_free_i64(t1); 1660 } 1661 } 1662 1663 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 1664 { 1665 if (TCG_TARGET_REG_BITS == 32) { 1666 TCGv_i32 t0, t1; 1667 t0 = tcg_temp_new_i32(); 1668 t1 = tcg_temp_new_i32(); 1669 1670 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 1671 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 1672 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1673 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 1674 tcg_temp_free_i32(t0); 1675 tcg_temp_free_i32(t1); 1676 } else if (TCG_TARGET_HAS_bswap64_i64) { 1677 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg); 1678 } else { 1679 TCGv_i64 t0 = tcg_temp_new_i64(); 1680 TCGv_i64 t1 = tcg_temp_new_i64(); 1681 1682 tcg_gen_shli_i64(t0, arg, 56); 1683 1684 tcg_gen_andi_i64(t1, arg, 0x0000ff00); 1685 tcg_gen_shli_i64(t1, t1, 40); 1686 tcg_gen_or_i64(t0, t0, t1); 1687 1688 tcg_gen_andi_i64(t1, arg, 0x00ff0000); 1689 tcg_gen_shli_i64(t1, t1, 24); 1690 tcg_gen_or_i64(t0, t0, t1); 1691 1692 tcg_gen_andi_i64(t1, arg, 0xff000000); 1693 tcg_gen_shli_i64(t1, t1, 8); 1694 tcg_gen_or_i64(t0, t0, t1); 1695 1696 tcg_gen_shri_i64(t1, arg, 8); 1697 tcg_gen_andi_i64(t1, t1, 0xff000000); 1698 tcg_gen_or_i64(t0, t0, t1); 1699 1700 tcg_gen_shri_i64(t1, arg, 24); 1701 tcg_gen_andi_i64(t1, t1, 0x00ff0000); 1702 tcg_gen_or_i64(t0, t0, t1); 1703 1704 tcg_gen_shri_i64(t1, arg, 40); 1705 tcg_gen_andi_i64(t1, t1, 0x0000ff00); 1706 tcg_gen_or_i64(t0, t0, t1); 1707 1708 tcg_gen_shri_i64(t1, arg, 56); 1709 tcg_gen_or_i64(ret, t0, t1); 1710 tcg_temp_free_i64(t0); 1711 tcg_temp_free_i64(t1); 1712 } 1713 } 1714 1715 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 1716 { 1717 if (TCG_TARGET_REG_BITS == 32) { 1718 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1719 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1720 } else if (TCG_TARGET_HAS_not_i64) { 1721 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg); 1722 } else { 1723 tcg_gen_xori_i64(ret, arg, -1); 1724 } 1725 } 1726 1727 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1728 { 1729 if (TCG_TARGET_REG_BITS == 32) { 1730 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1731 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1732 } else if (TCG_TARGET_HAS_andc_i64) { 1733 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2); 1734 } else { 1735 TCGv_i64 t0 = tcg_temp_new_i64(); 1736 tcg_gen_not_i64(t0, arg2); 1737 tcg_gen_and_i64(ret, arg1, t0); 1738 tcg_temp_free_i64(t0); 1739 } 1740 } 1741 1742 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1743 { 1744 if (TCG_TARGET_REG_BITS == 32) { 1745 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1746 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1747 } else if (TCG_TARGET_HAS_eqv_i64) { 1748 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2); 1749 } else { 1750 tcg_gen_xor_i64(ret, arg1, arg2); 1751 tcg_gen_not_i64(ret, ret); 1752 } 1753 } 1754 1755 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1756 { 1757 if (TCG_TARGET_REG_BITS == 32) { 1758 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1759 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1760 } else if (TCG_TARGET_HAS_nand_i64) { 1761 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2); 1762 } else { 1763 tcg_gen_and_i64(ret, arg1, arg2); 1764 tcg_gen_not_i64(ret, ret); 1765 } 1766 } 1767 1768 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1769 { 1770 if (TCG_TARGET_REG_BITS == 32) { 1771 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1772 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1773 } else if (TCG_TARGET_HAS_nor_i64) { 1774 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2); 1775 } else { 1776 tcg_gen_or_i64(ret, arg1, arg2); 1777 tcg_gen_not_i64(ret, ret); 1778 } 1779 } 1780 1781 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1782 { 1783 if (TCG_TARGET_REG_BITS == 32) { 1784 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1785 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1786 } else if (TCG_TARGET_HAS_orc_i64) { 1787 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2); 1788 } else { 1789 TCGv_i64 t0 = tcg_temp_new_i64(); 1790 tcg_gen_not_i64(t0, arg2); 1791 tcg_gen_or_i64(ret, arg1, t0); 1792 tcg_temp_free_i64(t0); 1793 } 1794 } 1795 1796 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1797 { 1798 if (TCG_TARGET_HAS_clz_i64) { 1799 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2); 1800 } else { 1801 gen_helper_clz_i64(ret, arg1, arg2); 1802 } 1803 } 1804 1805 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1806 { 1807 if (TCG_TARGET_REG_BITS == 32 1808 && TCG_TARGET_HAS_clz_i32 1809 && arg2 <= 0xffffffffu) { 1810 TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32); 1811 tcg_gen_clz_i32(t, TCGV_LOW(arg1), t); 1812 tcg_gen_addi_i32(t, t, 32); 1813 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 1814 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1815 tcg_temp_free_i32(t); 1816 } else { 1817 TCGv_i64 t = tcg_const_i64(arg2); 1818 tcg_gen_clz_i64(ret, arg1, t); 1819 tcg_temp_free_i64(t); 1820 } 1821 } 1822 1823 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1824 { 1825 if (TCG_TARGET_HAS_ctz_i64) { 1826 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); 1827 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) { 1828 TCGv_i64 z, t = tcg_temp_new_i64(); 1829 1830 if (TCG_TARGET_HAS_ctpop_i64) { 1831 tcg_gen_subi_i64(t, arg1, 1); 1832 tcg_gen_andc_i64(t, t, arg1); 1833 tcg_gen_ctpop_i64(t, t); 1834 } else { 1835 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */ 1836 tcg_gen_neg_i64(t, arg1); 1837 tcg_gen_and_i64(t, t, arg1); 1838 tcg_gen_clzi_i64(t, t, 64); 1839 tcg_gen_xori_i64(t, t, 63); 1840 } 1841 z = tcg_const_i64(0); 1842 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 1843 tcg_temp_free_i64(t); 1844 tcg_temp_free_i64(z); 1845 } else { 1846 gen_helper_ctz_i64(ret, arg1, arg2); 1847 } 1848 } 1849 1850 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1851 { 1852 if (TCG_TARGET_REG_BITS == 32 1853 && TCG_TARGET_HAS_ctz_i32 1854 && arg2 <= 0xffffffffu) { 1855 TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32); 1856 tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32); 1857 tcg_gen_addi_i32(t32, t32, 32); 1858 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 1859 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1860 tcg_temp_free_i32(t32); 1861 } else if (!TCG_TARGET_HAS_ctz_i64 1862 && TCG_TARGET_HAS_ctpop_i64 1863 && arg2 == 64) { 1864 /* This equivalence has the advantage of not requiring a fixup. */ 1865 TCGv_i64 t = tcg_temp_new_i64(); 1866 tcg_gen_subi_i64(t, arg1, 1); 1867 tcg_gen_andc_i64(t, t, arg1); 1868 tcg_gen_ctpop_i64(ret, t); 1869 tcg_temp_free_i64(t); 1870 } else { 1871 TCGv_i64 t64 = tcg_const_i64(arg2); 1872 tcg_gen_ctz_i64(ret, arg1, t64); 1873 tcg_temp_free_i64(t64); 1874 } 1875 } 1876 1877 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 1878 { 1879 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) { 1880 TCGv_i64 t = tcg_temp_new_i64(); 1881 tcg_gen_sari_i64(t, arg, 63); 1882 tcg_gen_xor_i64(t, t, arg); 1883 tcg_gen_clzi_i64(t, t, 64); 1884 tcg_gen_subi_i64(ret, t, 1); 1885 tcg_temp_free_i64(t); 1886 } else { 1887 gen_helper_clrsb_i64(ret, arg); 1888 } 1889 } 1890 1891 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 1892 { 1893 if (TCG_TARGET_HAS_ctpop_i64) { 1894 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 1895 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 1896 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1897 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1898 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 1899 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1900 } else { 1901 gen_helper_ctpop_i64(ret, arg1); 1902 } 1903 } 1904 1905 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1906 { 1907 if (TCG_TARGET_HAS_rot_i64) { 1908 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2); 1909 } else { 1910 TCGv_i64 t0, t1; 1911 t0 = tcg_temp_new_i64(); 1912 t1 = tcg_temp_new_i64(); 1913 tcg_gen_shl_i64(t0, arg1, arg2); 1914 tcg_gen_subfi_i64(t1, 64, arg2); 1915 tcg_gen_shr_i64(t1, arg1, t1); 1916 tcg_gen_or_i64(ret, t0, t1); 1917 tcg_temp_free_i64(t0); 1918 tcg_temp_free_i64(t1); 1919 } 1920 } 1921 1922 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1923 { 1924 tcg_debug_assert(arg2 < 64); 1925 /* some cases can be optimized here */ 1926 if (arg2 == 0) { 1927 tcg_gen_mov_i64(ret, arg1); 1928 } else if (TCG_TARGET_HAS_rot_i64) { 1929 TCGv_i64 t0 = tcg_const_i64(arg2); 1930 tcg_gen_rotl_i64(ret, arg1, t0); 1931 tcg_temp_free_i64(t0); 1932 } else { 1933 TCGv_i64 t0, t1; 1934 t0 = tcg_temp_new_i64(); 1935 t1 = tcg_temp_new_i64(); 1936 tcg_gen_shli_i64(t0, arg1, arg2); 1937 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 1938 tcg_gen_or_i64(ret, t0, t1); 1939 tcg_temp_free_i64(t0); 1940 tcg_temp_free_i64(t1); 1941 } 1942 } 1943 1944 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1945 { 1946 if (TCG_TARGET_HAS_rot_i64) { 1947 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2); 1948 } else { 1949 TCGv_i64 t0, t1; 1950 t0 = tcg_temp_new_i64(); 1951 t1 = tcg_temp_new_i64(); 1952 tcg_gen_shr_i64(t0, arg1, arg2); 1953 tcg_gen_subfi_i64(t1, 64, arg2); 1954 tcg_gen_shl_i64(t1, arg1, t1); 1955 tcg_gen_or_i64(ret, t0, t1); 1956 tcg_temp_free_i64(t0); 1957 tcg_temp_free_i64(t1); 1958 } 1959 } 1960 1961 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1962 { 1963 tcg_debug_assert(arg2 < 64); 1964 /* some cases can be optimized here */ 1965 if (arg2 == 0) { 1966 tcg_gen_mov_i64(ret, arg1); 1967 } else { 1968 tcg_gen_rotli_i64(ret, arg1, 64 - arg2); 1969 } 1970 } 1971 1972 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 1973 unsigned int ofs, unsigned int len) 1974 { 1975 uint64_t mask; 1976 TCGv_i64 t1; 1977 1978 tcg_debug_assert(ofs < 64); 1979 tcg_debug_assert(len > 0); 1980 tcg_debug_assert(len <= 64); 1981 tcg_debug_assert(ofs + len <= 64); 1982 1983 if (len == 64) { 1984 tcg_gen_mov_i64(ret, arg2); 1985 return; 1986 } 1987 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 1988 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 1989 return; 1990 } 1991 1992 if (TCG_TARGET_REG_BITS == 32) { 1993 if (ofs >= 32) { 1994 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 1995 TCGV_LOW(arg2), ofs - 32, len); 1996 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1997 return; 1998 } 1999 if (ofs + len <= 32) { 2000 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 2001 TCGV_LOW(arg2), ofs, len); 2002 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2003 return; 2004 } 2005 } 2006 2007 mask = (1ull << len) - 1; 2008 t1 = tcg_temp_new_i64(); 2009 2010 if (ofs + len < 64) { 2011 tcg_gen_andi_i64(t1, arg2, mask); 2012 tcg_gen_shli_i64(t1, t1, ofs); 2013 } else { 2014 tcg_gen_shli_i64(t1, arg2, ofs); 2015 } 2016 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2017 tcg_gen_or_i64(ret, ret, t1); 2018 2019 tcg_temp_free_i64(t1); 2020 } 2021 2022 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2023 unsigned int ofs, unsigned int len) 2024 { 2025 tcg_debug_assert(ofs < 64); 2026 tcg_debug_assert(len > 0); 2027 tcg_debug_assert(len <= 64); 2028 tcg_debug_assert(ofs + len <= 64); 2029 2030 if (ofs + len == 64) { 2031 tcg_gen_shli_i64(ret, arg, ofs); 2032 } else if (ofs == 0) { 2033 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2034 } else if (TCG_TARGET_HAS_deposit_i64 2035 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2036 TCGv_i64 zero = tcg_const_i64(0); 2037 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2038 tcg_temp_free_i64(zero); 2039 } else { 2040 if (TCG_TARGET_REG_BITS == 32) { 2041 if (ofs >= 32) { 2042 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2043 ofs - 32, len); 2044 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2045 return; 2046 } 2047 if (ofs + len <= 32) { 2048 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2049 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2050 return; 2051 } 2052 } 2053 /* To help two-operand hosts we prefer to zero-extend first, 2054 which allows ARG to stay live. */ 2055 switch (len) { 2056 case 32: 2057 if (TCG_TARGET_HAS_ext32u_i64) { 2058 tcg_gen_ext32u_i64(ret, arg); 2059 tcg_gen_shli_i64(ret, ret, ofs); 2060 return; 2061 } 2062 break; 2063 case 16: 2064 if (TCG_TARGET_HAS_ext16u_i64) { 2065 tcg_gen_ext16u_i64(ret, arg); 2066 tcg_gen_shli_i64(ret, ret, ofs); 2067 return; 2068 } 2069 break; 2070 case 8: 2071 if (TCG_TARGET_HAS_ext8u_i64) { 2072 tcg_gen_ext8u_i64(ret, arg); 2073 tcg_gen_shli_i64(ret, ret, ofs); 2074 return; 2075 } 2076 break; 2077 } 2078 /* Otherwise prefer zero-extension over AND for code size. */ 2079 switch (ofs + len) { 2080 case 32: 2081 if (TCG_TARGET_HAS_ext32u_i64) { 2082 tcg_gen_shli_i64(ret, arg, ofs); 2083 tcg_gen_ext32u_i64(ret, ret); 2084 return; 2085 } 2086 break; 2087 case 16: 2088 if (TCG_TARGET_HAS_ext16u_i64) { 2089 tcg_gen_shli_i64(ret, arg, ofs); 2090 tcg_gen_ext16u_i64(ret, ret); 2091 return; 2092 } 2093 break; 2094 case 8: 2095 if (TCG_TARGET_HAS_ext8u_i64) { 2096 tcg_gen_shli_i64(ret, arg, ofs); 2097 tcg_gen_ext8u_i64(ret, ret); 2098 return; 2099 } 2100 break; 2101 } 2102 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2103 tcg_gen_shli_i64(ret, ret, ofs); 2104 } 2105 } 2106 2107 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2108 unsigned int ofs, unsigned int len) 2109 { 2110 tcg_debug_assert(ofs < 64); 2111 tcg_debug_assert(len > 0); 2112 tcg_debug_assert(len <= 64); 2113 tcg_debug_assert(ofs + len <= 64); 2114 2115 /* Canonicalize certain special cases, even if extract is supported. */ 2116 if (ofs + len == 64) { 2117 tcg_gen_shri_i64(ret, arg, 64 - len); 2118 return; 2119 } 2120 if (ofs == 0) { 2121 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2122 return; 2123 } 2124 2125 if (TCG_TARGET_REG_BITS == 32) { 2126 /* Look for a 32-bit extract within one of the two words. */ 2127 if (ofs >= 32) { 2128 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2129 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2130 return; 2131 } 2132 if (ofs + len <= 32) { 2133 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2134 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2135 return; 2136 } 2137 /* The field is split across two words. One double-word 2138 shift is better than two double-word shifts. */ 2139 goto do_shift_and; 2140 } 2141 2142 if (TCG_TARGET_HAS_extract_i64 2143 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2144 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2145 return; 2146 } 2147 2148 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2149 switch (ofs + len) { 2150 case 32: 2151 if (TCG_TARGET_HAS_ext32u_i64) { 2152 tcg_gen_ext32u_i64(ret, arg); 2153 tcg_gen_shri_i64(ret, ret, ofs); 2154 return; 2155 } 2156 break; 2157 case 16: 2158 if (TCG_TARGET_HAS_ext16u_i64) { 2159 tcg_gen_ext16u_i64(ret, arg); 2160 tcg_gen_shri_i64(ret, ret, ofs); 2161 return; 2162 } 2163 break; 2164 case 8: 2165 if (TCG_TARGET_HAS_ext8u_i64) { 2166 tcg_gen_ext8u_i64(ret, arg); 2167 tcg_gen_shri_i64(ret, ret, ofs); 2168 return; 2169 } 2170 break; 2171 } 2172 2173 /* ??? Ideally we'd know what values are available for immediate AND. 2174 Assume that 8 bits are available, plus the special cases of 16 and 32, 2175 so that we get ext8u, ext16u, and ext32u. */ 2176 switch (len) { 2177 case 1 ... 8: case 16: case 32: 2178 do_shift_and: 2179 tcg_gen_shri_i64(ret, arg, ofs); 2180 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2181 break; 2182 default: 2183 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2184 tcg_gen_shri_i64(ret, ret, 64 - len); 2185 break; 2186 } 2187 } 2188 2189 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2190 unsigned int ofs, unsigned int len) 2191 { 2192 tcg_debug_assert(ofs < 64); 2193 tcg_debug_assert(len > 0); 2194 tcg_debug_assert(len <= 64); 2195 tcg_debug_assert(ofs + len <= 64); 2196 2197 /* Canonicalize certain special cases, even if sextract is supported. */ 2198 if (ofs + len == 64) { 2199 tcg_gen_sari_i64(ret, arg, 64 - len); 2200 return; 2201 } 2202 if (ofs == 0) { 2203 switch (len) { 2204 case 32: 2205 tcg_gen_ext32s_i64(ret, arg); 2206 return; 2207 case 16: 2208 tcg_gen_ext16s_i64(ret, arg); 2209 return; 2210 case 8: 2211 tcg_gen_ext8s_i64(ret, arg); 2212 return; 2213 } 2214 } 2215 2216 if (TCG_TARGET_REG_BITS == 32) { 2217 /* Look for a 32-bit extract within one of the two words. */ 2218 if (ofs >= 32) { 2219 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2220 } else if (ofs + len <= 32) { 2221 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2222 } else if (ofs == 0) { 2223 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2224 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2225 return; 2226 } else if (len > 32) { 2227 TCGv_i32 t = tcg_temp_new_i32(); 2228 /* Extract the bits for the high word normally. */ 2229 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2230 /* Shift the field down for the low part. */ 2231 tcg_gen_shri_i64(ret, arg, ofs); 2232 /* Overwrite the shift into the high part. */ 2233 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2234 tcg_temp_free_i32(t); 2235 return; 2236 } else { 2237 /* Shift the field down for the low part, such that the 2238 field sits at the MSB. */ 2239 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2240 /* Shift the field down from the MSB, sign extending. */ 2241 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2242 } 2243 /* Sign-extend the field from 32 bits. */ 2244 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2245 return; 2246 } 2247 2248 if (TCG_TARGET_HAS_sextract_i64 2249 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2250 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2251 return; 2252 } 2253 2254 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2255 switch (ofs + len) { 2256 case 32: 2257 if (TCG_TARGET_HAS_ext32s_i64) { 2258 tcg_gen_ext32s_i64(ret, arg); 2259 tcg_gen_sari_i64(ret, ret, ofs); 2260 return; 2261 } 2262 break; 2263 case 16: 2264 if (TCG_TARGET_HAS_ext16s_i64) { 2265 tcg_gen_ext16s_i64(ret, arg); 2266 tcg_gen_sari_i64(ret, ret, ofs); 2267 return; 2268 } 2269 break; 2270 case 8: 2271 if (TCG_TARGET_HAS_ext8s_i64) { 2272 tcg_gen_ext8s_i64(ret, arg); 2273 tcg_gen_sari_i64(ret, ret, ofs); 2274 return; 2275 } 2276 break; 2277 } 2278 switch (len) { 2279 case 32: 2280 if (TCG_TARGET_HAS_ext32s_i64) { 2281 tcg_gen_shri_i64(ret, arg, ofs); 2282 tcg_gen_ext32s_i64(ret, ret); 2283 return; 2284 } 2285 break; 2286 case 16: 2287 if (TCG_TARGET_HAS_ext16s_i64) { 2288 tcg_gen_shri_i64(ret, arg, ofs); 2289 tcg_gen_ext16s_i64(ret, ret); 2290 return; 2291 } 2292 break; 2293 case 8: 2294 if (TCG_TARGET_HAS_ext8s_i64) { 2295 tcg_gen_shri_i64(ret, arg, ofs); 2296 tcg_gen_ext8s_i64(ret, ret); 2297 return; 2298 } 2299 break; 2300 } 2301 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2302 tcg_gen_sari_i64(ret, ret, 64 - len); 2303 } 2304 2305 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2306 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2307 { 2308 if (cond == TCG_COND_ALWAYS) { 2309 tcg_gen_mov_i64(ret, v1); 2310 } else if (cond == TCG_COND_NEVER) { 2311 tcg_gen_mov_i64(ret, v2); 2312 } else if (TCG_TARGET_REG_BITS == 32) { 2313 TCGv_i32 t0 = tcg_temp_new_i32(); 2314 TCGv_i32 t1 = tcg_temp_new_i32(); 2315 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2316 TCGV_LOW(c1), TCGV_HIGH(c1), 2317 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2318 2319 if (TCG_TARGET_HAS_movcond_i32) { 2320 tcg_gen_movi_i32(t1, 0); 2321 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1, 2322 TCGV_LOW(v1), TCGV_LOW(v2)); 2323 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1, 2324 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2325 } else { 2326 tcg_gen_neg_i32(t0, t0); 2327 2328 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0); 2329 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0); 2330 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1); 2331 2332 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0); 2333 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0); 2334 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1); 2335 } 2336 tcg_temp_free_i32(t0); 2337 tcg_temp_free_i32(t1); 2338 } else if (TCG_TARGET_HAS_movcond_i64) { 2339 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2340 } else { 2341 TCGv_i64 t0 = tcg_temp_new_i64(); 2342 TCGv_i64 t1 = tcg_temp_new_i64(); 2343 tcg_gen_setcond_i64(cond, t0, c1, c2); 2344 tcg_gen_neg_i64(t0, t0); 2345 tcg_gen_and_i64(t1, v1, t0); 2346 tcg_gen_andc_i64(ret, v2, t0); 2347 tcg_gen_or_i64(ret, ret, t1); 2348 tcg_temp_free_i64(t0); 2349 tcg_temp_free_i64(t1); 2350 } 2351 } 2352 2353 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2354 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2355 { 2356 if (TCG_TARGET_HAS_add2_i64) { 2357 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2358 } else { 2359 TCGv_i64 t0 = tcg_temp_new_i64(); 2360 TCGv_i64 t1 = tcg_temp_new_i64(); 2361 tcg_gen_add_i64(t0, al, bl); 2362 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2363 tcg_gen_add_i64(rh, ah, bh); 2364 tcg_gen_add_i64(rh, rh, t1); 2365 tcg_gen_mov_i64(rl, t0); 2366 tcg_temp_free_i64(t0); 2367 tcg_temp_free_i64(t1); 2368 } 2369 } 2370 2371 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2372 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2373 { 2374 if (TCG_TARGET_HAS_sub2_i64) { 2375 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2376 } else { 2377 TCGv_i64 t0 = tcg_temp_new_i64(); 2378 TCGv_i64 t1 = tcg_temp_new_i64(); 2379 tcg_gen_sub_i64(t0, al, bl); 2380 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2381 tcg_gen_sub_i64(rh, ah, bh); 2382 tcg_gen_sub_i64(rh, rh, t1); 2383 tcg_gen_mov_i64(rl, t0); 2384 tcg_temp_free_i64(t0); 2385 tcg_temp_free_i64(t1); 2386 } 2387 } 2388 2389 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2390 { 2391 if (TCG_TARGET_HAS_mulu2_i64) { 2392 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2393 } else if (TCG_TARGET_HAS_muluh_i64) { 2394 TCGv_i64 t = tcg_temp_new_i64(); 2395 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2396 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2); 2397 tcg_gen_mov_i64(rl, t); 2398 tcg_temp_free_i64(t); 2399 } else { 2400 TCGv_i64 t0 = tcg_temp_new_i64(); 2401 tcg_gen_mul_i64(t0, arg1, arg2); 2402 gen_helper_muluh_i64(rh, arg1, arg2); 2403 tcg_gen_mov_i64(rl, t0); 2404 tcg_temp_free_i64(t0); 2405 } 2406 } 2407 2408 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2409 { 2410 if (TCG_TARGET_HAS_muls2_i64) { 2411 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 2412 } else if (TCG_TARGET_HAS_mulsh_i64) { 2413 TCGv_i64 t = tcg_temp_new_i64(); 2414 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2415 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2); 2416 tcg_gen_mov_i64(rl, t); 2417 tcg_temp_free_i64(t); 2418 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) { 2419 TCGv_i64 t0 = tcg_temp_new_i64(); 2420 TCGv_i64 t1 = tcg_temp_new_i64(); 2421 TCGv_i64 t2 = tcg_temp_new_i64(); 2422 TCGv_i64 t3 = tcg_temp_new_i64(); 2423 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2424 /* Adjust for negative inputs. */ 2425 tcg_gen_sari_i64(t2, arg1, 63); 2426 tcg_gen_sari_i64(t3, arg2, 63); 2427 tcg_gen_and_i64(t2, t2, arg2); 2428 tcg_gen_and_i64(t3, t3, arg1); 2429 tcg_gen_sub_i64(rh, t1, t2); 2430 tcg_gen_sub_i64(rh, rh, t3); 2431 tcg_gen_mov_i64(rl, t0); 2432 tcg_temp_free_i64(t0); 2433 tcg_temp_free_i64(t1); 2434 tcg_temp_free_i64(t2); 2435 tcg_temp_free_i64(t3); 2436 } else { 2437 TCGv_i64 t0 = tcg_temp_new_i64(); 2438 tcg_gen_mul_i64(t0, arg1, arg2); 2439 gen_helper_mulsh_i64(rh, arg1, arg2); 2440 tcg_gen_mov_i64(rl, t0); 2441 tcg_temp_free_i64(t0); 2442 } 2443 } 2444 2445 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2446 { 2447 TCGv_i64 t0 = tcg_temp_new_i64(); 2448 TCGv_i64 t1 = tcg_temp_new_i64(); 2449 TCGv_i64 t2 = tcg_temp_new_i64(); 2450 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2451 /* Adjust for negative input for the signed arg1. */ 2452 tcg_gen_sari_i64(t2, arg1, 63); 2453 tcg_gen_and_i64(t2, t2, arg2); 2454 tcg_gen_sub_i64(rh, t1, t2); 2455 tcg_gen_mov_i64(rl, t0); 2456 tcg_temp_free_i64(t0); 2457 tcg_temp_free_i64(t1); 2458 tcg_temp_free_i64(t2); 2459 } 2460 2461 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2462 { 2463 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 2464 } 2465 2466 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2467 { 2468 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 2469 } 2470 2471 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2472 { 2473 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 2474 } 2475 2476 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2477 { 2478 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 2479 } 2480 2481 /* Size changing operations. */ 2482 2483 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2484 { 2485 if (TCG_TARGET_REG_BITS == 32) { 2486 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2487 } else if (TCG_TARGET_HAS_extrl_i64_i32) { 2488 tcg_gen_op2(INDEX_op_extrl_i64_i32, 2489 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2490 } else { 2491 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2492 } 2493 } 2494 2495 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2496 { 2497 if (TCG_TARGET_REG_BITS == 32) { 2498 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2499 } else if (TCG_TARGET_HAS_extrh_i64_i32) { 2500 tcg_gen_op2(INDEX_op_extrh_i64_i32, 2501 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2502 } else { 2503 TCGv_i64 t = tcg_temp_new_i64(); 2504 tcg_gen_shri_i64(t, arg, 32); 2505 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2506 tcg_temp_free_i64(t); 2507 } 2508 } 2509 2510 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2511 { 2512 if (TCG_TARGET_REG_BITS == 32) { 2513 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2514 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2515 } else { 2516 tcg_gen_op2(INDEX_op_extu_i32_i64, 2517 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2518 } 2519 } 2520 2521 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2522 { 2523 if (TCG_TARGET_REG_BITS == 32) { 2524 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2525 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2526 } else { 2527 tcg_gen_op2(INDEX_op_ext_i32_i64, 2528 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2529 } 2530 } 2531 2532 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 2533 { 2534 TCGv_i64 tmp; 2535 2536 if (TCG_TARGET_REG_BITS == 32) { 2537 tcg_gen_mov_i32(TCGV_LOW(dest), low); 2538 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 2539 return; 2540 } 2541 2542 tmp = tcg_temp_new_i64(); 2543 /* These extensions are only needed for type correctness. 2544 We may be able to do better given target specific information. */ 2545 tcg_gen_extu_i32_i64(tmp, high); 2546 tcg_gen_extu_i32_i64(dest, low); 2547 /* If deposit is available, use it. Otherwise use the extra 2548 knowledge that we have of the zero-extensions above. */ 2549 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) { 2550 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 2551 } else { 2552 tcg_gen_shli_i64(tmp, tmp, 32); 2553 tcg_gen_or_i64(dest, dest, tmp); 2554 } 2555 tcg_temp_free_i64(tmp); 2556 } 2557 2558 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 2559 { 2560 if (TCG_TARGET_REG_BITS == 32) { 2561 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 2562 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 2563 } else { 2564 tcg_gen_extrl_i64_i32(lo, arg); 2565 tcg_gen_extrh_i64_i32(hi, arg); 2566 } 2567 } 2568 2569 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 2570 { 2571 tcg_gen_ext32u_i64(lo, arg); 2572 tcg_gen_shri_i64(hi, arg, 32); 2573 } 2574 2575 /* QEMU specific operations. */ 2576 2577 void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx) 2578 { 2579 uintptr_t val = (uintptr_t)tb + idx; 2580 2581 if (tb == NULL) { 2582 tcg_debug_assert(idx == 0); 2583 } else if (idx <= TB_EXIT_IDXMAX) { 2584 #ifdef CONFIG_DEBUG_TCG 2585 /* This is an exit following a goto_tb. Verify that we have 2586 seen this numbered exit before, via tcg_gen_goto_tb. */ 2587 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); 2588 #endif 2589 /* When not chaining, exit without indicating a link. */ 2590 if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2591 val = 0; 2592 } 2593 } else { 2594 /* This is an exit via the exitreq label. */ 2595 tcg_debug_assert(idx == TB_EXIT_REQUESTED); 2596 } 2597 2598 tcg_gen_op1i(INDEX_op_exit_tb, val); 2599 } 2600 2601 void tcg_gen_goto_tb(unsigned idx) 2602 { 2603 /* We only support two chained exits. */ 2604 tcg_debug_assert(idx <= TB_EXIT_IDXMAX); 2605 #ifdef CONFIG_DEBUG_TCG 2606 /* Verify that we havn't seen this numbered exit before. */ 2607 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 2608 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 2609 #endif 2610 /* When not chaining, we simply fall through to the "fallback" exit. */ 2611 if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2612 tcg_gen_op1i(INDEX_op_goto_tb, idx); 2613 } 2614 } 2615 2616 void tcg_gen_lookup_and_goto_ptr(void) 2617 { 2618 if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2619 TCGv_ptr ptr = tcg_temp_new_ptr(); 2620 gen_helper_lookup_tb_ptr(ptr, cpu_env); 2621 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr)); 2622 tcg_temp_free_ptr(ptr); 2623 } else { 2624 tcg_gen_exit_tb(NULL, 0); 2625 } 2626 } 2627 2628 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st) 2629 { 2630 /* Trigger the asserts within as early as possible. */ 2631 (void)get_alignment_bits(op); 2632 2633 switch (op & MO_SIZE) { 2634 case MO_8: 2635 op &= ~MO_BSWAP; 2636 break; 2637 case MO_16: 2638 break; 2639 case MO_32: 2640 if (!is64) { 2641 op &= ~MO_SIGN; 2642 } 2643 break; 2644 case MO_64: 2645 if (!is64) { 2646 tcg_abort(); 2647 } 2648 break; 2649 } 2650 if (st) { 2651 op &= ~MO_SIGN; 2652 } 2653 return op; 2654 } 2655 2656 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr, 2657 TCGMemOp memop, TCGArg idx) 2658 { 2659 TCGMemOpIdx oi = make_memop_idx(memop, idx); 2660 #if TARGET_LONG_BITS == 32 2661 tcg_gen_op3i_i32(opc, val, addr, oi); 2662 #else 2663 if (TCG_TARGET_REG_BITS == 32) { 2664 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2665 } else { 2666 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi); 2667 } 2668 #endif 2669 } 2670 2671 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr, 2672 TCGMemOp memop, TCGArg idx) 2673 { 2674 TCGMemOpIdx oi = make_memop_idx(memop, idx); 2675 #if TARGET_LONG_BITS == 32 2676 if (TCG_TARGET_REG_BITS == 32) { 2677 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi); 2678 } else { 2679 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi); 2680 } 2681 #else 2682 if (TCG_TARGET_REG_BITS == 32) { 2683 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), 2684 TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2685 } else { 2686 tcg_gen_op3i_i64(opc, val, addr, oi); 2687 } 2688 #endif 2689 } 2690 2691 static void tcg_gen_req_mo(TCGBar type) 2692 { 2693 #ifdef TCG_GUEST_DEFAULT_MO 2694 type &= TCG_GUEST_DEFAULT_MO; 2695 #endif 2696 type &= ~TCG_TARGET_DEFAULT_MO; 2697 if (type) { 2698 tcg_gen_mb(type | TCG_BAR_SC); 2699 } 2700 } 2701 2702 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2703 { 2704 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2705 memop = tcg_canonicalize_memop(memop, 0, 0); 2706 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2707 addr, trace_mem_get_info(memop, 0)); 2708 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx); 2709 } 2710 2711 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2712 { 2713 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2714 memop = tcg_canonicalize_memop(memop, 0, 1); 2715 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2716 addr, trace_mem_get_info(memop, 1)); 2717 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); 2718 } 2719 2720 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2721 { 2722 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2723 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 2724 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); 2725 if (memop & MO_SIGN) { 2726 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 2727 } else { 2728 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 2729 } 2730 return; 2731 } 2732 2733 memop = tcg_canonicalize_memop(memop, 1, 0); 2734 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2735 addr, trace_mem_get_info(memop, 0)); 2736 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx); 2737 } 2738 2739 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2740 { 2741 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2742 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 2743 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); 2744 return; 2745 } 2746 2747 memop = tcg_canonicalize_memop(memop, 1, 1); 2748 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2749 addr, trace_mem_get_info(memop, 1)); 2750 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx); 2751 } 2752 2753 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc) 2754 { 2755 switch (opc & MO_SSIZE) { 2756 case MO_SB: 2757 tcg_gen_ext8s_i32(ret, val); 2758 break; 2759 case MO_UB: 2760 tcg_gen_ext8u_i32(ret, val); 2761 break; 2762 case MO_SW: 2763 tcg_gen_ext16s_i32(ret, val); 2764 break; 2765 case MO_UW: 2766 tcg_gen_ext16u_i32(ret, val); 2767 break; 2768 default: 2769 tcg_gen_mov_i32(ret, val); 2770 break; 2771 } 2772 } 2773 2774 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc) 2775 { 2776 switch (opc & MO_SSIZE) { 2777 case MO_SB: 2778 tcg_gen_ext8s_i64(ret, val); 2779 break; 2780 case MO_UB: 2781 tcg_gen_ext8u_i64(ret, val); 2782 break; 2783 case MO_SW: 2784 tcg_gen_ext16s_i64(ret, val); 2785 break; 2786 case MO_UW: 2787 tcg_gen_ext16u_i64(ret, val); 2788 break; 2789 case MO_SL: 2790 tcg_gen_ext32s_i64(ret, val); 2791 break; 2792 case MO_UL: 2793 tcg_gen_ext32u_i64(ret, val); 2794 break; 2795 default: 2796 tcg_gen_mov_i64(ret, val); 2797 break; 2798 } 2799 } 2800 2801 #ifdef CONFIG_SOFTMMU 2802 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, 2803 TCGv_i32, TCGv_i32, TCGv_i32); 2804 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, 2805 TCGv_i64, TCGv_i64, TCGv_i32); 2806 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, 2807 TCGv_i32, TCGv_i32); 2808 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, 2809 TCGv_i64, TCGv_i32); 2810 #else 2811 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32); 2812 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64); 2813 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32); 2814 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64); 2815 #endif 2816 2817 #ifdef CONFIG_ATOMIC64 2818 # define WITH_ATOMIC64(X) X, 2819 #else 2820 # define WITH_ATOMIC64(X) 2821 #endif 2822 2823 static void * const table_cmpxchg[16] = { 2824 [MO_8] = gen_helper_atomic_cmpxchgb, 2825 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, 2826 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, 2827 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, 2828 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, 2829 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) 2830 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) 2831 }; 2832 2833 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, 2834 TCGv_i32 newv, TCGArg idx, TCGMemOp memop) 2835 { 2836 memop = tcg_canonicalize_memop(memop, 0, 0); 2837 2838 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { 2839 TCGv_i32 t1 = tcg_temp_new_i32(); 2840 TCGv_i32 t2 = tcg_temp_new_i32(); 2841 2842 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); 2843 2844 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 2845 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); 2846 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 2847 tcg_temp_free_i32(t2); 2848 2849 if (memop & MO_SIGN) { 2850 tcg_gen_ext_i32(retv, t1, memop); 2851 } else { 2852 tcg_gen_mov_i32(retv, t1); 2853 } 2854 tcg_temp_free_i32(t1); 2855 } else { 2856 gen_atomic_cx_i32 gen; 2857 2858 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 2859 tcg_debug_assert(gen != NULL); 2860 2861 #ifdef CONFIG_SOFTMMU 2862 { 2863 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 2864 gen(retv, cpu_env, addr, cmpv, newv, oi); 2865 tcg_temp_free_i32(oi); 2866 } 2867 #else 2868 gen(retv, cpu_env, addr, cmpv, newv); 2869 #endif 2870 2871 if (memop & MO_SIGN) { 2872 tcg_gen_ext_i32(retv, retv, memop); 2873 } 2874 } 2875 } 2876 2877 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, 2878 TCGv_i64 newv, TCGArg idx, TCGMemOp memop) 2879 { 2880 memop = tcg_canonicalize_memop(memop, 1, 0); 2881 2882 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { 2883 TCGv_i64 t1 = tcg_temp_new_i64(); 2884 TCGv_i64 t2 = tcg_temp_new_i64(); 2885 2886 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); 2887 2888 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 2889 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); 2890 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 2891 tcg_temp_free_i64(t2); 2892 2893 if (memop & MO_SIGN) { 2894 tcg_gen_ext_i64(retv, t1, memop); 2895 } else { 2896 tcg_gen_mov_i64(retv, t1); 2897 } 2898 tcg_temp_free_i64(t1); 2899 } else if ((memop & MO_SIZE) == MO_64) { 2900 #ifdef CONFIG_ATOMIC64 2901 gen_atomic_cx_i64 gen; 2902 2903 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 2904 tcg_debug_assert(gen != NULL); 2905 2906 #ifdef CONFIG_SOFTMMU 2907 { 2908 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx)); 2909 gen(retv, cpu_env, addr, cmpv, newv, oi); 2910 tcg_temp_free_i32(oi); 2911 } 2912 #else 2913 gen(retv, cpu_env, addr, cmpv, newv); 2914 #endif 2915 #else 2916 gen_helper_exit_atomic(cpu_env); 2917 /* Produce a result, so that we have a well-formed opcode stream 2918 with respect to uses of the result in the (dead) code following. */ 2919 tcg_gen_movi_i64(retv, 0); 2920 #endif /* CONFIG_ATOMIC64 */ 2921 } else { 2922 TCGv_i32 c32 = tcg_temp_new_i32(); 2923 TCGv_i32 n32 = tcg_temp_new_i32(); 2924 TCGv_i32 r32 = tcg_temp_new_i32(); 2925 2926 tcg_gen_extrl_i64_i32(c32, cmpv); 2927 tcg_gen_extrl_i64_i32(n32, newv); 2928 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN); 2929 tcg_temp_free_i32(c32); 2930 tcg_temp_free_i32(n32); 2931 2932 tcg_gen_extu_i32_i64(retv, r32); 2933 tcg_temp_free_i32(r32); 2934 2935 if (memop & MO_SIGN) { 2936 tcg_gen_ext_i64(retv, retv, memop); 2937 } 2938 } 2939 } 2940 2941 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 2942 TCGArg idx, TCGMemOp memop, bool new_val, 2943 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) 2944 { 2945 TCGv_i32 t1 = tcg_temp_new_i32(); 2946 TCGv_i32 t2 = tcg_temp_new_i32(); 2947 2948 memop = tcg_canonicalize_memop(memop, 0, 0); 2949 2950 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 2951 gen(t2, t1, val); 2952 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 2953 2954 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); 2955 tcg_temp_free_i32(t1); 2956 tcg_temp_free_i32(t2); 2957 } 2958 2959 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 2960 TCGArg idx, TCGMemOp memop, void * const table[]) 2961 { 2962 gen_atomic_op_i32 gen; 2963 2964 memop = tcg_canonicalize_memop(memop, 0, 0); 2965 2966 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 2967 tcg_debug_assert(gen != NULL); 2968 2969 #ifdef CONFIG_SOFTMMU 2970 { 2971 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 2972 gen(ret, cpu_env, addr, val, oi); 2973 tcg_temp_free_i32(oi); 2974 } 2975 #else 2976 gen(ret, cpu_env, addr, val); 2977 #endif 2978 2979 if (memop & MO_SIGN) { 2980 tcg_gen_ext_i32(ret, ret, memop); 2981 } 2982 } 2983 2984 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 2985 TCGArg idx, TCGMemOp memop, bool new_val, 2986 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) 2987 { 2988 TCGv_i64 t1 = tcg_temp_new_i64(); 2989 TCGv_i64 t2 = tcg_temp_new_i64(); 2990 2991 memop = tcg_canonicalize_memop(memop, 1, 0); 2992 2993 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 2994 gen(t2, t1, val); 2995 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 2996 2997 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); 2998 tcg_temp_free_i64(t1); 2999 tcg_temp_free_i64(t2); 3000 } 3001 3002 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 3003 TCGArg idx, TCGMemOp memop, void * const table[]) 3004 { 3005 memop = tcg_canonicalize_memop(memop, 1, 0); 3006 3007 if ((memop & MO_SIZE) == MO_64) { 3008 #ifdef CONFIG_ATOMIC64 3009 gen_atomic_op_i64 gen; 3010 3011 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 3012 tcg_debug_assert(gen != NULL); 3013 3014 #ifdef CONFIG_SOFTMMU 3015 { 3016 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 3017 gen(ret, cpu_env, addr, val, oi); 3018 tcg_temp_free_i32(oi); 3019 } 3020 #else 3021 gen(ret, cpu_env, addr, val); 3022 #endif 3023 #else 3024 gen_helper_exit_atomic(cpu_env); 3025 /* Produce a result, so that we have a well-formed opcode stream 3026 with respect to uses of the result in the (dead) code following. */ 3027 tcg_gen_movi_i64(ret, 0); 3028 #endif /* CONFIG_ATOMIC64 */ 3029 } else { 3030 TCGv_i32 v32 = tcg_temp_new_i32(); 3031 TCGv_i32 r32 = tcg_temp_new_i32(); 3032 3033 tcg_gen_extrl_i64_i32(v32, val); 3034 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); 3035 tcg_temp_free_i32(v32); 3036 3037 tcg_gen_extu_i32_i64(ret, r32); 3038 tcg_temp_free_i32(r32); 3039 3040 if (memop & MO_SIGN) { 3041 tcg_gen_ext_i64(ret, ret, memop); 3042 } 3043 } 3044 } 3045 3046 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ 3047 static void * const table_##NAME[16] = { \ 3048 [MO_8] = gen_helper_atomic_##NAME##b, \ 3049 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ 3050 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ 3051 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ 3052 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ 3053 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ 3054 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ 3055 }; \ 3056 void tcg_gen_atomic_##NAME##_i32 \ 3057 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \ 3058 { \ 3059 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ 3060 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ 3061 } else { \ 3062 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ 3063 tcg_gen_##OP##_i32); \ 3064 } \ 3065 } \ 3066 void tcg_gen_atomic_##NAME##_i64 \ 3067 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \ 3068 { \ 3069 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ 3070 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ 3071 } else { \ 3072 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ 3073 tcg_gen_##OP##_i64); \ 3074 } \ 3075 } 3076 3077 GEN_ATOMIC_HELPER(fetch_add, add, 0) 3078 GEN_ATOMIC_HELPER(fetch_and, and, 0) 3079 GEN_ATOMIC_HELPER(fetch_or, or, 0) 3080 GEN_ATOMIC_HELPER(fetch_xor, xor, 0) 3081 GEN_ATOMIC_HELPER(fetch_smin, smin, 0) 3082 GEN_ATOMIC_HELPER(fetch_umin, umin, 0) 3083 GEN_ATOMIC_HELPER(fetch_smax, smax, 0) 3084 GEN_ATOMIC_HELPER(fetch_umax, umax, 0) 3085 3086 GEN_ATOMIC_HELPER(add_fetch, add, 1) 3087 GEN_ATOMIC_HELPER(and_fetch, and, 1) 3088 GEN_ATOMIC_HELPER(or_fetch, or, 1) 3089 GEN_ATOMIC_HELPER(xor_fetch, xor, 1) 3090 GEN_ATOMIC_HELPER(smin_fetch, smin, 1) 3091 GEN_ATOMIC_HELPER(umin_fetch, umin, 1) 3092 GEN_ATOMIC_HELPER(smax_fetch, smax, 1) 3093 GEN_ATOMIC_HELPER(umax_fetch, umax, 1) 3094 3095 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) 3096 { 3097 tcg_gen_mov_i32(r, b); 3098 } 3099 3100 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) 3101 { 3102 tcg_gen_mov_i64(r, b); 3103 } 3104 3105 GEN_ATOMIC_HELPER(xchg, mov2, 0) 3106 3107 #undef GEN_ATOMIC_HELPER 3108