1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "qemu-common.h" 27 #include "cpu.h" 28 #include "exec/exec-all.h" 29 #include "tcg.h" 30 #include "tcg-op.h" 31 #include "tcg-mo.h" 32 #include "trace-tcg.h" 33 #include "trace/mem.h" 34 35 /* Reduce the number of ifdefs below. This assumes that all uses of 36 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that 37 the compiler can eliminate. */ 38 #if TCG_TARGET_REG_BITS == 64 39 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); 40 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); 41 #define TCGV_LOW TCGV_LOW_link_error 42 #define TCGV_HIGH TCGV_HIGH_link_error 43 #endif 44 45 void tcg_gen_op1(TCGOpcode opc, TCGArg a1) 46 { 47 TCGOp *op = tcg_emit_op(opc); 48 op->args[0] = a1; 49 } 50 51 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2) 52 { 53 TCGOp *op = tcg_emit_op(opc); 54 op->args[0] = a1; 55 op->args[1] = a2; 56 } 57 58 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) 59 { 60 TCGOp *op = tcg_emit_op(opc); 61 op->args[0] = a1; 62 op->args[1] = a2; 63 op->args[2] = a3; 64 } 65 66 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4) 67 { 68 TCGOp *op = tcg_emit_op(opc); 69 op->args[0] = a1; 70 op->args[1] = a2; 71 op->args[2] = a3; 72 op->args[3] = a4; 73 } 74 75 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 76 TCGArg a4, TCGArg a5) 77 { 78 TCGOp *op = tcg_emit_op(opc); 79 op->args[0] = a1; 80 op->args[1] = a2; 81 op->args[2] = a3; 82 op->args[3] = a4; 83 op->args[4] = a5; 84 } 85 86 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 87 TCGArg a4, TCGArg a5, TCGArg a6) 88 { 89 TCGOp *op = tcg_emit_op(opc); 90 op->args[0] = a1; 91 op->args[1] = a2; 92 op->args[2] = a3; 93 op->args[3] = a4; 94 op->args[4] = a5; 95 op->args[5] = a6; 96 } 97 98 void tcg_gen_mb(TCGBar mb_type) 99 { 100 if (tcg_ctx->tb_cflags & CF_PARALLEL) { 101 tcg_gen_op1(INDEX_op_mb, mb_type); 102 } 103 } 104 105 /* 32 bit ops */ 106 107 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 108 { 109 /* some cases can be optimized here */ 110 if (arg2 == 0) { 111 tcg_gen_mov_i32(ret, arg1); 112 } else { 113 TCGv_i32 t0 = tcg_const_i32(arg2); 114 tcg_gen_add_i32(ret, arg1, t0); 115 tcg_temp_free_i32(t0); 116 } 117 } 118 119 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 120 { 121 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) { 122 /* Don't recurse with tcg_gen_neg_i32. */ 123 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2); 124 } else { 125 TCGv_i32 t0 = tcg_const_i32(arg1); 126 tcg_gen_sub_i32(ret, t0, arg2); 127 tcg_temp_free_i32(t0); 128 } 129 } 130 131 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 132 { 133 /* some cases can be optimized here */ 134 if (arg2 == 0) { 135 tcg_gen_mov_i32(ret, arg1); 136 } else { 137 TCGv_i32 t0 = tcg_const_i32(arg2); 138 tcg_gen_sub_i32(ret, arg1, t0); 139 tcg_temp_free_i32(t0); 140 } 141 } 142 143 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 144 { 145 TCGv_i32 t0; 146 /* Some cases can be optimized here. */ 147 switch (arg2) { 148 case 0: 149 tcg_gen_movi_i32(ret, 0); 150 return; 151 case 0xffffffffu: 152 tcg_gen_mov_i32(ret, arg1); 153 return; 154 case 0xffu: 155 /* Don't recurse with tcg_gen_ext8u_i32. */ 156 if (TCG_TARGET_HAS_ext8u_i32) { 157 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1); 158 return; 159 } 160 break; 161 case 0xffffu: 162 if (TCG_TARGET_HAS_ext16u_i32) { 163 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1); 164 return; 165 } 166 break; 167 } 168 t0 = tcg_const_i32(arg2); 169 tcg_gen_and_i32(ret, arg1, t0); 170 tcg_temp_free_i32(t0); 171 } 172 173 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 174 { 175 /* Some cases can be optimized here. */ 176 if (arg2 == -1) { 177 tcg_gen_movi_i32(ret, -1); 178 } else if (arg2 == 0) { 179 tcg_gen_mov_i32(ret, arg1); 180 } else { 181 TCGv_i32 t0 = tcg_const_i32(arg2); 182 tcg_gen_or_i32(ret, arg1, t0); 183 tcg_temp_free_i32(t0); 184 } 185 } 186 187 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 188 { 189 /* Some cases can be optimized here. */ 190 if (arg2 == 0) { 191 tcg_gen_mov_i32(ret, arg1); 192 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) { 193 /* Don't recurse with tcg_gen_not_i32. */ 194 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1); 195 } else { 196 TCGv_i32 t0 = tcg_const_i32(arg2); 197 tcg_gen_xor_i32(ret, arg1, t0); 198 tcg_temp_free_i32(t0); 199 } 200 } 201 202 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 203 { 204 tcg_debug_assert(arg2 < 32); 205 if (arg2 == 0) { 206 tcg_gen_mov_i32(ret, arg1); 207 } else { 208 TCGv_i32 t0 = tcg_const_i32(arg2); 209 tcg_gen_shl_i32(ret, arg1, t0); 210 tcg_temp_free_i32(t0); 211 } 212 } 213 214 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 215 { 216 tcg_debug_assert(arg2 < 32); 217 if (arg2 == 0) { 218 tcg_gen_mov_i32(ret, arg1); 219 } else { 220 TCGv_i32 t0 = tcg_const_i32(arg2); 221 tcg_gen_shr_i32(ret, arg1, t0); 222 tcg_temp_free_i32(t0); 223 } 224 } 225 226 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 227 { 228 tcg_debug_assert(arg2 < 32); 229 if (arg2 == 0) { 230 tcg_gen_mov_i32(ret, arg1); 231 } else { 232 TCGv_i32 t0 = tcg_const_i32(arg2); 233 tcg_gen_sar_i32(ret, arg1, t0); 234 tcg_temp_free_i32(t0); 235 } 236 } 237 238 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 239 { 240 if (cond == TCG_COND_ALWAYS) { 241 tcg_gen_br(l); 242 } else if (cond != TCG_COND_NEVER) { 243 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l)); 244 } 245 } 246 247 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 248 { 249 if (cond == TCG_COND_ALWAYS) { 250 tcg_gen_br(l); 251 } else if (cond != TCG_COND_NEVER) { 252 TCGv_i32 t0 = tcg_const_i32(arg2); 253 tcg_gen_brcond_i32(cond, arg1, t0, l); 254 tcg_temp_free_i32(t0); 255 } 256 } 257 258 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 259 TCGv_i32 arg1, TCGv_i32 arg2) 260 { 261 if (cond == TCG_COND_ALWAYS) { 262 tcg_gen_movi_i32(ret, 1); 263 } else if (cond == TCG_COND_NEVER) { 264 tcg_gen_movi_i32(ret, 0); 265 } else { 266 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 267 } 268 } 269 270 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 271 TCGv_i32 arg1, int32_t arg2) 272 { 273 TCGv_i32 t0 = tcg_const_i32(arg2); 274 tcg_gen_setcond_i32(cond, ret, arg1, t0); 275 tcg_temp_free_i32(t0); 276 } 277 278 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 279 { 280 TCGv_i32 t0 = tcg_const_i32(arg2); 281 tcg_gen_mul_i32(ret, arg1, t0); 282 tcg_temp_free_i32(t0); 283 } 284 285 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 286 { 287 if (TCG_TARGET_HAS_div_i32) { 288 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2); 289 } else if (TCG_TARGET_HAS_div2_i32) { 290 TCGv_i32 t0 = tcg_temp_new_i32(); 291 tcg_gen_sari_i32(t0, arg1, 31); 292 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2); 293 tcg_temp_free_i32(t0); 294 } else { 295 gen_helper_div_i32(ret, arg1, arg2); 296 } 297 } 298 299 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 300 { 301 if (TCG_TARGET_HAS_rem_i32) { 302 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2); 303 } else if (TCG_TARGET_HAS_div_i32) { 304 TCGv_i32 t0 = tcg_temp_new_i32(); 305 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2); 306 tcg_gen_mul_i32(t0, t0, arg2); 307 tcg_gen_sub_i32(ret, arg1, t0); 308 tcg_temp_free_i32(t0); 309 } else if (TCG_TARGET_HAS_div2_i32) { 310 TCGv_i32 t0 = tcg_temp_new_i32(); 311 tcg_gen_sari_i32(t0, arg1, 31); 312 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2); 313 tcg_temp_free_i32(t0); 314 } else { 315 gen_helper_rem_i32(ret, arg1, arg2); 316 } 317 } 318 319 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 320 { 321 if (TCG_TARGET_HAS_div_i32) { 322 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2); 323 } else if (TCG_TARGET_HAS_div2_i32) { 324 TCGv_i32 t0 = tcg_temp_new_i32(); 325 tcg_gen_movi_i32(t0, 0); 326 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2); 327 tcg_temp_free_i32(t0); 328 } else { 329 gen_helper_divu_i32(ret, arg1, arg2); 330 } 331 } 332 333 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 334 { 335 if (TCG_TARGET_HAS_rem_i32) { 336 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2); 337 } else if (TCG_TARGET_HAS_div_i32) { 338 TCGv_i32 t0 = tcg_temp_new_i32(); 339 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2); 340 tcg_gen_mul_i32(t0, t0, arg2); 341 tcg_gen_sub_i32(ret, arg1, t0); 342 tcg_temp_free_i32(t0); 343 } else if (TCG_TARGET_HAS_div2_i32) { 344 TCGv_i32 t0 = tcg_temp_new_i32(); 345 tcg_gen_movi_i32(t0, 0); 346 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2); 347 tcg_temp_free_i32(t0); 348 } else { 349 gen_helper_remu_i32(ret, arg1, arg2); 350 } 351 } 352 353 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 354 { 355 if (TCG_TARGET_HAS_andc_i32) { 356 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2); 357 } else { 358 TCGv_i32 t0 = tcg_temp_new_i32(); 359 tcg_gen_not_i32(t0, arg2); 360 tcg_gen_and_i32(ret, arg1, t0); 361 tcg_temp_free_i32(t0); 362 } 363 } 364 365 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 366 { 367 if (TCG_TARGET_HAS_eqv_i32) { 368 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2); 369 } else { 370 tcg_gen_xor_i32(ret, arg1, arg2); 371 tcg_gen_not_i32(ret, ret); 372 } 373 } 374 375 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 376 { 377 if (TCG_TARGET_HAS_nand_i32) { 378 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2); 379 } else { 380 tcg_gen_and_i32(ret, arg1, arg2); 381 tcg_gen_not_i32(ret, ret); 382 } 383 } 384 385 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 386 { 387 if (TCG_TARGET_HAS_nor_i32) { 388 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2); 389 } else { 390 tcg_gen_or_i32(ret, arg1, arg2); 391 tcg_gen_not_i32(ret, ret); 392 } 393 } 394 395 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 396 { 397 if (TCG_TARGET_HAS_orc_i32) { 398 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2); 399 } else { 400 TCGv_i32 t0 = tcg_temp_new_i32(); 401 tcg_gen_not_i32(t0, arg2); 402 tcg_gen_or_i32(ret, arg1, t0); 403 tcg_temp_free_i32(t0); 404 } 405 } 406 407 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 408 { 409 if (TCG_TARGET_HAS_clz_i32) { 410 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2); 411 } else if (TCG_TARGET_HAS_clz_i64) { 412 TCGv_i64 t1 = tcg_temp_new_i64(); 413 TCGv_i64 t2 = tcg_temp_new_i64(); 414 tcg_gen_extu_i32_i64(t1, arg1); 415 tcg_gen_extu_i32_i64(t2, arg2); 416 tcg_gen_addi_i64(t2, t2, 32); 417 tcg_gen_clz_i64(t1, t1, t2); 418 tcg_gen_extrl_i64_i32(ret, t1); 419 tcg_temp_free_i64(t1); 420 tcg_temp_free_i64(t2); 421 tcg_gen_subi_i32(ret, ret, 32); 422 } else { 423 gen_helper_clz_i32(ret, arg1, arg2); 424 } 425 } 426 427 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 428 { 429 TCGv_i32 t = tcg_const_i32(arg2); 430 tcg_gen_clz_i32(ret, arg1, t); 431 tcg_temp_free_i32(t); 432 } 433 434 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 435 { 436 if (TCG_TARGET_HAS_ctz_i32) { 437 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); 438 } else if (TCG_TARGET_HAS_ctz_i64) { 439 TCGv_i64 t1 = tcg_temp_new_i64(); 440 TCGv_i64 t2 = tcg_temp_new_i64(); 441 tcg_gen_extu_i32_i64(t1, arg1); 442 tcg_gen_extu_i32_i64(t2, arg2); 443 tcg_gen_ctz_i64(t1, t1, t2); 444 tcg_gen_extrl_i64_i32(ret, t1); 445 tcg_temp_free_i64(t1); 446 tcg_temp_free_i64(t2); 447 } else if (TCG_TARGET_HAS_ctpop_i32 448 || TCG_TARGET_HAS_ctpop_i64 449 || TCG_TARGET_HAS_clz_i32 450 || TCG_TARGET_HAS_clz_i64) { 451 TCGv_i32 z, t = tcg_temp_new_i32(); 452 453 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 454 tcg_gen_subi_i32(t, arg1, 1); 455 tcg_gen_andc_i32(t, t, arg1); 456 tcg_gen_ctpop_i32(t, t); 457 } else { 458 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */ 459 tcg_gen_neg_i32(t, arg1); 460 tcg_gen_and_i32(t, t, arg1); 461 tcg_gen_clzi_i32(t, t, 32); 462 tcg_gen_xori_i32(t, t, 31); 463 } 464 z = tcg_const_i32(0); 465 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 466 tcg_temp_free_i32(t); 467 tcg_temp_free_i32(z); 468 } else { 469 gen_helper_ctz_i32(ret, arg1, arg2); 470 } 471 } 472 473 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 474 { 475 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 476 /* This equivalence has the advantage of not requiring a fixup. */ 477 TCGv_i32 t = tcg_temp_new_i32(); 478 tcg_gen_subi_i32(t, arg1, 1); 479 tcg_gen_andc_i32(t, t, arg1); 480 tcg_gen_ctpop_i32(ret, t); 481 tcg_temp_free_i32(t); 482 } else { 483 TCGv_i32 t = tcg_const_i32(arg2); 484 tcg_gen_ctz_i32(ret, arg1, t); 485 tcg_temp_free_i32(t); 486 } 487 } 488 489 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 490 { 491 if (TCG_TARGET_HAS_clz_i32) { 492 TCGv_i32 t = tcg_temp_new_i32(); 493 tcg_gen_sari_i32(t, arg, 31); 494 tcg_gen_xor_i32(t, t, arg); 495 tcg_gen_clzi_i32(t, t, 32); 496 tcg_gen_subi_i32(ret, t, 1); 497 tcg_temp_free_i32(t); 498 } else { 499 gen_helper_clrsb_i32(ret, arg); 500 } 501 } 502 503 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 504 { 505 if (TCG_TARGET_HAS_ctpop_i32) { 506 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 507 } else if (TCG_TARGET_HAS_ctpop_i64) { 508 TCGv_i64 t = tcg_temp_new_i64(); 509 tcg_gen_extu_i32_i64(t, arg1); 510 tcg_gen_ctpop_i64(t, t); 511 tcg_gen_extrl_i64_i32(ret, t); 512 tcg_temp_free_i64(t); 513 } else { 514 gen_helper_ctpop_i32(ret, arg1); 515 } 516 } 517 518 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 519 { 520 if (TCG_TARGET_HAS_rot_i32) { 521 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2); 522 } else { 523 TCGv_i32 t0, t1; 524 525 t0 = tcg_temp_new_i32(); 526 t1 = tcg_temp_new_i32(); 527 tcg_gen_shl_i32(t0, arg1, arg2); 528 tcg_gen_subfi_i32(t1, 32, arg2); 529 tcg_gen_shr_i32(t1, arg1, t1); 530 tcg_gen_or_i32(ret, t0, t1); 531 tcg_temp_free_i32(t0); 532 tcg_temp_free_i32(t1); 533 } 534 } 535 536 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 537 { 538 tcg_debug_assert(arg2 < 32); 539 /* some cases can be optimized here */ 540 if (arg2 == 0) { 541 tcg_gen_mov_i32(ret, arg1); 542 } else if (TCG_TARGET_HAS_rot_i32) { 543 TCGv_i32 t0 = tcg_const_i32(arg2); 544 tcg_gen_rotl_i32(ret, arg1, t0); 545 tcg_temp_free_i32(t0); 546 } else { 547 TCGv_i32 t0, t1; 548 t0 = tcg_temp_new_i32(); 549 t1 = tcg_temp_new_i32(); 550 tcg_gen_shli_i32(t0, arg1, arg2); 551 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 552 tcg_gen_or_i32(ret, t0, t1); 553 tcg_temp_free_i32(t0); 554 tcg_temp_free_i32(t1); 555 } 556 } 557 558 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 559 { 560 if (TCG_TARGET_HAS_rot_i32) { 561 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2); 562 } else { 563 TCGv_i32 t0, t1; 564 565 t0 = tcg_temp_new_i32(); 566 t1 = tcg_temp_new_i32(); 567 tcg_gen_shr_i32(t0, arg1, arg2); 568 tcg_gen_subfi_i32(t1, 32, arg2); 569 tcg_gen_shl_i32(t1, arg1, t1); 570 tcg_gen_or_i32(ret, t0, t1); 571 tcg_temp_free_i32(t0); 572 tcg_temp_free_i32(t1); 573 } 574 } 575 576 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 577 { 578 tcg_debug_assert(arg2 < 32); 579 /* some cases can be optimized here */ 580 if (arg2 == 0) { 581 tcg_gen_mov_i32(ret, arg1); 582 } else { 583 tcg_gen_rotli_i32(ret, arg1, 32 - arg2); 584 } 585 } 586 587 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 588 unsigned int ofs, unsigned int len) 589 { 590 uint32_t mask; 591 TCGv_i32 t1; 592 593 tcg_debug_assert(ofs < 32); 594 tcg_debug_assert(len > 0); 595 tcg_debug_assert(len <= 32); 596 tcg_debug_assert(ofs + len <= 32); 597 598 if (len == 32) { 599 tcg_gen_mov_i32(ret, arg2); 600 return; 601 } 602 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 603 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 604 return; 605 } 606 607 mask = (1u << len) - 1; 608 t1 = tcg_temp_new_i32(); 609 610 if (ofs + len < 32) { 611 tcg_gen_andi_i32(t1, arg2, mask); 612 tcg_gen_shli_i32(t1, t1, ofs); 613 } else { 614 tcg_gen_shli_i32(t1, arg2, ofs); 615 } 616 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 617 tcg_gen_or_i32(ret, ret, t1); 618 619 tcg_temp_free_i32(t1); 620 } 621 622 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 623 unsigned int ofs, unsigned int len) 624 { 625 tcg_debug_assert(ofs < 32); 626 tcg_debug_assert(len > 0); 627 tcg_debug_assert(len <= 32); 628 tcg_debug_assert(ofs + len <= 32); 629 630 if (ofs + len == 32) { 631 tcg_gen_shli_i32(ret, arg, ofs); 632 } else if (ofs == 0) { 633 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 634 } else if (TCG_TARGET_HAS_deposit_i32 635 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 636 TCGv_i32 zero = tcg_const_i32(0); 637 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 638 tcg_temp_free_i32(zero); 639 } else { 640 /* To help two-operand hosts we prefer to zero-extend first, 641 which allows ARG to stay live. */ 642 switch (len) { 643 case 16: 644 if (TCG_TARGET_HAS_ext16u_i32) { 645 tcg_gen_ext16u_i32(ret, arg); 646 tcg_gen_shli_i32(ret, ret, ofs); 647 return; 648 } 649 break; 650 case 8: 651 if (TCG_TARGET_HAS_ext8u_i32) { 652 tcg_gen_ext8u_i32(ret, arg); 653 tcg_gen_shli_i32(ret, ret, ofs); 654 return; 655 } 656 break; 657 } 658 /* Otherwise prefer zero-extension over AND for code size. */ 659 switch (ofs + len) { 660 case 16: 661 if (TCG_TARGET_HAS_ext16u_i32) { 662 tcg_gen_shli_i32(ret, arg, ofs); 663 tcg_gen_ext16u_i32(ret, ret); 664 return; 665 } 666 break; 667 case 8: 668 if (TCG_TARGET_HAS_ext8u_i32) { 669 tcg_gen_shli_i32(ret, arg, ofs); 670 tcg_gen_ext8u_i32(ret, ret); 671 return; 672 } 673 break; 674 } 675 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 676 tcg_gen_shli_i32(ret, ret, ofs); 677 } 678 } 679 680 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 681 unsigned int ofs, unsigned int len) 682 { 683 tcg_debug_assert(ofs < 32); 684 tcg_debug_assert(len > 0); 685 tcg_debug_assert(len <= 32); 686 tcg_debug_assert(ofs + len <= 32); 687 688 /* Canonicalize certain special cases, even if extract is supported. */ 689 if (ofs + len == 32) { 690 tcg_gen_shri_i32(ret, arg, 32 - len); 691 return; 692 } 693 if (ofs == 0) { 694 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 695 return; 696 } 697 698 if (TCG_TARGET_HAS_extract_i32 699 && TCG_TARGET_extract_i32_valid(ofs, len)) { 700 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 701 return; 702 } 703 704 /* Assume that zero-extension, if available, is cheaper than a shift. */ 705 switch (ofs + len) { 706 case 16: 707 if (TCG_TARGET_HAS_ext16u_i32) { 708 tcg_gen_ext16u_i32(ret, arg); 709 tcg_gen_shri_i32(ret, ret, ofs); 710 return; 711 } 712 break; 713 case 8: 714 if (TCG_TARGET_HAS_ext8u_i32) { 715 tcg_gen_ext8u_i32(ret, arg); 716 tcg_gen_shri_i32(ret, ret, ofs); 717 return; 718 } 719 break; 720 } 721 722 /* ??? Ideally we'd know what values are available for immediate AND. 723 Assume that 8 bits are available, plus the special case of 16, 724 so that we get ext8u, ext16u. */ 725 switch (len) { 726 case 1 ... 8: case 16: 727 tcg_gen_shri_i32(ret, arg, ofs); 728 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 729 break; 730 default: 731 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 732 tcg_gen_shri_i32(ret, ret, 32 - len); 733 break; 734 } 735 } 736 737 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 738 unsigned int ofs, unsigned int len) 739 { 740 tcg_debug_assert(ofs < 32); 741 tcg_debug_assert(len > 0); 742 tcg_debug_assert(len <= 32); 743 tcg_debug_assert(ofs + len <= 32); 744 745 /* Canonicalize certain special cases, even if extract is supported. */ 746 if (ofs + len == 32) { 747 tcg_gen_sari_i32(ret, arg, 32 - len); 748 return; 749 } 750 if (ofs == 0) { 751 switch (len) { 752 case 16: 753 tcg_gen_ext16s_i32(ret, arg); 754 return; 755 case 8: 756 tcg_gen_ext8s_i32(ret, arg); 757 return; 758 } 759 } 760 761 if (TCG_TARGET_HAS_sextract_i32 762 && TCG_TARGET_extract_i32_valid(ofs, len)) { 763 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 764 return; 765 } 766 767 /* Assume that sign-extension, if available, is cheaper than a shift. */ 768 switch (ofs + len) { 769 case 16: 770 if (TCG_TARGET_HAS_ext16s_i32) { 771 tcg_gen_ext16s_i32(ret, arg); 772 tcg_gen_sari_i32(ret, ret, ofs); 773 return; 774 } 775 break; 776 case 8: 777 if (TCG_TARGET_HAS_ext8s_i32) { 778 tcg_gen_ext8s_i32(ret, arg); 779 tcg_gen_sari_i32(ret, ret, ofs); 780 return; 781 } 782 break; 783 } 784 switch (len) { 785 case 16: 786 if (TCG_TARGET_HAS_ext16s_i32) { 787 tcg_gen_shri_i32(ret, arg, ofs); 788 tcg_gen_ext16s_i32(ret, ret); 789 return; 790 } 791 break; 792 case 8: 793 if (TCG_TARGET_HAS_ext8s_i32) { 794 tcg_gen_shri_i32(ret, arg, ofs); 795 tcg_gen_ext8s_i32(ret, ret); 796 return; 797 } 798 break; 799 } 800 801 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 802 tcg_gen_sari_i32(ret, ret, 32 - len); 803 } 804 805 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 806 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 807 { 808 if (cond == TCG_COND_ALWAYS) { 809 tcg_gen_mov_i32(ret, v1); 810 } else if (cond == TCG_COND_NEVER) { 811 tcg_gen_mov_i32(ret, v2); 812 } else if (TCG_TARGET_HAS_movcond_i32) { 813 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 814 } else { 815 TCGv_i32 t0 = tcg_temp_new_i32(); 816 TCGv_i32 t1 = tcg_temp_new_i32(); 817 tcg_gen_setcond_i32(cond, t0, c1, c2); 818 tcg_gen_neg_i32(t0, t0); 819 tcg_gen_and_i32(t1, v1, t0); 820 tcg_gen_andc_i32(ret, v2, t0); 821 tcg_gen_or_i32(ret, ret, t1); 822 tcg_temp_free_i32(t0); 823 tcg_temp_free_i32(t1); 824 } 825 } 826 827 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 828 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 829 { 830 if (TCG_TARGET_HAS_add2_i32) { 831 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 832 } else { 833 TCGv_i64 t0 = tcg_temp_new_i64(); 834 TCGv_i64 t1 = tcg_temp_new_i64(); 835 tcg_gen_concat_i32_i64(t0, al, ah); 836 tcg_gen_concat_i32_i64(t1, bl, bh); 837 tcg_gen_add_i64(t0, t0, t1); 838 tcg_gen_extr_i64_i32(rl, rh, t0); 839 tcg_temp_free_i64(t0); 840 tcg_temp_free_i64(t1); 841 } 842 } 843 844 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 845 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 846 { 847 if (TCG_TARGET_HAS_sub2_i32) { 848 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 849 } else { 850 TCGv_i64 t0 = tcg_temp_new_i64(); 851 TCGv_i64 t1 = tcg_temp_new_i64(); 852 tcg_gen_concat_i32_i64(t0, al, ah); 853 tcg_gen_concat_i32_i64(t1, bl, bh); 854 tcg_gen_sub_i64(t0, t0, t1); 855 tcg_gen_extr_i64_i32(rl, rh, t0); 856 tcg_temp_free_i64(t0); 857 tcg_temp_free_i64(t1); 858 } 859 } 860 861 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 862 { 863 if (TCG_TARGET_HAS_mulu2_i32) { 864 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 865 } else if (TCG_TARGET_HAS_muluh_i32) { 866 TCGv_i32 t = tcg_temp_new_i32(); 867 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 868 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2); 869 tcg_gen_mov_i32(rl, t); 870 tcg_temp_free_i32(t); 871 } else { 872 TCGv_i64 t0 = tcg_temp_new_i64(); 873 TCGv_i64 t1 = tcg_temp_new_i64(); 874 tcg_gen_extu_i32_i64(t0, arg1); 875 tcg_gen_extu_i32_i64(t1, arg2); 876 tcg_gen_mul_i64(t0, t0, t1); 877 tcg_gen_extr_i64_i32(rl, rh, t0); 878 tcg_temp_free_i64(t0); 879 tcg_temp_free_i64(t1); 880 } 881 } 882 883 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 884 { 885 if (TCG_TARGET_HAS_muls2_i32) { 886 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 887 } else if (TCG_TARGET_HAS_mulsh_i32) { 888 TCGv_i32 t = tcg_temp_new_i32(); 889 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 890 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2); 891 tcg_gen_mov_i32(rl, t); 892 tcg_temp_free_i32(t); 893 } else if (TCG_TARGET_REG_BITS == 32) { 894 TCGv_i32 t0 = tcg_temp_new_i32(); 895 TCGv_i32 t1 = tcg_temp_new_i32(); 896 TCGv_i32 t2 = tcg_temp_new_i32(); 897 TCGv_i32 t3 = tcg_temp_new_i32(); 898 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 899 /* Adjust for negative inputs. */ 900 tcg_gen_sari_i32(t2, arg1, 31); 901 tcg_gen_sari_i32(t3, arg2, 31); 902 tcg_gen_and_i32(t2, t2, arg2); 903 tcg_gen_and_i32(t3, t3, arg1); 904 tcg_gen_sub_i32(rh, t1, t2); 905 tcg_gen_sub_i32(rh, rh, t3); 906 tcg_gen_mov_i32(rl, t0); 907 tcg_temp_free_i32(t0); 908 tcg_temp_free_i32(t1); 909 tcg_temp_free_i32(t2); 910 tcg_temp_free_i32(t3); 911 } else { 912 TCGv_i64 t0 = tcg_temp_new_i64(); 913 TCGv_i64 t1 = tcg_temp_new_i64(); 914 tcg_gen_ext_i32_i64(t0, arg1); 915 tcg_gen_ext_i32_i64(t1, arg2); 916 tcg_gen_mul_i64(t0, t0, t1); 917 tcg_gen_extr_i64_i32(rl, rh, t0); 918 tcg_temp_free_i64(t0); 919 tcg_temp_free_i64(t1); 920 } 921 } 922 923 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 924 { 925 if (TCG_TARGET_REG_BITS == 32) { 926 TCGv_i32 t0 = tcg_temp_new_i32(); 927 TCGv_i32 t1 = tcg_temp_new_i32(); 928 TCGv_i32 t2 = tcg_temp_new_i32(); 929 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 930 /* Adjust for negative input for the signed arg1. */ 931 tcg_gen_sari_i32(t2, arg1, 31); 932 tcg_gen_and_i32(t2, t2, arg2); 933 tcg_gen_sub_i32(rh, t1, t2); 934 tcg_gen_mov_i32(rl, t0); 935 tcg_temp_free_i32(t0); 936 tcg_temp_free_i32(t1); 937 tcg_temp_free_i32(t2); 938 } else { 939 TCGv_i64 t0 = tcg_temp_new_i64(); 940 TCGv_i64 t1 = tcg_temp_new_i64(); 941 tcg_gen_ext_i32_i64(t0, arg1); 942 tcg_gen_extu_i32_i64(t1, arg2); 943 tcg_gen_mul_i64(t0, t0, t1); 944 tcg_gen_extr_i64_i32(rl, rh, t0); 945 tcg_temp_free_i64(t0); 946 tcg_temp_free_i64(t1); 947 } 948 } 949 950 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 951 { 952 if (TCG_TARGET_HAS_ext8s_i32) { 953 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg); 954 } else { 955 tcg_gen_shli_i32(ret, arg, 24); 956 tcg_gen_sari_i32(ret, ret, 24); 957 } 958 } 959 960 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 961 { 962 if (TCG_TARGET_HAS_ext16s_i32) { 963 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg); 964 } else { 965 tcg_gen_shli_i32(ret, arg, 16); 966 tcg_gen_sari_i32(ret, ret, 16); 967 } 968 } 969 970 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 971 { 972 if (TCG_TARGET_HAS_ext8u_i32) { 973 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg); 974 } else { 975 tcg_gen_andi_i32(ret, arg, 0xffu); 976 } 977 } 978 979 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 980 { 981 if (TCG_TARGET_HAS_ext16u_i32) { 982 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg); 983 } else { 984 tcg_gen_andi_i32(ret, arg, 0xffffu); 985 } 986 } 987 988 /* Note: we assume the two high bytes are set to zero */ 989 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg) 990 { 991 if (TCG_TARGET_HAS_bswap16_i32) { 992 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg); 993 } else { 994 TCGv_i32 t0 = tcg_temp_new_i32(); 995 996 tcg_gen_ext8u_i32(t0, arg); 997 tcg_gen_shli_i32(t0, t0, 8); 998 tcg_gen_shri_i32(ret, arg, 8); 999 tcg_gen_or_i32(ret, ret, t0); 1000 tcg_temp_free_i32(t0); 1001 } 1002 } 1003 1004 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1005 { 1006 if (TCG_TARGET_HAS_bswap32_i32) { 1007 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg); 1008 } else { 1009 TCGv_i32 t0, t1; 1010 t0 = tcg_temp_new_i32(); 1011 t1 = tcg_temp_new_i32(); 1012 1013 tcg_gen_shli_i32(t0, arg, 24); 1014 1015 tcg_gen_andi_i32(t1, arg, 0x0000ff00); 1016 tcg_gen_shli_i32(t1, t1, 8); 1017 tcg_gen_or_i32(t0, t0, t1); 1018 1019 tcg_gen_shri_i32(t1, arg, 8); 1020 tcg_gen_andi_i32(t1, t1, 0x0000ff00); 1021 tcg_gen_or_i32(t0, t0, t1); 1022 1023 tcg_gen_shri_i32(t1, arg, 24); 1024 tcg_gen_or_i32(ret, t0, t1); 1025 tcg_temp_free_i32(t0); 1026 tcg_temp_free_i32(t1); 1027 } 1028 } 1029 1030 /* 64-bit ops */ 1031 1032 #if TCG_TARGET_REG_BITS == 32 1033 /* These are all inline for TCG_TARGET_REG_BITS == 64. */ 1034 1035 void tcg_gen_discard_i64(TCGv_i64 arg) 1036 { 1037 tcg_gen_discard_i32(TCGV_LOW(arg)); 1038 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1039 } 1040 1041 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1042 { 1043 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1044 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1045 } 1046 1047 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1048 { 1049 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1050 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1051 } 1052 1053 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1054 { 1055 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1056 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1057 } 1058 1059 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1060 { 1061 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1062 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1063 } 1064 1065 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1066 { 1067 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1068 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1069 } 1070 1071 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1072 { 1073 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1074 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1075 } 1076 1077 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1078 { 1079 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1080 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1081 } 1082 1083 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1084 { 1085 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1086 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1087 } 1088 1089 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1090 { 1091 /* Since arg2 and ret have different types, 1092 they cannot be the same temporary */ 1093 #ifdef HOST_WORDS_BIGENDIAN 1094 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1095 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1096 #else 1097 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1098 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1099 #endif 1100 } 1101 1102 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1103 { 1104 #ifdef HOST_WORDS_BIGENDIAN 1105 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1106 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1107 #else 1108 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1109 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1110 #endif 1111 } 1112 1113 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1114 { 1115 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1116 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1117 } 1118 1119 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1120 { 1121 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1122 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1123 } 1124 1125 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1126 { 1127 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1128 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1129 } 1130 1131 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1132 { 1133 gen_helper_shl_i64(ret, arg1, arg2); 1134 } 1135 1136 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1137 { 1138 gen_helper_shr_i64(ret, arg1, arg2); 1139 } 1140 1141 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1142 { 1143 gen_helper_sar_i64(ret, arg1, arg2); 1144 } 1145 1146 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1147 { 1148 TCGv_i64 t0; 1149 TCGv_i32 t1; 1150 1151 t0 = tcg_temp_new_i64(); 1152 t1 = tcg_temp_new_i32(); 1153 1154 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1155 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1156 1157 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1158 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1159 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1160 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1161 1162 tcg_gen_mov_i64(ret, t0); 1163 tcg_temp_free_i64(t0); 1164 tcg_temp_free_i32(t1); 1165 } 1166 #endif /* TCG_TARGET_REG_SIZE == 32 */ 1167 1168 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1169 { 1170 /* some cases can be optimized here */ 1171 if (arg2 == 0) { 1172 tcg_gen_mov_i64(ret, arg1); 1173 } else { 1174 TCGv_i64 t0 = tcg_const_i64(arg2); 1175 tcg_gen_add_i64(ret, arg1, t0); 1176 tcg_temp_free_i64(t0); 1177 } 1178 } 1179 1180 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1181 { 1182 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) { 1183 /* Don't recurse with tcg_gen_neg_i64. */ 1184 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2); 1185 } else { 1186 TCGv_i64 t0 = tcg_const_i64(arg1); 1187 tcg_gen_sub_i64(ret, t0, arg2); 1188 tcg_temp_free_i64(t0); 1189 } 1190 } 1191 1192 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1193 { 1194 /* some cases can be optimized here */ 1195 if (arg2 == 0) { 1196 tcg_gen_mov_i64(ret, arg1); 1197 } else { 1198 TCGv_i64 t0 = tcg_const_i64(arg2); 1199 tcg_gen_sub_i64(ret, arg1, t0); 1200 tcg_temp_free_i64(t0); 1201 } 1202 } 1203 1204 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1205 { 1206 TCGv_i64 t0; 1207 1208 if (TCG_TARGET_REG_BITS == 32) { 1209 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1210 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1211 return; 1212 } 1213 1214 /* Some cases can be optimized here. */ 1215 switch (arg2) { 1216 case 0: 1217 tcg_gen_movi_i64(ret, 0); 1218 return; 1219 case 0xffffffffffffffffull: 1220 tcg_gen_mov_i64(ret, arg1); 1221 return; 1222 case 0xffull: 1223 /* Don't recurse with tcg_gen_ext8u_i64. */ 1224 if (TCG_TARGET_HAS_ext8u_i64) { 1225 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1); 1226 return; 1227 } 1228 break; 1229 case 0xffffu: 1230 if (TCG_TARGET_HAS_ext16u_i64) { 1231 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1); 1232 return; 1233 } 1234 break; 1235 case 0xffffffffull: 1236 if (TCG_TARGET_HAS_ext32u_i64) { 1237 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1); 1238 return; 1239 } 1240 break; 1241 } 1242 t0 = tcg_const_i64(arg2); 1243 tcg_gen_and_i64(ret, arg1, t0); 1244 tcg_temp_free_i64(t0); 1245 } 1246 1247 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1248 { 1249 if (TCG_TARGET_REG_BITS == 32) { 1250 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1251 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1252 return; 1253 } 1254 /* Some cases can be optimized here. */ 1255 if (arg2 == -1) { 1256 tcg_gen_movi_i64(ret, -1); 1257 } else if (arg2 == 0) { 1258 tcg_gen_mov_i64(ret, arg1); 1259 } else { 1260 TCGv_i64 t0 = tcg_const_i64(arg2); 1261 tcg_gen_or_i64(ret, arg1, t0); 1262 tcg_temp_free_i64(t0); 1263 } 1264 } 1265 1266 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1267 { 1268 if (TCG_TARGET_REG_BITS == 32) { 1269 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1270 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1271 return; 1272 } 1273 /* Some cases can be optimized here. */ 1274 if (arg2 == 0) { 1275 tcg_gen_mov_i64(ret, arg1); 1276 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) { 1277 /* Don't recurse with tcg_gen_not_i64. */ 1278 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1); 1279 } else { 1280 TCGv_i64 t0 = tcg_const_i64(arg2); 1281 tcg_gen_xor_i64(ret, arg1, t0); 1282 tcg_temp_free_i64(t0); 1283 } 1284 } 1285 1286 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1287 unsigned c, bool right, bool arith) 1288 { 1289 tcg_debug_assert(c < 64); 1290 if (c == 0) { 1291 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1292 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1293 } else if (c >= 32) { 1294 c -= 32; 1295 if (right) { 1296 if (arith) { 1297 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1298 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1299 } else { 1300 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1301 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1302 } 1303 } else { 1304 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1305 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1306 } 1307 } else { 1308 TCGv_i32 t0, t1; 1309 1310 t0 = tcg_temp_new_i32(); 1311 t1 = tcg_temp_new_i32(); 1312 if (right) { 1313 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c); 1314 if (arith) { 1315 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c); 1316 } else { 1317 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c); 1318 } 1319 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1320 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0); 1321 tcg_gen_mov_i32(TCGV_HIGH(ret), t1); 1322 } else { 1323 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1324 /* Note: ret can be the same as arg1, so we use t1 */ 1325 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c); 1326 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1327 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0); 1328 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1329 } 1330 tcg_temp_free_i32(t0); 1331 tcg_temp_free_i32(t1); 1332 } 1333 } 1334 1335 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1336 { 1337 tcg_debug_assert(arg2 < 64); 1338 if (TCG_TARGET_REG_BITS == 32) { 1339 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1340 } else if (arg2 == 0) { 1341 tcg_gen_mov_i64(ret, arg1); 1342 } else { 1343 TCGv_i64 t0 = tcg_const_i64(arg2); 1344 tcg_gen_shl_i64(ret, arg1, t0); 1345 tcg_temp_free_i64(t0); 1346 } 1347 } 1348 1349 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1350 { 1351 tcg_debug_assert(arg2 < 64); 1352 if (TCG_TARGET_REG_BITS == 32) { 1353 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1354 } else if (arg2 == 0) { 1355 tcg_gen_mov_i64(ret, arg1); 1356 } else { 1357 TCGv_i64 t0 = tcg_const_i64(arg2); 1358 tcg_gen_shr_i64(ret, arg1, t0); 1359 tcg_temp_free_i64(t0); 1360 } 1361 } 1362 1363 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1364 { 1365 tcg_debug_assert(arg2 < 64); 1366 if (TCG_TARGET_REG_BITS == 32) { 1367 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1368 } else if (arg2 == 0) { 1369 tcg_gen_mov_i64(ret, arg1); 1370 } else { 1371 TCGv_i64 t0 = tcg_const_i64(arg2); 1372 tcg_gen_sar_i64(ret, arg1, t0); 1373 tcg_temp_free_i64(t0); 1374 } 1375 } 1376 1377 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1378 { 1379 if (cond == TCG_COND_ALWAYS) { 1380 tcg_gen_br(l); 1381 } else if (cond != TCG_COND_NEVER) { 1382 if (TCG_TARGET_REG_BITS == 32) { 1383 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1384 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1385 TCGV_HIGH(arg2), cond, label_arg(l)); 1386 } else { 1387 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1388 label_arg(l)); 1389 } 1390 } 1391 } 1392 1393 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1394 { 1395 if (cond == TCG_COND_ALWAYS) { 1396 tcg_gen_br(l); 1397 } else if (cond != TCG_COND_NEVER) { 1398 TCGv_i64 t0 = tcg_const_i64(arg2); 1399 tcg_gen_brcond_i64(cond, arg1, t0, l); 1400 tcg_temp_free_i64(t0); 1401 } 1402 } 1403 1404 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1405 TCGv_i64 arg1, TCGv_i64 arg2) 1406 { 1407 if (cond == TCG_COND_ALWAYS) { 1408 tcg_gen_movi_i64(ret, 1); 1409 } else if (cond == TCG_COND_NEVER) { 1410 tcg_gen_movi_i64(ret, 0); 1411 } else { 1412 if (TCG_TARGET_REG_BITS == 32) { 1413 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1414 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1415 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1416 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1417 } else { 1418 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1419 } 1420 } 1421 } 1422 1423 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1424 TCGv_i64 arg1, int64_t arg2) 1425 { 1426 TCGv_i64 t0 = tcg_const_i64(arg2); 1427 tcg_gen_setcond_i64(cond, ret, arg1, t0); 1428 tcg_temp_free_i64(t0); 1429 } 1430 1431 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1432 { 1433 TCGv_i64 t0 = tcg_const_i64(arg2); 1434 tcg_gen_mul_i64(ret, arg1, t0); 1435 tcg_temp_free_i64(t0); 1436 } 1437 1438 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1439 { 1440 if (TCG_TARGET_HAS_div_i64) { 1441 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2); 1442 } else if (TCG_TARGET_HAS_div2_i64) { 1443 TCGv_i64 t0 = tcg_temp_new_i64(); 1444 tcg_gen_sari_i64(t0, arg1, 63); 1445 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2); 1446 tcg_temp_free_i64(t0); 1447 } else { 1448 gen_helper_div_i64(ret, arg1, arg2); 1449 } 1450 } 1451 1452 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1453 { 1454 if (TCG_TARGET_HAS_rem_i64) { 1455 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2); 1456 } else if (TCG_TARGET_HAS_div_i64) { 1457 TCGv_i64 t0 = tcg_temp_new_i64(); 1458 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2); 1459 tcg_gen_mul_i64(t0, t0, arg2); 1460 tcg_gen_sub_i64(ret, arg1, t0); 1461 tcg_temp_free_i64(t0); 1462 } else if (TCG_TARGET_HAS_div2_i64) { 1463 TCGv_i64 t0 = tcg_temp_new_i64(); 1464 tcg_gen_sari_i64(t0, arg1, 63); 1465 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2); 1466 tcg_temp_free_i64(t0); 1467 } else { 1468 gen_helper_rem_i64(ret, arg1, arg2); 1469 } 1470 } 1471 1472 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1473 { 1474 if (TCG_TARGET_HAS_div_i64) { 1475 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2); 1476 } else if (TCG_TARGET_HAS_div2_i64) { 1477 TCGv_i64 t0 = tcg_temp_new_i64(); 1478 tcg_gen_movi_i64(t0, 0); 1479 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2); 1480 tcg_temp_free_i64(t0); 1481 } else { 1482 gen_helper_divu_i64(ret, arg1, arg2); 1483 } 1484 } 1485 1486 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1487 { 1488 if (TCG_TARGET_HAS_rem_i64) { 1489 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2); 1490 } else if (TCG_TARGET_HAS_div_i64) { 1491 TCGv_i64 t0 = tcg_temp_new_i64(); 1492 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2); 1493 tcg_gen_mul_i64(t0, t0, arg2); 1494 tcg_gen_sub_i64(ret, arg1, t0); 1495 tcg_temp_free_i64(t0); 1496 } else if (TCG_TARGET_HAS_div2_i64) { 1497 TCGv_i64 t0 = tcg_temp_new_i64(); 1498 tcg_gen_movi_i64(t0, 0); 1499 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2); 1500 tcg_temp_free_i64(t0); 1501 } else { 1502 gen_helper_remu_i64(ret, arg1, arg2); 1503 } 1504 } 1505 1506 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 1507 { 1508 if (TCG_TARGET_REG_BITS == 32) { 1509 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1510 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1511 } else if (TCG_TARGET_HAS_ext8s_i64) { 1512 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg); 1513 } else { 1514 tcg_gen_shli_i64(ret, arg, 56); 1515 tcg_gen_sari_i64(ret, ret, 56); 1516 } 1517 } 1518 1519 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 1520 { 1521 if (TCG_TARGET_REG_BITS == 32) { 1522 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1523 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1524 } else if (TCG_TARGET_HAS_ext16s_i64) { 1525 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg); 1526 } else { 1527 tcg_gen_shli_i64(ret, arg, 48); 1528 tcg_gen_sari_i64(ret, ret, 48); 1529 } 1530 } 1531 1532 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 1533 { 1534 if (TCG_TARGET_REG_BITS == 32) { 1535 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1536 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1537 } else if (TCG_TARGET_HAS_ext32s_i64) { 1538 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg); 1539 } else { 1540 tcg_gen_shli_i64(ret, arg, 32); 1541 tcg_gen_sari_i64(ret, ret, 32); 1542 } 1543 } 1544 1545 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 1546 { 1547 if (TCG_TARGET_REG_BITS == 32) { 1548 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1549 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1550 } else if (TCG_TARGET_HAS_ext8u_i64) { 1551 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg); 1552 } else { 1553 tcg_gen_andi_i64(ret, arg, 0xffu); 1554 } 1555 } 1556 1557 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 1558 { 1559 if (TCG_TARGET_REG_BITS == 32) { 1560 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1561 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1562 } else if (TCG_TARGET_HAS_ext16u_i64) { 1563 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg); 1564 } else { 1565 tcg_gen_andi_i64(ret, arg, 0xffffu); 1566 } 1567 } 1568 1569 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 1570 { 1571 if (TCG_TARGET_REG_BITS == 32) { 1572 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1573 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1574 } else if (TCG_TARGET_HAS_ext32u_i64) { 1575 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg); 1576 } else { 1577 tcg_gen_andi_i64(ret, arg, 0xffffffffu); 1578 } 1579 } 1580 1581 /* Note: we assume the six high bytes are set to zero */ 1582 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg) 1583 { 1584 if (TCG_TARGET_REG_BITS == 32) { 1585 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1586 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1587 } else if (TCG_TARGET_HAS_bswap16_i64) { 1588 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg); 1589 } else { 1590 TCGv_i64 t0 = tcg_temp_new_i64(); 1591 1592 tcg_gen_ext8u_i64(t0, arg); 1593 tcg_gen_shli_i64(t0, t0, 8); 1594 tcg_gen_shri_i64(ret, arg, 8); 1595 tcg_gen_or_i64(ret, ret, t0); 1596 tcg_temp_free_i64(t0); 1597 } 1598 } 1599 1600 /* Note: we assume the four high bytes are set to zero */ 1601 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg) 1602 { 1603 if (TCG_TARGET_REG_BITS == 32) { 1604 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1605 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1606 } else if (TCG_TARGET_HAS_bswap32_i64) { 1607 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg); 1608 } else { 1609 TCGv_i64 t0, t1; 1610 t0 = tcg_temp_new_i64(); 1611 t1 = tcg_temp_new_i64(); 1612 1613 tcg_gen_shli_i64(t0, arg, 24); 1614 tcg_gen_ext32u_i64(t0, t0); 1615 1616 tcg_gen_andi_i64(t1, arg, 0x0000ff00); 1617 tcg_gen_shli_i64(t1, t1, 8); 1618 tcg_gen_or_i64(t0, t0, t1); 1619 1620 tcg_gen_shri_i64(t1, arg, 8); 1621 tcg_gen_andi_i64(t1, t1, 0x0000ff00); 1622 tcg_gen_or_i64(t0, t0, t1); 1623 1624 tcg_gen_shri_i64(t1, arg, 24); 1625 tcg_gen_or_i64(ret, t0, t1); 1626 tcg_temp_free_i64(t0); 1627 tcg_temp_free_i64(t1); 1628 } 1629 } 1630 1631 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 1632 { 1633 if (TCG_TARGET_REG_BITS == 32) { 1634 TCGv_i32 t0, t1; 1635 t0 = tcg_temp_new_i32(); 1636 t1 = tcg_temp_new_i32(); 1637 1638 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 1639 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 1640 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1641 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 1642 tcg_temp_free_i32(t0); 1643 tcg_temp_free_i32(t1); 1644 } else if (TCG_TARGET_HAS_bswap64_i64) { 1645 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg); 1646 } else { 1647 TCGv_i64 t0 = tcg_temp_new_i64(); 1648 TCGv_i64 t1 = tcg_temp_new_i64(); 1649 1650 tcg_gen_shli_i64(t0, arg, 56); 1651 1652 tcg_gen_andi_i64(t1, arg, 0x0000ff00); 1653 tcg_gen_shli_i64(t1, t1, 40); 1654 tcg_gen_or_i64(t0, t0, t1); 1655 1656 tcg_gen_andi_i64(t1, arg, 0x00ff0000); 1657 tcg_gen_shli_i64(t1, t1, 24); 1658 tcg_gen_or_i64(t0, t0, t1); 1659 1660 tcg_gen_andi_i64(t1, arg, 0xff000000); 1661 tcg_gen_shli_i64(t1, t1, 8); 1662 tcg_gen_or_i64(t0, t0, t1); 1663 1664 tcg_gen_shri_i64(t1, arg, 8); 1665 tcg_gen_andi_i64(t1, t1, 0xff000000); 1666 tcg_gen_or_i64(t0, t0, t1); 1667 1668 tcg_gen_shri_i64(t1, arg, 24); 1669 tcg_gen_andi_i64(t1, t1, 0x00ff0000); 1670 tcg_gen_or_i64(t0, t0, t1); 1671 1672 tcg_gen_shri_i64(t1, arg, 40); 1673 tcg_gen_andi_i64(t1, t1, 0x0000ff00); 1674 tcg_gen_or_i64(t0, t0, t1); 1675 1676 tcg_gen_shri_i64(t1, arg, 56); 1677 tcg_gen_or_i64(ret, t0, t1); 1678 tcg_temp_free_i64(t0); 1679 tcg_temp_free_i64(t1); 1680 } 1681 } 1682 1683 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 1684 { 1685 if (TCG_TARGET_REG_BITS == 32) { 1686 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1687 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1688 } else if (TCG_TARGET_HAS_not_i64) { 1689 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg); 1690 } else { 1691 tcg_gen_xori_i64(ret, arg, -1); 1692 } 1693 } 1694 1695 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1696 { 1697 if (TCG_TARGET_REG_BITS == 32) { 1698 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1699 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1700 } else if (TCG_TARGET_HAS_andc_i64) { 1701 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2); 1702 } else { 1703 TCGv_i64 t0 = tcg_temp_new_i64(); 1704 tcg_gen_not_i64(t0, arg2); 1705 tcg_gen_and_i64(ret, arg1, t0); 1706 tcg_temp_free_i64(t0); 1707 } 1708 } 1709 1710 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1711 { 1712 if (TCG_TARGET_REG_BITS == 32) { 1713 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1714 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1715 } else if (TCG_TARGET_HAS_eqv_i64) { 1716 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2); 1717 } else { 1718 tcg_gen_xor_i64(ret, arg1, arg2); 1719 tcg_gen_not_i64(ret, ret); 1720 } 1721 } 1722 1723 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1724 { 1725 if (TCG_TARGET_REG_BITS == 32) { 1726 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1727 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1728 } else if (TCG_TARGET_HAS_nand_i64) { 1729 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2); 1730 } else { 1731 tcg_gen_and_i64(ret, arg1, arg2); 1732 tcg_gen_not_i64(ret, ret); 1733 } 1734 } 1735 1736 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1737 { 1738 if (TCG_TARGET_REG_BITS == 32) { 1739 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1740 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1741 } else if (TCG_TARGET_HAS_nor_i64) { 1742 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2); 1743 } else { 1744 tcg_gen_or_i64(ret, arg1, arg2); 1745 tcg_gen_not_i64(ret, ret); 1746 } 1747 } 1748 1749 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1750 { 1751 if (TCG_TARGET_REG_BITS == 32) { 1752 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1753 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1754 } else if (TCG_TARGET_HAS_orc_i64) { 1755 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2); 1756 } else { 1757 TCGv_i64 t0 = tcg_temp_new_i64(); 1758 tcg_gen_not_i64(t0, arg2); 1759 tcg_gen_or_i64(ret, arg1, t0); 1760 tcg_temp_free_i64(t0); 1761 } 1762 } 1763 1764 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1765 { 1766 if (TCG_TARGET_HAS_clz_i64) { 1767 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2); 1768 } else { 1769 gen_helper_clz_i64(ret, arg1, arg2); 1770 } 1771 } 1772 1773 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1774 { 1775 if (TCG_TARGET_REG_BITS == 32 1776 && TCG_TARGET_HAS_clz_i32 1777 && arg2 <= 0xffffffffu) { 1778 TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32); 1779 tcg_gen_clz_i32(t, TCGV_LOW(arg1), t); 1780 tcg_gen_addi_i32(t, t, 32); 1781 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 1782 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1783 tcg_temp_free_i32(t); 1784 } else { 1785 TCGv_i64 t = tcg_const_i64(arg2); 1786 tcg_gen_clz_i64(ret, arg1, t); 1787 tcg_temp_free_i64(t); 1788 } 1789 } 1790 1791 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1792 { 1793 if (TCG_TARGET_HAS_ctz_i64) { 1794 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); 1795 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) { 1796 TCGv_i64 z, t = tcg_temp_new_i64(); 1797 1798 if (TCG_TARGET_HAS_ctpop_i64) { 1799 tcg_gen_subi_i64(t, arg1, 1); 1800 tcg_gen_andc_i64(t, t, arg1); 1801 tcg_gen_ctpop_i64(t, t); 1802 } else { 1803 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */ 1804 tcg_gen_neg_i64(t, arg1); 1805 tcg_gen_and_i64(t, t, arg1); 1806 tcg_gen_clzi_i64(t, t, 64); 1807 tcg_gen_xori_i64(t, t, 63); 1808 } 1809 z = tcg_const_i64(0); 1810 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 1811 tcg_temp_free_i64(t); 1812 tcg_temp_free_i64(z); 1813 } else { 1814 gen_helper_ctz_i64(ret, arg1, arg2); 1815 } 1816 } 1817 1818 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1819 { 1820 if (TCG_TARGET_REG_BITS == 32 1821 && TCG_TARGET_HAS_ctz_i32 1822 && arg2 <= 0xffffffffu) { 1823 TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32); 1824 tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32); 1825 tcg_gen_addi_i32(t32, t32, 32); 1826 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 1827 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1828 tcg_temp_free_i32(t32); 1829 } else if (!TCG_TARGET_HAS_ctz_i64 1830 && TCG_TARGET_HAS_ctpop_i64 1831 && arg2 == 64) { 1832 /* This equivalence has the advantage of not requiring a fixup. */ 1833 TCGv_i64 t = tcg_temp_new_i64(); 1834 tcg_gen_subi_i64(t, arg1, 1); 1835 tcg_gen_andc_i64(t, t, arg1); 1836 tcg_gen_ctpop_i64(ret, t); 1837 tcg_temp_free_i64(t); 1838 } else { 1839 TCGv_i64 t64 = tcg_const_i64(arg2); 1840 tcg_gen_ctz_i64(ret, arg1, t64); 1841 tcg_temp_free_i64(t64); 1842 } 1843 } 1844 1845 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 1846 { 1847 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) { 1848 TCGv_i64 t = tcg_temp_new_i64(); 1849 tcg_gen_sari_i64(t, arg, 63); 1850 tcg_gen_xor_i64(t, t, arg); 1851 tcg_gen_clzi_i64(t, t, 64); 1852 tcg_gen_subi_i64(ret, t, 1); 1853 tcg_temp_free_i64(t); 1854 } else { 1855 gen_helper_clrsb_i64(ret, arg); 1856 } 1857 } 1858 1859 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 1860 { 1861 if (TCG_TARGET_HAS_ctpop_i64) { 1862 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 1863 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 1864 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1865 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1866 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 1867 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1868 } else { 1869 gen_helper_ctpop_i64(ret, arg1); 1870 } 1871 } 1872 1873 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1874 { 1875 if (TCG_TARGET_HAS_rot_i64) { 1876 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2); 1877 } else { 1878 TCGv_i64 t0, t1; 1879 t0 = tcg_temp_new_i64(); 1880 t1 = tcg_temp_new_i64(); 1881 tcg_gen_shl_i64(t0, arg1, arg2); 1882 tcg_gen_subfi_i64(t1, 64, arg2); 1883 tcg_gen_shr_i64(t1, arg1, t1); 1884 tcg_gen_or_i64(ret, t0, t1); 1885 tcg_temp_free_i64(t0); 1886 tcg_temp_free_i64(t1); 1887 } 1888 } 1889 1890 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1891 { 1892 tcg_debug_assert(arg2 < 64); 1893 /* some cases can be optimized here */ 1894 if (arg2 == 0) { 1895 tcg_gen_mov_i64(ret, arg1); 1896 } else if (TCG_TARGET_HAS_rot_i64) { 1897 TCGv_i64 t0 = tcg_const_i64(arg2); 1898 tcg_gen_rotl_i64(ret, arg1, t0); 1899 tcg_temp_free_i64(t0); 1900 } else { 1901 TCGv_i64 t0, t1; 1902 t0 = tcg_temp_new_i64(); 1903 t1 = tcg_temp_new_i64(); 1904 tcg_gen_shli_i64(t0, arg1, arg2); 1905 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 1906 tcg_gen_or_i64(ret, t0, t1); 1907 tcg_temp_free_i64(t0); 1908 tcg_temp_free_i64(t1); 1909 } 1910 } 1911 1912 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1913 { 1914 if (TCG_TARGET_HAS_rot_i64) { 1915 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2); 1916 } else { 1917 TCGv_i64 t0, t1; 1918 t0 = tcg_temp_new_i64(); 1919 t1 = tcg_temp_new_i64(); 1920 tcg_gen_shr_i64(t0, arg1, arg2); 1921 tcg_gen_subfi_i64(t1, 64, arg2); 1922 tcg_gen_shl_i64(t1, arg1, t1); 1923 tcg_gen_or_i64(ret, t0, t1); 1924 tcg_temp_free_i64(t0); 1925 tcg_temp_free_i64(t1); 1926 } 1927 } 1928 1929 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1930 { 1931 tcg_debug_assert(arg2 < 64); 1932 /* some cases can be optimized here */ 1933 if (arg2 == 0) { 1934 tcg_gen_mov_i64(ret, arg1); 1935 } else { 1936 tcg_gen_rotli_i64(ret, arg1, 64 - arg2); 1937 } 1938 } 1939 1940 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 1941 unsigned int ofs, unsigned int len) 1942 { 1943 uint64_t mask; 1944 TCGv_i64 t1; 1945 1946 tcg_debug_assert(ofs < 64); 1947 tcg_debug_assert(len > 0); 1948 tcg_debug_assert(len <= 64); 1949 tcg_debug_assert(ofs + len <= 64); 1950 1951 if (len == 64) { 1952 tcg_gen_mov_i64(ret, arg2); 1953 return; 1954 } 1955 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 1956 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 1957 return; 1958 } 1959 1960 if (TCG_TARGET_REG_BITS == 32) { 1961 if (ofs >= 32) { 1962 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 1963 TCGV_LOW(arg2), ofs - 32, len); 1964 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1965 return; 1966 } 1967 if (ofs + len <= 32) { 1968 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 1969 TCGV_LOW(arg2), ofs, len); 1970 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1971 return; 1972 } 1973 } 1974 1975 mask = (1ull << len) - 1; 1976 t1 = tcg_temp_new_i64(); 1977 1978 if (ofs + len < 64) { 1979 tcg_gen_andi_i64(t1, arg2, mask); 1980 tcg_gen_shli_i64(t1, t1, ofs); 1981 } else { 1982 tcg_gen_shli_i64(t1, arg2, ofs); 1983 } 1984 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 1985 tcg_gen_or_i64(ret, ret, t1); 1986 1987 tcg_temp_free_i64(t1); 1988 } 1989 1990 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 1991 unsigned int ofs, unsigned int len) 1992 { 1993 tcg_debug_assert(ofs < 64); 1994 tcg_debug_assert(len > 0); 1995 tcg_debug_assert(len <= 64); 1996 tcg_debug_assert(ofs + len <= 64); 1997 1998 if (ofs + len == 64) { 1999 tcg_gen_shli_i64(ret, arg, ofs); 2000 } else if (ofs == 0) { 2001 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2002 } else if (TCG_TARGET_HAS_deposit_i64 2003 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2004 TCGv_i64 zero = tcg_const_i64(0); 2005 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2006 tcg_temp_free_i64(zero); 2007 } else { 2008 if (TCG_TARGET_REG_BITS == 32) { 2009 if (ofs >= 32) { 2010 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2011 ofs - 32, len); 2012 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2013 return; 2014 } 2015 if (ofs + len <= 32) { 2016 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2017 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2018 return; 2019 } 2020 } 2021 /* To help two-operand hosts we prefer to zero-extend first, 2022 which allows ARG to stay live. */ 2023 switch (len) { 2024 case 32: 2025 if (TCG_TARGET_HAS_ext32u_i64) { 2026 tcg_gen_ext32u_i64(ret, arg); 2027 tcg_gen_shli_i64(ret, ret, ofs); 2028 return; 2029 } 2030 break; 2031 case 16: 2032 if (TCG_TARGET_HAS_ext16u_i64) { 2033 tcg_gen_ext16u_i64(ret, arg); 2034 tcg_gen_shli_i64(ret, ret, ofs); 2035 return; 2036 } 2037 break; 2038 case 8: 2039 if (TCG_TARGET_HAS_ext8u_i64) { 2040 tcg_gen_ext8u_i64(ret, arg); 2041 tcg_gen_shli_i64(ret, ret, ofs); 2042 return; 2043 } 2044 break; 2045 } 2046 /* Otherwise prefer zero-extension over AND for code size. */ 2047 switch (ofs + len) { 2048 case 32: 2049 if (TCG_TARGET_HAS_ext32u_i64) { 2050 tcg_gen_shli_i64(ret, arg, ofs); 2051 tcg_gen_ext32u_i64(ret, ret); 2052 return; 2053 } 2054 break; 2055 case 16: 2056 if (TCG_TARGET_HAS_ext16u_i64) { 2057 tcg_gen_shli_i64(ret, arg, ofs); 2058 tcg_gen_ext16u_i64(ret, ret); 2059 return; 2060 } 2061 break; 2062 case 8: 2063 if (TCG_TARGET_HAS_ext8u_i64) { 2064 tcg_gen_shli_i64(ret, arg, ofs); 2065 tcg_gen_ext8u_i64(ret, ret); 2066 return; 2067 } 2068 break; 2069 } 2070 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2071 tcg_gen_shli_i64(ret, ret, ofs); 2072 } 2073 } 2074 2075 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2076 unsigned int ofs, unsigned int len) 2077 { 2078 tcg_debug_assert(ofs < 64); 2079 tcg_debug_assert(len > 0); 2080 tcg_debug_assert(len <= 64); 2081 tcg_debug_assert(ofs + len <= 64); 2082 2083 /* Canonicalize certain special cases, even if extract is supported. */ 2084 if (ofs + len == 64) { 2085 tcg_gen_shri_i64(ret, arg, 64 - len); 2086 return; 2087 } 2088 if (ofs == 0) { 2089 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2090 return; 2091 } 2092 2093 if (TCG_TARGET_REG_BITS == 32) { 2094 /* Look for a 32-bit extract within one of the two words. */ 2095 if (ofs >= 32) { 2096 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2097 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2098 return; 2099 } 2100 if (ofs + len <= 32) { 2101 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2102 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2103 return; 2104 } 2105 /* The field is split across two words. One double-word 2106 shift is better than two double-word shifts. */ 2107 goto do_shift_and; 2108 } 2109 2110 if (TCG_TARGET_HAS_extract_i64 2111 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2112 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2113 return; 2114 } 2115 2116 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2117 switch (ofs + len) { 2118 case 32: 2119 if (TCG_TARGET_HAS_ext32u_i64) { 2120 tcg_gen_ext32u_i64(ret, arg); 2121 tcg_gen_shri_i64(ret, ret, ofs); 2122 return; 2123 } 2124 break; 2125 case 16: 2126 if (TCG_TARGET_HAS_ext16u_i64) { 2127 tcg_gen_ext16u_i64(ret, arg); 2128 tcg_gen_shri_i64(ret, ret, ofs); 2129 return; 2130 } 2131 break; 2132 case 8: 2133 if (TCG_TARGET_HAS_ext8u_i64) { 2134 tcg_gen_ext8u_i64(ret, arg); 2135 tcg_gen_shri_i64(ret, ret, ofs); 2136 return; 2137 } 2138 break; 2139 } 2140 2141 /* ??? Ideally we'd know what values are available for immediate AND. 2142 Assume that 8 bits are available, plus the special cases of 16 and 32, 2143 so that we get ext8u, ext16u, and ext32u. */ 2144 switch (len) { 2145 case 1 ... 8: case 16: case 32: 2146 do_shift_and: 2147 tcg_gen_shri_i64(ret, arg, ofs); 2148 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2149 break; 2150 default: 2151 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2152 tcg_gen_shri_i64(ret, ret, 64 - len); 2153 break; 2154 } 2155 } 2156 2157 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2158 unsigned int ofs, unsigned int len) 2159 { 2160 tcg_debug_assert(ofs < 64); 2161 tcg_debug_assert(len > 0); 2162 tcg_debug_assert(len <= 64); 2163 tcg_debug_assert(ofs + len <= 64); 2164 2165 /* Canonicalize certain special cases, even if sextract is supported. */ 2166 if (ofs + len == 64) { 2167 tcg_gen_sari_i64(ret, arg, 64 - len); 2168 return; 2169 } 2170 if (ofs == 0) { 2171 switch (len) { 2172 case 32: 2173 tcg_gen_ext32s_i64(ret, arg); 2174 return; 2175 case 16: 2176 tcg_gen_ext16s_i64(ret, arg); 2177 return; 2178 case 8: 2179 tcg_gen_ext8s_i64(ret, arg); 2180 return; 2181 } 2182 } 2183 2184 if (TCG_TARGET_REG_BITS == 32) { 2185 /* Look for a 32-bit extract within one of the two words. */ 2186 if (ofs >= 32) { 2187 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2188 } else if (ofs + len <= 32) { 2189 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2190 } else if (ofs == 0) { 2191 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2192 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2193 return; 2194 } else if (len > 32) { 2195 TCGv_i32 t = tcg_temp_new_i32(); 2196 /* Extract the bits for the high word normally. */ 2197 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2198 /* Shift the field down for the low part. */ 2199 tcg_gen_shri_i64(ret, arg, ofs); 2200 /* Overwrite the shift into the high part. */ 2201 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2202 tcg_temp_free_i32(t); 2203 return; 2204 } else { 2205 /* Shift the field down for the low part, such that the 2206 field sits at the MSB. */ 2207 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2208 /* Shift the field down from the MSB, sign extending. */ 2209 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2210 } 2211 /* Sign-extend the field from 32 bits. */ 2212 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2213 return; 2214 } 2215 2216 if (TCG_TARGET_HAS_sextract_i64 2217 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2218 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2219 return; 2220 } 2221 2222 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2223 switch (ofs + len) { 2224 case 32: 2225 if (TCG_TARGET_HAS_ext32s_i64) { 2226 tcg_gen_ext32s_i64(ret, arg); 2227 tcg_gen_sari_i64(ret, ret, ofs); 2228 return; 2229 } 2230 break; 2231 case 16: 2232 if (TCG_TARGET_HAS_ext16s_i64) { 2233 tcg_gen_ext16s_i64(ret, arg); 2234 tcg_gen_sari_i64(ret, ret, ofs); 2235 return; 2236 } 2237 break; 2238 case 8: 2239 if (TCG_TARGET_HAS_ext8s_i64) { 2240 tcg_gen_ext8s_i64(ret, arg); 2241 tcg_gen_sari_i64(ret, ret, ofs); 2242 return; 2243 } 2244 break; 2245 } 2246 switch (len) { 2247 case 32: 2248 if (TCG_TARGET_HAS_ext32s_i64) { 2249 tcg_gen_shri_i64(ret, arg, ofs); 2250 tcg_gen_ext32s_i64(ret, ret); 2251 return; 2252 } 2253 break; 2254 case 16: 2255 if (TCG_TARGET_HAS_ext16s_i64) { 2256 tcg_gen_shri_i64(ret, arg, ofs); 2257 tcg_gen_ext16s_i64(ret, ret); 2258 return; 2259 } 2260 break; 2261 case 8: 2262 if (TCG_TARGET_HAS_ext8s_i64) { 2263 tcg_gen_shri_i64(ret, arg, ofs); 2264 tcg_gen_ext8s_i64(ret, ret); 2265 return; 2266 } 2267 break; 2268 } 2269 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2270 tcg_gen_sari_i64(ret, ret, 64 - len); 2271 } 2272 2273 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2274 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2275 { 2276 if (cond == TCG_COND_ALWAYS) { 2277 tcg_gen_mov_i64(ret, v1); 2278 } else if (cond == TCG_COND_NEVER) { 2279 tcg_gen_mov_i64(ret, v2); 2280 } else if (TCG_TARGET_REG_BITS == 32) { 2281 TCGv_i32 t0 = tcg_temp_new_i32(); 2282 TCGv_i32 t1 = tcg_temp_new_i32(); 2283 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2284 TCGV_LOW(c1), TCGV_HIGH(c1), 2285 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2286 2287 if (TCG_TARGET_HAS_movcond_i32) { 2288 tcg_gen_movi_i32(t1, 0); 2289 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1, 2290 TCGV_LOW(v1), TCGV_LOW(v2)); 2291 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1, 2292 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2293 } else { 2294 tcg_gen_neg_i32(t0, t0); 2295 2296 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0); 2297 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0); 2298 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1); 2299 2300 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0); 2301 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0); 2302 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1); 2303 } 2304 tcg_temp_free_i32(t0); 2305 tcg_temp_free_i32(t1); 2306 } else if (TCG_TARGET_HAS_movcond_i64) { 2307 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2308 } else { 2309 TCGv_i64 t0 = tcg_temp_new_i64(); 2310 TCGv_i64 t1 = tcg_temp_new_i64(); 2311 tcg_gen_setcond_i64(cond, t0, c1, c2); 2312 tcg_gen_neg_i64(t0, t0); 2313 tcg_gen_and_i64(t1, v1, t0); 2314 tcg_gen_andc_i64(ret, v2, t0); 2315 tcg_gen_or_i64(ret, ret, t1); 2316 tcg_temp_free_i64(t0); 2317 tcg_temp_free_i64(t1); 2318 } 2319 } 2320 2321 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2322 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2323 { 2324 if (TCG_TARGET_HAS_add2_i64) { 2325 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2326 } else { 2327 TCGv_i64 t0 = tcg_temp_new_i64(); 2328 TCGv_i64 t1 = tcg_temp_new_i64(); 2329 tcg_gen_add_i64(t0, al, bl); 2330 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2331 tcg_gen_add_i64(rh, ah, bh); 2332 tcg_gen_add_i64(rh, rh, t1); 2333 tcg_gen_mov_i64(rl, t0); 2334 tcg_temp_free_i64(t0); 2335 tcg_temp_free_i64(t1); 2336 } 2337 } 2338 2339 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2340 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2341 { 2342 if (TCG_TARGET_HAS_sub2_i64) { 2343 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2344 } else { 2345 TCGv_i64 t0 = tcg_temp_new_i64(); 2346 TCGv_i64 t1 = tcg_temp_new_i64(); 2347 tcg_gen_sub_i64(t0, al, bl); 2348 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2349 tcg_gen_sub_i64(rh, ah, bh); 2350 tcg_gen_sub_i64(rh, rh, t1); 2351 tcg_gen_mov_i64(rl, t0); 2352 tcg_temp_free_i64(t0); 2353 tcg_temp_free_i64(t1); 2354 } 2355 } 2356 2357 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2358 { 2359 if (TCG_TARGET_HAS_mulu2_i64) { 2360 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2361 } else if (TCG_TARGET_HAS_muluh_i64) { 2362 TCGv_i64 t = tcg_temp_new_i64(); 2363 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2364 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2); 2365 tcg_gen_mov_i64(rl, t); 2366 tcg_temp_free_i64(t); 2367 } else { 2368 TCGv_i64 t0 = tcg_temp_new_i64(); 2369 tcg_gen_mul_i64(t0, arg1, arg2); 2370 gen_helper_muluh_i64(rh, arg1, arg2); 2371 tcg_gen_mov_i64(rl, t0); 2372 tcg_temp_free_i64(t0); 2373 } 2374 } 2375 2376 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2377 { 2378 if (TCG_TARGET_HAS_muls2_i64) { 2379 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 2380 } else if (TCG_TARGET_HAS_mulsh_i64) { 2381 TCGv_i64 t = tcg_temp_new_i64(); 2382 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2383 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2); 2384 tcg_gen_mov_i64(rl, t); 2385 tcg_temp_free_i64(t); 2386 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) { 2387 TCGv_i64 t0 = tcg_temp_new_i64(); 2388 TCGv_i64 t1 = tcg_temp_new_i64(); 2389 TCGv_i64 t2 = tcg_temp_new_i64(); 2390 TCGv_i64 t3 = tcg_temp_new_i64(); 2391 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2392 /* Adjust for negative inputs. */ 2393 tcg_gen_sari_i64(t2, arg1, 63); 2394 tcg_gen_sari_i64(t3, arg2, 63); 2395 tcg_gen_and_i64(t2, t2, arg2); 2396 tcg_gen_and_i64(t3, t3, arg1); 2397 tcg_gen_sub_i64(rh, t1, t2); 2398 tcg_gen_sub_i64(rh, rh, t3); 2399 tcg_gen_mov_i64(rl, t0); 2400 tcg_temp_free_i64(t0); 2401 tcg_temp_free_i64(t1); 2402 tcg_temp_free_i64(t2); 2403 tcg_temp_free_i64(t3); 2404 } else { 2405 TCGv_i64 t0 = tcg_temp_new_i64(); 2406 tcg_gen_mul_i64(t0, arg1, arg2); 2407 gen_helper_mulsh_i64(rh, arg1, arg2); 2408 tcg_gen_mov_i64(rl, t0); 2409 tcg_temp_free_i64(t0); 2410 } 2411 } 2412 2413 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2414 { 2415 TCGv_i64 t0 = tcg_temp_new_i64(); 2416 TCGv_i64 t1 = tcg_temp_new_i64(); 2417 TCGv_i64 t2 = tcg_temp_new_i64(); 2418 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2419 /* Adjust for negative input for the signed arg1. */ 2420 tcg_gen_sari_i64(t2, arg1, 63); 2421 tcg_gen_and_i64(t2, t2, arg2); 2422 tcg_gen_sub_i64(rh, t1, t2); 2423 tcg_gen_mov_i64(rl, t0); 2424 tcg_temp_free_i64(t0); 2425 tcg_temp_free_i64(t1); 2426 tcg_temp_free_i64(t2); 2427 } 2428 2429 /* Size changing operations. */ 2430 2431 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2432 { 2433 if (TCG_TARGET_REG_BITS == 32) { 2434 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2435 } else if (TCG_TARGET_HAS_extrl_i64_i32) { 2436 tcg_gen_op2(INDEX_op_extrl_i64_i32, 2437 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2438 } else { 2439 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2440 } 2441 } 2442 2443 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2444 { 2445 if (TCG_TARGET_REG_BITS == 32) { 2446 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2447 } else if (TCG_TARGET_HAS_extrh_i64_i32) { 2448 tcg_gen_op2(INDEX_op_extrh_i64_i32, 2449 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2450 } else { 2451 TCGv_i64 t = tcg_temp_new_i64(); 2452 tcg_gen_shri_i64(t, arg, 32); 2453 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2454 tcg_temp_free_i64(t); 2455 } 2456 } 2457 2458 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2459 { 2460 if (TCG_TARGET_REG_BITS == 32) { 2461 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2462 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2463 } else { 2464 tcg_gen_op2(INDEX_op_extu_i32_i64, 2465 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2466 } 2467 } 2468 2469 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2470 { 2471 if (TCG_TARGET_REG_BITS == 32) { 2472 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2473 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2474 } else { 2475 tcg_gen_op2(INDEX_op_ext_i32_i64, 2476 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2477 } 2478 } 2479 2480 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 2481 { 2482 TCGv_i64 tmp; 2483 2484 if (TCG_TARGET_REG_BITS == 32) { 2485 tcg_gen_mov_i32(TCGV_LOW(dest), low); 2486 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 2487 return; 2488 } 2489 2490 tmp = tcg_temp_new_i64(); 2491 /* These extensions are only needed for type correctness. 2492 We may be able to do better given target specific information. */ 2493 tcg_gen_extu_i32_i64(tmp, high); 2494 tcg_gen_extu_i32_i64(dest, low); 2495 /* If deposit is available, use it. Otherwise use the extra 2496 knowledge that we have of the zero-extensions above. */ 2497 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) { 2498 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 2499 } else { 2500 tcg_gen_shli_i64(tmp, tmp, 32); 2501 tcg_gen_or_i64(dest, dest, tmp); 2502 } 2503 tcg_temp_free_i64(tmp); 2504 } 2505 2506 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 2507 { 2508 if (TCG_TARGET_REG_BITS == 32) { 2509 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 2510 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 2511 } else { 2512 tcg_gen_extrl_i64_i32(lo, arg); 2513 tcg_gen_extrh_i64_i32(hi, arg); 2514 } 2515 } 2516 2517 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 2518 { 2519 tcg_gen_ext32u_i64(lo, arg); 2520 tcg_gen_shri_i64(hi, arg, 32); 2521 } 2522 2523 /* QEMU specific operations. */ 2524 2525 void tcg_gen_goto_tb(unsigned idx) 2526 { 2527 /* We only support two chained exits. */ 2528 tcg_debug_assert(idx <= 1); 2529 #ifdef CONFIG_DEBUG_TCG 2530 /* Verify that we havn't seen this numbered exit before. */ 2531 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 2532 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 2533 #endif 2534 tcg_gen_op1i(INDEX_op_goto_tb, idx); 2535 } 2536 2537 void tcg_gen_lookup_and_goto_ptr(void) 2538 { 2539 if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2540 TCGv_ptr ptr = tcg_temp_new_ptr(); 2541 gen_helper_lookup_tb_ptr(ptr, cpu_env); 2542 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr)); 2543 tcg_temp_free_ptr(ptr); 2544 } else { 2545 tcg_gen_exit_tb(0); 2546 } 2547 } 2548 2549 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st) 2550 { 2551 /* Trigger the asserts within as early as possible. */ 2552 (void)get_alignment_bits(op); 2553 2554 switch (op & MO_SIZE) { 2555 case MO_8: 2556 op &= ~MO_BSWAP; 2557 break; 2558 case MO_16: 2559 break; 2560 case MO_32: 2561 if (!is64) { 2562 op &= ~MO_SIGN; 2563 } 2564 break; 2565 case MO_64: 2566 if (!is64) { 2567 tcg_abort(); 2568 } 2569 break; 2570 } 2571 if (st) { 2572 op &= ~MO_SIGN; 2573 } 2574 return op; 2575 } 2576 2577 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr, 2578 TCGMemOp memop, TCGArg idx) 2579 { 2580 TCGMemOpIdx oi = make_memop_idx(memop, idx); 2581 #if TARGET_LONG_BITS == 32 2582 tcg_gen_op3i_i32(opc, val, addr, oi); 2583 #else 2584 if (TCG_TARGET_REG_BITS == 32) { 2585 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2586 } else { 2587 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi); 2588 } 2589 #endif 2590 } 2591 2592 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr, 2593 TCGMemOp memop, TCGArg idx) 2594 { 2595 TCGMemOpIdx oi = make_memop_idx(memop, idx); 2596 #if TARGET_LONG_BITS == 32 2597 if (TCG_TARGET_REG_BITS == 32) { 2598 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi); 2599 } else { 2600 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi); 2601 } 2602 #else 2603 if (TCG_TARGET_REG_BITS == 32) { 2604 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), 2605 TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2606 } else { 2607 tcg_gen_op3i_i64(opc, val, addr, oi); 2608 } 2609 #endif 2610 } 2611 2612 static void tcg_gen_req_mo(TCGBar type) 2613 { 2614 #ifdef TCG_GUEST_DEFAULT_MO 2615 type &= TCG_GUEST_DEFAULT_MO; 2616 #endif 2617 type &= ~TCG_TARGET_DEFAULT_MO; 2618 if (type) { 2619 tcg_gen_mb(type | TCG_BAR_SC); 2620 } 2621 } 2622 2623 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2624 { 2625 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2626 memop = tcg_canonicalize_memop(memop, 0, 0); 2627 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2628 addr, trace_mem_get_info(memop, 0)); 2629 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx); 2630 } 2631 2632 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2633 { 2634 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2635 memop = tcg_canonicalize_memop(memop, 0, 1); 2636 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2637 addr, trace_mem_get_info(memop, 1)); 2638 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); 2639 } 2640 2641 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2642 { 2643 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2644 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 2645 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); 2646 if (memop & MO_SIGN) { 2647 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 2648 } else { 2649 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 2650 } 2651 return; 2652 } 2653 2654 memop = tcg_canonicalize_memop(memop, 1, 0); 2655 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2656 addr, trace_mem_get_info(memop, 0)); 2657 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx); 2658 } 2659 2660 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2661 { 2662 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2663 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 2664 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); 2665 return; 2666 } 2667 2668 memop = tcg_canonicalize_memop(memop, 1, 1); 2669 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2670 addr, trace_mem_get_info(memop, 1)); 2671 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx); 2672 } 2673 2674 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc) 2675 { 2676 switch (opc & MO_SSIZE) { 2677 case MO_SB: 2678 tcg_gen_ext8s_i32(ret, val); 2679 break; 2680 case MO_UB: 2681 tcg_gen_ext8u_i32(ret, val); 2682 break; 2683 case MO_SW: 2684 tcg_gen_ext16s_i32(ret, val); 2685 break; 2686 case MO_UW: 2687 tcg_gen_ext16u_i32(ret, val); 2688 break; 2689 default: 2690 tcg_gen_mov_i32(ret, val); 2691 break; 2692 } 2693 } 2694 2695 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc) 2696 { 2697 switch (opc & MO_SSIZE) { 2698 case MO_SB: 2699 tcg_gen_ext8s_i64(ret, val); 2700 break; 2701 case MO_UB: 2702 tcg_gen_ext8u_i64(ret, val); 2703 break; 2704 case MO_SW: 2705 tcg_gen_ext16s_i64(ret, val); 2706 break; 2707 case MO_UW: 2708 tcg_gen_ext16u_i64(ret, val); 2709 break; 2710 case MO_SL: 2711 tcg_gen_ext32s_i64(ret, val); 2712 break; 2713 case MO_UL: 2714 tcg_gen_ext32u_i64(ret, val); 2715 break; 2716 default: 2717 tcg_gen_mov_i64(ret, val); 2718 break; 2719 } 2720 } 2721 2722 #ifdef CONFIG_SOFTMMU 2723 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, 2724 TCGv_i32, TCGv_i32, TCGv_i32); 2725 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, 2726 TCGv_i64, TCGv_i64, TCGv_i32); 2727 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, 2728 TCGv_i32, TCGv_i32); 2729 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, 2730 TCGv_i64, TCGv_i32); 2731 #else 2732 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32); 2733 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64); 2734 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32); 2735 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64); 2736 #endif 2737 2738 #ifdef CONFIG_ATOMIC64 2739 # define WITH_ATOMIC64(X) X, 2740 #else 2741 # define WITH_ATOMIC64(X) 2742 #endif 2743 2744 static void * const table_cmpxchg[16] = { 2745 [MO_8] = gen_helper_atomic_cmpxchgb, 2746 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, 2747 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, 2748 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, 2749 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, 2750 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) 2751 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) 2752 }; 2753 2754 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, 2755 TCGv_i32 newv, TCGArg idx, TCGMemOp memop) 2756 { 2757 memop = tcg_canonicalize_memop(memop, 0, 0); 2758 2759 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { 2760 TCGv_i32 t1 = tcg_temp_new_i32(); 2761 TCGv_i32 t2 = tcg_temp_new_i32(); 2762 2763 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); 2764 2765 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 2766 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); 2767 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 2768 tcg_temp_free_i32(t2); 2769 2770 if (memop & MO_SIGN) { 2771 tcg_gen_ext_i32(retv, t1, memop); 2772 } else { 2773 tcg_gen_mov_i32(retv, t1); 2774 } 2775 tcg_temp_free_i32(t1); 2776 } else { 2777 gen_atomic_cx_i32 gen; 2778 2779 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 2780 tcg_debug_assert(gen != NULL); 2781 2782 #ifdef CONFIG_SOFTMMU 2783 { 2784 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 2785 gen(retv, cpu_env, addr, cmpv, newv, oi); 2786 tcg_temp_free_i32(oi); 2787 } 2788 #else 2789 gen(retv, cpu_env, addr, cmpv, newv); 2790 #endif 2791 2792 if (memop & MO_SIGN) { 2793 tcg_gen_ext_i32(retv, retv, memop); 2794 } 2795 } 2796 } 2797 2798 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, 2799 TCGv_i64 newv, TCGArg idx, TCGMemOp memop) 2800 { 2801 memop = tcg_canonicalize_memop(memop, 1, 0); 2802 2803 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { 2804 TCGv_i64 t1 = tcg_temp_new_i64(); 2805 TCGv_i64 t2 = tcg_temp_new_i64(); 2806 2807 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); 2808 2809 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 2810 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); 2811 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 2812 tcg_temp_free_i64(t2); 2813 2814 if (memop & MO_SIGN) { 2815 tcg_gen_ext_i64(retv, t1, memop); 2816 } else { 2817 tcg_gen_mov_i64(retv, t1); 2818 } 2819 tcg_temp_free_i64(t1); 2820 } else if ((memop & MO_SIZE) == MO_64) { 2821 #ifdef CONFIG_ATOMIC64 2822 gen_atomic_cx_i64 gen; 2823 2824 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 2825 tcg_debug_assert(gen != NULL); 2826 2827 #ifdef CONFIG_SOFTMMU 2828 { 2829 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx)); 2830 gen(retv, cpu_env, addr, cmpv, newv, oi); 2831 tcg_temp_free_i32(oi); 2832 } 2833 #else 2834 gen(retv, cpu_env, addr, cmpv, newv); 2835 #endif 2836 #else 2837 gen_helper_exit_atomic(cpu_env); 2838 /* Produce a result, so that we have a well-formed opcode stream 2839 with respect to uses of the result in the (dead) code following. */ 2840 tcg_gen_movi_i64(retv, 0); 2841 #endif /* CONFIG_ATOMIC64 */ 2842 } else { 2843 TCGv_i32 c32 = tcg_temp_new_i32(); 2844 TCGv_i32 n32 = tcg_temp_new_i32(); 2845 TCGv_i32 r32 = tcg_temp_new_i32(); 2846 2847 tcg_gen_extrl_i64_i32(c32, cmpv); 2848 tcg_gen_extrl_i64_i32(n32, newv); 2849 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN); 2850 tcg_temp_free_i32(c32); 2851 tcg_temp_free_i32(n32); 2852 2853 tcg_gen_extu_i32_i64(retv, r32); 2854 tcg_temp_free_i32(r32); 2855 2856 if (memop & MO_SIGN) { 2857 tcg_gen_ext_i64(retv, retv, memop); 2858 } 2859 } 2860 } 2861 2862 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 2863 TCGArg idx, TCGMemOp memop, bool new_val, 2864 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) 2865 { 2866 TCGv_i32 t1 = tcg_temp_new_i32(); 2867 TCGv_i32 t2 = tcg_temp_new_i32(); 2868 2869 memop = tcg_canonicalize_memop(memop, 0, 0); 2870 2871 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 2872 gen(t2, t1, val); 2873 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 2874 2875 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); 2876 tcg_temp_free_i32(t1); 2877 tcg_temp_free_i32(t2); 2878 } 2879 2880 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 2881 TCGArg idx, TCGMemOp memop, void * const table[]) 2882 { 2883 gen_atomic_op_i32 gen; 2884 2885 memop = tcg_canonicalize_memop(memop, 0, 0); 2886 2887 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 2888 tcg_debug_assert(gen != NULL); 2889 2890 #ifdef CONFIG_SOFTMMU 2891 { 2892 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 2893 gen(ret, cpu_env, addr, val, oi); 2894 tcg_temp_free_i32(oi); 2895 } 2896 #else 2897 gen(ret, cpu_env, addr, val); 2898 #endif 2899 2900 if (memop & MO_SIGN) { 2901 tcg_gen_ext_i32(ret, ret, memop); 2902 } 2903 } 2904 2905 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 2906 TCGArg idx, TCGMemOp memop, bool new_val, 2907 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) 2908 { 2909 TCGv_i64 t1 = tcg_temp_new_i64(); 2910 TCGv_i64 t2 = tcg_temp_new_i64(); 2911 2912 memop = tcg_canonicalize_memop(memop, 1, 0); 2913 2914 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 2915 gen(t2, t1, val); 2916 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 2917 2918 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); 2919 tcg_temp_free_i64(t1); 2920 tcg_temp_free_i64(t2); 2921 } 2922 2923 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 2924 TCGArg idx, TCGMemOp memop, void * const table[]) 2925 { 2926 memop = tcg_canonicalize_memop(memop, 1, 0); 2927 2928 if ((memop & MO_SIZE) == MO_64) { 2929 #ifdef CONFIG_ATOMIC64 2930 gen_atomic_op_i64 gen; 2931 2932 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 2933 tcg_debug_assert(gen != NULL); 2934 2935 #ifdef CONFIG_SOFTMMU 2936 { 2937 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 2938 gen(ret, cpu_env, addr, val, oi); 2939 tcg_temp_free_i32(oi); 2940 } 2941 #else 2942 gen(ret, cpu_env, addr, val); 2943 #endif 2944 #else 2945 gen_helper_exit_atomic(cpu_env); 2946 /* Produce a result, so that we have a well-formed opcode stream 2947 with respect to uses of the result in the (dead) code following. */ 2948 tcg_gen_movi_i64(ret, 0); 2949 #endif /* CONFIG_ATOMIC64 */ 2950 } else { 2951 TCGv_i32 v32 = tcg_temp_new_i32(); 2952 TCGv_i32 r32 = tcg_temp_new_i32(); 2953 2954 tcg_gen_extrl_i64_i32(v32, val); 2955 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); 2956 tcg_temp_free_i32(v32); 2957 2958 tcg_gen_extu_i32_i64(ret, r32); 2959 tcg_temp_free_i32(r32); 2960 2961 if (memop & MO_SIGN) { 2962 tcg_gen_ext_i64(ret, ret, memop); 2963 } 2964 } 2965 } 2966 2967 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ 2968 static void * const table_##NAME[16] = { \ 2969 [MO_8] = gen_helper_atomic_##NAME##b, \ 2970 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ 2971 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ 2972 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ 2973 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ 2974 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ 2975 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ 2976 }; \ 2977 void tcg_gen_atomic_##NAME##_i32 \ 2978 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \ 2979 { \ 2980 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ 2981 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ 2982 } else { \ 2983 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ 2984 tcg_gen_##OP##_i32); \ 2985 } \ 2986 } \ 2987 void tcg_gen_atomic_##NAME##_i64 \ 2988 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \ 2989 { \ 2990 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ 2991 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ 2992 } else { \ 2993 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ 2994 tcg_gen_##OP##_i64); \ 2995 } \ 2996 } 2997 2998 GEN_ATOMIC_HELPER(fetch_add, add, 0) 2999 GEN_ATOMIC_HELPER(fetch_and, and, 0) 3000 GEN_ATOMIC_HELPER(fetch_or, or, 0) 3001 GEN_ATOMIC_HELPER(fetch_xor, xor, 0) 3002 3003 GEN_ATOMIC_HELPER(add_fetch, add, 1) 3004 GEN_ATOMIC_HELPER(and_fetch, and, 1) 3005 GEN_ATOMIC_HELPER(or_fetch, or, 1) 3006 GEN_ATOMIC_HELPER(xor_fetch, xor, 1) 3007 3008 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) 3009 { 3010 tcg_gen_mov_i32(r, b); 3011 } 3012 3013 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) 3014 { 3015 tcg_gen_mov_i64(r, b); 3016 } 3017 3018 GEN_ATOMIC_HELPER(xchg, mov2, 0) 3019 3020 #undef GEN_ATOMIC_HELPER 3021