1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "qemu-common.h" 27 #include "cpu.h" 28 #include "exec/exec-all.h" 29 #include "tcg.h" 30 #include "tcg-op.h" 31 #include "tcg-mo.h" 32 #include "trace-tcg.h" 33 #include "trace/mem.h" 34 35 /* Reduce the number of ifdefs below. This assumes that all uses of 36 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that 37 the compiler can eliminate. */ 38 #if TCG_TARGET_REG_BITS == 64 39 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); 40 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); 41 #define TCGV_LOW TCGV_LOW_link_error 42 #define TCGV_HIGH TCGV_HIGH_link_error 43 #endif 44 45 void tcg_gen_op1(TCGOpcode opc, TCGArg a1) 46 { 47 TCGOp *op = tcg_emit_op(opc); 48 op->args[0] = a1; 49 } 50 51 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2) 52 { 53 TCGOp *op = tcg_emit_op(opc); 54 op->args[0] = a1; 55 op->args[1] = a2; 56 } 57 58 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) 59 { 60 TCGOp *op = tcg_emit_op(opc); 61 op->args[0] = a1; 62 op->args[1] = a2; 63 op->args[2] = a3; 64 } 65 66 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4) 67 { 68 TCGOp *op = tcg_emit_op(opc); 69 op->args[0] = a1; 70 op->args[1] = a2; 71 op->args[2] = a3; 72 op->args[3] = a4; 73 } 74 75 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 76 TCGArg a4, TCGArg a5) 77 { 78 TCGOp *op = tcg_emit_op(opc); 79 op->args[0] = a1; 80 op->args[1] = a2; 81 op->args[2] = a3; 82 op->args[3] = a4; 83 op->args[4] = a5; 84 } 85 86 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 87 TCGArg a4, TCGArg a5, TCGArg a6) 88 { 89 TCGOp *op = tcg_emit_op(opc); 90 op->args[0] = a1; 91 op->args[1] = a2; 92 op->args[2] = a3; 93 op->args[3] = a4; 94 op->args[4] = a5; 95 op->args[5] = a6; 96 } 97 98 void tcg_gen_mb(TCGBar mb_type) 99 { 100 if (tcg_ctx->tb_cflags & CF_PARALLEL) { 101 tcg_gen_op1(INDEX_op_mb, mb_type); 102 } 103 } 104 105 /* 32 bit ops */ 106 107 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 108 { 109 /* some cases can be optimized here */ 110 if (arg2 == 0) { 111 tcg_gen_mov_i32(ret, arg1); 112 } else { 113 TCGv_i32 t0 = tcg_const_i32(arg2); 114 tcg_gen_add_i32(ret, arg1, t0); 115 tcg_temp_free_i32(t0); 116 } 117 } 118 119 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 120 { 121 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) { 122 /* Don't recurse with tcg_gen_neg_i32. */ 123 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2); 124 } else { 125 TCGv_i32 t0 = tcg_const_i32(arg1); 126 tcg_gen_sub_i32(ret, t0, arg2); 127 tcg_temp_free_i32(t0); 128 } 129 } 130 131 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 132 { 133 /* some cases can be optimized here */ 134 if (arg2 == 0) { 135 tcg_gen_mov_i32(ret, arg1); 136 } else { 137 TCGv_i32 t0 = tcg_const_i32(arg2); 138 tcg_gen_sub_i32(ret, arg1, t0); 139 tcg_temp_free_i32(t0); 140 } 141 } 142 143 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 144 { 145 TCGv_i32 t0; 146 /* Some cases can be optimized here. */ 147 switch (arg2) { 148 case 0: 149 tcg_gen_movi_i32(ret, 0); 150 return; 151 case -1: 152 tcg_gen_mov_i32(ret, arg1); 153 return; 154 case 0xff: 155 /* Don't recurse with tcg_gen_ext8u_i32. */ 156 if (TCG_TARGET_HAS_ext8u_i32) { 157 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1); 158 return; 159 } 160 break; 161 case 0xffff: 162 if (TCG_TARGET_HAS_ext16u_i32) { 163 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1); 164 return; 165 } 166 break; 167 } 168 t0 = tcg_const_i32(arg2); 169 tcg_gen_and_i32(ret, arg1, t0); 170 tcg_temp_free_i32(t0); 171 } 172 173 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 174 { 175 /* Some cases can be optimized here. */ 176 if (arg2 == -1) { 177 tcg_gen_movi_i32(ret, -1); 178 } else if (arg2 == 0) { 179 tcg_gen_mov_i32(ret, arg1); 180 } else { 181 TCGv_i32 t0 = tcg_const_i32(arg2); 182 tcg_gen_or_i32(ret, arg1, t0); 183 tcg_temp_free_i32(t0); 184 } 185 } 186 187 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 188 { 189 /* Some cases can be optimized here. */ 190 if (arg2 == 0) { 191 tcg_gen_mov_i32(ret, arg1); 192 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) { 193 /* Don't recurse with tcg_gen_not_i32. */ 194 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1); 195 } else { 196 TCGv_i32 t0 = tcg_const_i32(arg2); 197 tcg_gen_xor_i32(ret, arg1, t0); 198 tcg_temp_free_i32(t0); 199 } 200 } 201 202 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 203 { 204 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 205 if (arg2 == 0) { 206 tcg_gen_mov_i32(ret, arg1); 207 } else { 208 TCGv_i32 t0 = tcg_const_i32(arg2); 209 tcg_gen_shl_i32(ret, arg1, t0); 210 tcg_temp_free_i32(t0); 211 } 212 } 213 214 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 215 { 216 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 217 if (arg2 == 0) { 218 tcg_gen_mov_i32(ret, arg1); 219 } else { 220 TCGv_i32 t0 = tcg_const_i32(arg2); 221 tcg_gen_shr_i32(ret, arg1, t0); 222 tcg_temp_free_i32(t0); 223 } 224 } 225 226 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 227 { 228 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 229 if (arg2 == 0) { 230 tcg_gen_mov_i32(ret, arg1); 231 } else { 232 TCGv_i32 t0 = tcg_const_i32(arg2); 233 tcg_gen_sar_i32(ret, arg1, t0); 234 tcg_temp_free_i32(t0); 235 } 236 } 237 238 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 239 { 240 if (cond == TCG_COND_ALWAYS) { 241 tcg_gen_br(l); 242 } else if (cond != TCG_COND_NEVER) { 243 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l)); 244 } 245 } 246 247 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 248 { 249 if (cond == TCG_COND_ALWAYS) { 250 tcg_gen_br(l); 251 } else if (cond != TCG_COND_NEVER) { 252 TCGv_i32 t0 = tcg_const_i32(arg2); 253 tcg_gen_brcond_i32(cond, arg1, t0, l); 254 tcg_temp_free_i32(t0); 255 } 256 } 257 258 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 259 TCGv_i32 arg1, TCGv_i32 arg2) 260 { 261 if (cond == TCG_COND_ALWAYS) { 262 tcg_gen_movi_i32(ret, 1); 263 } else if (cond == TCG_COND_NEVER) { 264 tcg_gen_movi_i32(ret, 0); 265 } else { 266 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 267 } 268 } 269 270 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 271 TCGv_i32 arg1, int32_t arg2) 272 { 273 TCGv_i32 t0 = tcg_const_i32(arg2); 274 tcg_gen_setcond_i32(cond, ret, arg1, t0); 275 tcg_temp_free_i32(t0); 276 } 277 278 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 279 { 280 if (arg2 == 0) { 281 tcg_gen_movi_i32(ret, 0); 282 } else if (is_power_of_2(arg2)) { 283 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 284 } else { 285 TCGv_i32 t0 = tcg_const_i32(arg2); 286 tcg_gen_mul_i32(ret, arg1, t0); 287 tcg_temp_free_i32(t0); 288 } 289 } 290 291 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 292 { 293 if (TCG_TARGET_HAS_div_i32) { 294 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2); 295 } else if (TCG_TARGET_HAS_div2_i32) { 296 TCGv_i32 t0 = tcg_temp_new_i32(); 297 tcg_gen_sari_i32(t0, arg1, 31); 298 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2); 299 tcg_temp_free_i32(t0); 300 } else { 301 gen_helper_div_i32(ret, arg1, arg2); 302 } 303 } 304 305 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 306 { 307 if (TCG_TARGET_HAS_rem_i32) { 308 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2); 309 } else if (TCG_TARGET_HAS_div_i32) { 310 TCGv_i32 t0 = tcg_temp_new_i32(); 311 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2); 312 tcg_gen_mul_i32(t0, t0, arg2); 313 tcg_gen_sub_i32(ret, arg1, t0); 314 tcg_temp_free_i32(t0); 315 } else if (TCG_TARGET_HAS_div2_i32) { 316 TCGv_i32 t0 = tcg_temp_new_i32(); 317 tcg_gen_sari_i32(t0, arg1, 31); 318 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2); 319 tcg_temp_free_i32(t0); 320 } else { 321 gen_helper_rem_i32(ret, arg1, arg2); 322 } 323 } 324 325 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 326 { 327 if (TCG_TARGET_HAS_div_i32) { 328 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2); 329 } else if (TCG_TARGET_HAS_div2_i32) { 330 TCGv_i32 t0 = tcg_temp_new_i32(); 331 tcg_gen_movi_i32(t0, 0); 332 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2); 333 tcg_temp_free_i32(t0); 334 } else { 335 gen_helper_divu_i32(ret, arg1, arg2); 336 } 337 } 338 339 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 340 { 341 if (TCG_TARGET_HAS_rem_i32) { 342 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2); 343 } else if (TCG_TARGET_HAS_div_i32) { 344 TCGv_i32 t0 = tcg_temp_new_i32(); 345 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2); 346 tcg_gen_mul_i32(t0, t0, arg2); 347 tcg_gen_sub_i32(ret, arg1, t0); 348 tcg_temp_free_i32(t0); 349 } else if (TCG_TARGET_HAS_div2_i32) { 350 TCGv_i32 t0 = tcg_temp_new_i32(); 351 tcg_gen_movi_i32(t0, 0); 352 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2); 353 tcg_temp_free_i32(t0); 354 } else { 355 gen_helper_remu_i32(ret, arg1, arg2); 356 } 357 } 358 359 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 360 { 361 if (TCG_TARGET_HAS_andc_i32) { 362 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2); 363 } else { 364 TCGv_i32 t0 = tcg_temp_new_i32(); 365 tcg_gen_not_i32(t0, arg2); 366 tcg_gen_and_i32(ret, arg1, t0); 367 tcg_temp_free_i32(t0); 368 } 369 } 370 371 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 372 { 373 if (TCG_TARGET_HAS_eqv_i32) { 374 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2); 375 } else { 376 tcg_gen_xor_i32(ret, arg1, arg2); 377 tcg_gen_not_i32(ret, ret); 378 } 379 } 380 381 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 382 { 383 if (TCG_TARGET_HAS_nand_i32) { 384 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2); 385 } else { 386 tcg_gen_and_i32(ret, arg1, arg2); 387 tcg_gen_not_i32(ret, ret); 388 } 389 } 390 391 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 392 { 393 if (TCG_TARGET_HAS_nor_i32) { 394 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2); 395 } else { 396 tcg_gen_or_i32(ret, arg1, arg2); 397 tcg_gen_not_i32(ret, ret); 398 } 399 } 400 401 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 402 { 403 if (TCG_TARGET_HAS_orc_i32) { 404 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2); 405 } else { 406 TCGv_i32 t0 = tcg_temp_new_i32(); 407 tcg_gen_not_i32(t0, arg2); 408 tcg_gen_or_i32(ret, arg1, t0); 409 tcg_temp_free_i32(t0); 410 } 411 } 412 413 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 414 { 415 if (TCG_TARGET_HAS_clz_i32) { 416 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2); 417 } else if (TCG_TARGET_HAS_clz_i64) { 418 TCGv_i64 t1 = tcg_temp_new_i64(); 419 TCGv_i64 t2 = tcg_temp_new_i64(); 420 tcg_gen_extu_i32_i64(t1, arg1); 421 tcg_gen_extu_i32_i64(t2, arg2); 422 tcg_gen_addi_i64(t2, t2, 32); 423 tcg_gen_clz_i64(t1, t1, t2); 424 tcg_gen_extrl_i64_i32(ret, t1); 425 tcg_temp_free_i64(t1); 426 tcg_temp_free_i64(t2); 427 tcg_gen_subi_i32(ret, ret, 32); 428 } else { 429 gen_helper_clz_i32(ret, arg1, arg2); 430 } 431 } 432 433 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 434 { 435 TCGv_i32 t = tcg_const_i32(arg2); 436 tcg_gen_clz_i32(ret, arg1, t); 437 tcg_temp_free_i32(t); 438 } 439 440 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 441 { 442 if (TCG_TARGET_HAS_ctz_i32) { 443 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); 444 } else if (TCG_TARGET_HAS_ctz_i64) { 445 TCGv_i64 t1 = tcg_temp_new_i64(); 446 TCGv_i64 t2 = tcg_temp_new_i64(); 447 tcg_gen_extu_i32_i64(t1, arg1); 448 tcg_gen_extu_i32_i64(t2, arg2); 449 tcg_gen_ctz_i64(t1, t1, t2); 450 tcg_gen_extrl_i64_i32(ret, t1); 451 tcg_temp_free_i64(t1); 452 tcg_temp_free_i64(t2); 453 } else if (TCG_TARGET_HAS_ctpop_i32 454 || TCG_TARGET_HAS_ctpop_i64 455 || TCG_TARGET_HAS_clz_i32 456 || TCG_TARGET_HAS_clz_i64) { 457 TCGv_i32 z, t = tcg_temp_new_i32(); 458 459 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 460 tcg_gen_subi_i32(t, arg1, 1); 461 tcg_gen_andc_i32(t, t, arg1); 462 tcg_gen_ctpop_i32(t, t); 463 } else { 464 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */ 465 tcg_gen_neg_i32(t, arg1); 466 tcg_gen_and_i32(t, t, arg1); 467 tcg_gen_clzi_i32(t, t, 32); 468 tcg_gen_xori_i32(t, t, 31); 469 } 470 z = tcg_const_i32(0); 471 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 472 tcg_temp_free_i32(t); 473 tcg_temp_free_i32(z); 474 } else { 475 gen_helper_ctz_i32(ret, arg1, arg2); 476 } 477 } 478 479 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 480 { 481 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 482 /* This equivalence has the advantage of not requiring a fixup. */ 483 TCGv_i32 t = tcg_temp_new_i32(); 484 tcg_gen_subi_i32(t, arg1, 1); 485 tcg_gen_andc_i32(t, t, arg1); 486 tcg_gen_ctpop_i32(ret, t); 487 tcg_temp_free_i32(t); 488 } else { 489 TCGv_i32 t = tcg_const_i32(arg2); 490 tcg_gen_ctz_i32(ret, arg1, t); 491 tcg_temp_free_i32(t); 492 } 493 } 494 495 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 496 { 497 if (TCG_TARGET_HAS_clz_i32) { 498 TCGv_i32 t = tcg_temp_new_i32(); 499 tcg_gen_sari_i32(t, arg, 31); 500 tcg_gen_xor_i32(t, t, arg); 501 tcg_gen_clzi_i32(t, t, 32); 502 tcg_gen_subi_i32(ret, t, 1); 503 tcg_temp_free_i32(t); 504 } else { 505 gen_helper_clrsb_i32(ret, arg); 506 } 507 } 508 509 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 510 { 511 if (TCG_TARGET_HAS_ctpop_i32) { 512 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 513 } else if (TCG_TARGET_HAS_ctpop_i64) { 514 TCGv_i64 t = tcg_temp_new_i64(); 515 tcg_gen_extu_i32_i64(t, arg1); 516 tcg_gen_ctpop_i64(t, t); 517 tcg_gen_extrl_i64_i32(ret, t); 518 tcg_temp_free_i64(t); 519 } else { 520 gen_helper_ctpop_i32(ret, arg1); 521 } 522 } 523 524 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 525 { 526 if (TCG_TARGET_HAS_rot_i32) { 527 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2); 528 } else { 529 TCGv_i32 t0, t1; 530 531 t0 = tcg_temp_new_i32(); 532 t1 = tcg_temp_new_i32(); 533 tcg_gen_shl_i32(t0, arg1, arg2); 534 tcg_gen_subfi_i32(t1, 32, arg2); 535 tcg_gen_shr_i32(t1, arg1, t1); 536 tcg_gen_or_i32(ret, t0, t1); 537 tcg_temp_free_i32(t0); 538 tcg_temp_free_i32(t1); 539 } 540 } 541 542 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 543 { 544 tcg_debug_assert(arg2 < 32); 545 /* some cases can be optimized here */ 546 if (arg2 == 0) { 547 tcg_gen_mov_i32(ret, arg1); 548 } else if (TCG_TARGET_HAS_rot_i32) { 549 TCGv_i32 t0 = tcg_const_i32(arg2); 550 tcg_gen_rotl_i32(ret, arg1, t0); 551 tcg_temp_free_i32(t0); 552 } else { 553 TCGv_i32 t0, t1; 554 t0 = tcg_temp_new_i32(); 555 t1 = tcg_temp_new_i32(); 556 tcg_gen_shli_i32(t0, arg1, arg2); 557 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 558 tcg_gen_or_i32(ret, t0, t1); 559 tcg_temp_free_i32(t0); 560 tcg_temp_free_i32(t1); 561 } 562 } 563 564 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 565 { 566 if (TCG_TARGET_HAS_rot_i32) { 567 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2); 568 } else { 569 TCGv_i32 t0, t1; 570 571 t0 = tcg_temp_new_i32(); 572 t1 = tcg_temp_new_i32(); 573 tcg_gen_shr_i32(t0, arg1, arg2); 574 tcg_gen_subfi_i32(t1, 32, arg2); 575 tcg_gen_shl_i32(t1, arg1, t1); 576 tcg_gen_or_i32(ret, t0, t1); 577 tcg_temp_free_i32(t0); 578 tcg_temp_free_i32(t1); 579 } 580 } 581 582 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 583 { 584 tcg_debug_assert(arg2 < 32); 585 /* some cases can be optimized here */ 586 if (arg2 == 0) { 587 tcg_gen_mov_i32(ret, arg1); 588 } else { 589 tcg_gen_rotli_i32(ret, arg1, 32 - arg2); 590 } 591 } 592 593 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 594 unsigned int ofs, unsigned int len) 595 { 596 uint32_t mask; 597 TCGv_i32 t1; 598 599 tcg_debug_assert(ofs < 32); 600 tcg_debug_assert(len > 0); 601 tcg_debug_assert(len <= 32); 602 tcg_debug_assert(ofs + len <= 32); 603 604 if (len == 32) { 605 tcg_gen_mov_i32(ret, arg2); 606 return; 607 } 608 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 609 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 610 return; 611 } 612 613 mask = (1u << len) - 1; 614 t1 = tcg_temp_new_i32(); 615 616 if (ofs + len < 32) { 617 tcg_gen_andi_i32(t1, arg2, mask); 618 tcg_gen_shli_i32(t1, t1, ofs); 619 } else { 620 tcg_gen_shli_i32(t1, arg2, ofs); 621 } 622 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 623 tcg_gen_or_i32(ret, ret, t1); 624 625 tcg_temp_free_i32(t1); 626 } 627 628 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 629 unsigned int ofs, unsigned int len) 630 { 631 tcg_debug_assert(ofs < 32); 632 tcg_debug_assert(len > 0); 633 tcg_debug_assert(len <= 32); 634 tcg_debug_assert(ofs + len <= 32); 635 636 if (ofs + len == 32) { 637 tcg_gen_shli_i32(ret, arg, ofs); 638 } else if (ofs == 0) { 639 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 640 } else if (TCG_TARGET_HAS_deposit_i32 641 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 642 TCGv_i32 zero = tcg_const_i32(0); 643 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 644 tcg_temp_free_i32(zero); 645 } else { 646 /* To help two-operand hosts we prefer to zero-extend first, 647 which allows ARG to stay live. */ 648 switch (len) { 649 case 16: 650 if (TCG_TARGET_HAS_ext16u_i32) { 651 tcg_gen_ext16u_i32(ret, arg); 652 tcg_gen_shli_i32(ret, ret, ofs); 653 return; 654 } 655 break; 656 case 8: 657 if (TCG_TARGET_HAS_ext8u_i32) { 658 tcg_gen_ext8u_i32(ret, arg); 659 tcg_gen_shli_i32(ret, ret, ofs); 660 return; 661 } 662 break; 663 } 664 /* Otherwise prefer zero-extension over AND for code size. */ 665 switch (ofs + len) { 666 case 16: 667 if (TCG_TARGET_HAS_ext16u_i32) { 668 tcg_gen_shli_i32(ret, arg, ofs); 669 tcg_gen_ext16u_i32(ret, ret); 670 return; 671 } 672 break; 673 case 8: 674 if (TCG_TARGET_HAS_ext8u_i32) { 675 tcg_gen_shli_i32(ret, arg, ofs); 676 tcg_gen_ext8u_i32(ret, ret); 677 return; 678 } 679 break; 680 } 681 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 682 tcg_gen_shli_i32(ret, ret, ofs); 683 } 684 } 685 686 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 687 unsigned int ofs, unsigned int len) 688 { 689 tcg_debug_assert(ofs < 32); 690 tcg_debug_assert(len > 0); 691 tcg_debug_assert(len <= 32); 692 tcg_debug_assert(ofs + len <= 32); 693 694 /* Canonicalize certain special cases, even if extract is supported. */ 695 if (ofs + len == 32) { 696 tcg_gen_shri_i32(ret, arg, 32 - len); 697 return; 698 } 699 if (ofs == 0) { 700 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 701 return; 702 } 703 704 if (TCG_TARGET_HAS_extract_i32 705 && TCG_TARGET_extract_i32_valid(ofs, len)) { 706 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 707 return; 708 } 709 710 /* Assume that zero-extension, if available, is cheaper than a shift. */ 711 switch (ofs + len) { 712 case 16: 713 if (TCG_TARGET_HAS_ext16u_i32) { 714 tcg_gen_ext16u_i32(ret, arg); 715 tcg_gen_shri_i32(ret, ret, ofs); 716 return; 717 } 718 break; 719 case 8: 720 if (TCG_TARGET_HAS_ext8u_i32) { 721 tcg_gen_ext8u_i32(ret, arg); 722 tcg_gen_shri_i32(ret, ret, ofs); 723 return; 724 } 725 break; 726 } 727 728 /* ??? Ideally we'd know what values are available for immediate AND. 729 Assume that 8 bits are available, plus the special case of 16, 730 so that we get ext8u, ext16u. */ 731 switch (len) { 732 case 1 ... 8: case 16: 733 tcg_gen_shri_i32(ret, arg, ofs); 734 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 735 break; 736 default: 737 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 738 tcg_gen_shri_i32(ret, ret, 32 - len); 739 break; 740 } 741 } 742 743 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 744 unsigned int ofs, unsigned int len) 745 { 746 tcg_debug_assert(ofs < 32); 747 tcg_debug_assert(len > 0); 748 tcg_debug_assert(len <= 32); 749 tcg_debug_assert(ofs + len <= 32); 750 751 /* Canonicalize certain special cases, even if extract is supported. */ 752 if (ofs + len == 32) { 753 tcg_gen_sari_i32(ret, arg, 32 - len); 754 return; 755 } 756 if (ofs == 0) { 757 switch (len) { 758 case 16: 759 tcg_gen_ext16s_i32(ret, arg); 760 return; 761 case 8: 762 tcg_gen_ext8s_i32(ret, arg); 763 return; 764 } 765 } 766 767 if (TCG_TARGET_HAS_sextract_i32 768 && TCG_TARGET_extract_i32_valid(ofs, len)) { 769 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 770 return; 771 } 772 773 /* Assume that sign-extension, if available, is cheaper than a shift. */ 774 switch (ofs + len) { 775 case 16: 776 if (TCG_TARGET_HAS_ext16s_i32) { 777 tcg_gen_ext16s_i32(ret, arg); 778 tcg_gen_sari_i32(ret, ret, ofs); 779 return; 780 } 781 break; 782 case 8: 783 if (TCG_TARGET_HAS_ext8s_i32) { 784 tcg_gen_ext8s_i32(ret, arg); 785 tcg_gen_sari_i32(ret, ret, ofs); 786 return; 787 } 788 break; 789 } 790 switch (len) { 791 case 16: 792 if (TCG_TARGET_HAS_ext16s_i32) { 793 tcg_gen_shri_i32(ret, arg, ofs); 794 tcg_gen_ext16s_i32(ret, ret); 795 return; 796 } 797 break; 798 case 8: 799 if (TCG_TARGET_HAS_ext8s_i32) { 800 tcg_gen_shri_i32(ret, arg, ofs); 801 tcg_gen_ext8s_i32(ret, ret); 802 return; 803 } 804 break; 805 } 806 807 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 808 tcg_gen_sari_i32(ret, ret, 32 - len); 809 } 810 811 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 812 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 813 { 814 if (cond == TCG_COND_ALWAYS) { 815 tcg_gen_mov_i32(ret, v1); 816 } else if (cond == TCG_COND_NEVER) { 817 tcg_gen_mov_i32(ret, v2); 818 } else if (TCG_TARGET_HAS_movcond_i32) { 819 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 820 } else { 821 TCGv_i32 t0 = tcg_temp_new_i32(); 822 TCGv_i32 t1 = tcg_temp_new_i32(); 823 tcg_gen_setcond_i32(cond, t0, c1, c2); 824 tcg_gen_neg_i32(t0, t0); 825 tcg_gen_and_i32(t1, v1, t0); 826 tcg_gen_andc_i32(ret, v2, t0); 827 tcg_gen_or_i32(ret, ret, t1); 828 tcg_temp_free_i32(t0); 829 tcg_temp_free_i32(t1); 830 } 831 } 832 833 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 834 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 835 { 836 if (TCG_TARGET_HAS_add2_i32) { 837 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 838 } else { 839 TCGv_i64 t0 = tcg_temp_new_i64(); 840 TCGv_i64 t1 = tcg_temp_new_i64(); 841 tcg_gen_concat_i32_i64(t0, al, ah); 842 tcg_gen_concat_i32_i64(t1, bl, bh); 843 tcg_gen_add_i64(t0, t0, t1); 844 tcg_gen_extr_i64_i32(rl, rh, t0); 845 tcg_temp_free_i64(t0); 846 tcg_temp_free_i64(t1); 847 } 848 } 849 850 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 851 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 852 { 853 if (TCG_TARGET_HAS_sub2_i32) { 854 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 855 } else { 856 TCGv_i64 t0 = tcg_temp_new_i64(); 857 TCGv_i64 t1 = tcg_temp_new_i64(); 858 tcg_gen_concat_i32_i64(t0, al, ah); 859 tcg_gen_concat_i32_i64(t1, bl, bh); 860 tcg_gen_sub_i64(t0, t0, t1); 861 tcg_gen_extr_i64_i32(rl, rh, t0); 862 tcg_temp_free_i64(t0); 863 tcg_temp_free_i64(t1); 864 } 865 } 866 867 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 868 { 869 if (TCG_TARGET_HAS_mulu2_i32) { 870 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 871 } else if (TCG_TARGET_HAS_muluh_i32) { 872 TCGv_i32 t = tcg_temp_new_i32(); 873 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 874 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2); 875 tcg_gen_mov_i32(rl, t); 876 tcg_temp_free_i32(t); 877 } else { 878 TCGv_i64 t0 = tcg_temp_new_i64(); 879 TCGv_i64 t1 = tcg_temp_new_i64(); 880 tcg_gen_extu_i32_i64(t0, arg1); 881 tcg_gen_extu_i32_i64(t1, arg2); 882 tcg_gen_mul_i64(t0, t0, t1); 883 tcg_gen_extr_i64_i32(rl, rh, t0); 884 tcg_temp_free_i64(t0); 885 tcg_temp_free_i64(t1); 886 } 887 } 888 889 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 890 { 891 if (TCG_TARGET_HAS_muls2_i32) { 892 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 893 } else if (TCG_TARGET_HAS_mulsh_i32) { 894 TCGv_i32 t = tcg_temp_new_i32(); 895 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 896 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2); 897 tcg_gen_mov_i32(rl, t); 898 tcg_temp_free_i32(t); 899 } else if (TCG_TARGET_REG_BITS == 32) { 900 TCGv_i32 t0 = tcg_temp_new_i32(); 901 TCGv_i32 t1 = tcg_temp_new_i32(); 902 TCGv_i32 t2 = tcg_temp_new_i32(); 903 TCGv_i32 t3 = tcg_temp_new_i32(); 904 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 905 /* Adjust for negative inputs. */ 906 tcg_gen_sari_i32(t2, arg1, 31); 907 tcg_gen_sari_i32(t3, arg2, 31); 908 tcg_gen_and_i32(t2, t2, arg2); 909 tcg_gen_and_i32(t3, t3, arg1); 910 tcg_gen_sub_i32(rh, t1, t2); 911 tcg_gen_sub_i32(rh, rh, t3); 912 tcg_gen_mov_i32(rl, t0); 913 tcg_temp_free_i32(t0); 914 tcg_temp_free_i32(t1); 915 tcg_temp_free_i32(t2); 916 tcg_temp_free_i32(t3); 917 } else { 918 TCGv_i64 t0 = tcg_temp_new_i64(); 919 TCGv_i64 t1 = tcg_temp_new_i64(); 920 tcg_gen_ext_i32_i64(t0, arg1); 921 tcg_gen_ext_i32_i64(t1, arg2); 922 tcg_gen_mul_i64(t0, t0, t1); 923 tcg_gen_extr_i64_i32(rl, rh, t0); 924 tcg_temp_free_i64(t0); 925 tcg_temp_free_i64(t1); 926 } 927 } 928 929 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 930 { 931 if (TCG_TARGET_REG_BITS == 32) { 932 TCGv_i32 t0 = tcg_temp_new_i32(); 933 TCGv_i32 t1 = tcg_temp_new_i32(); 934 TCGv_i32 t2 = tcg_temp_new_i32(); 935 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 936 /* Adjust for negative input for the signed arg1. */ 937 tcg_gen_sari_i32(t2, arg1, 31); 938 tcg_gen_and_i32(t2, t2, arg2); 939 tcg_gen_sub_i32(rh, t1, t2); 940 tcg_gen_mov_i32(rl, t0); 941 tcg_temp_free_i32(t0); 942 tcg_temp_free_i32(t1); 943 tcg_temp_free_i32(t2); 944 } else { 945 TCGv_i64 t0 = tcg_temp_new_i64(); 946 TCGv_i64 t1 = tcg_temp_new_i64(); 947 tcg_gen_ext_i32_i64(t0, arg1); 948 tcg_gen_extu_i32_i64(t1, arg2); 949 tcg_gen_mul_i64(t0, t0, t1); 950 tcg_gen_extr_i64_i32(rl, rh, t0); 951 tcg_temp_free_i64(t0); 952 tcg_temp_free_i64(t1); 953 } 954 } 955 956 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 957 { 958 if (TCG_TARGET_HAS_ext8s_i32) { 959 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg); 960 } else { 961 tcg_gen_shli_i32(ret, arg, 24); 962 tcg_gen_sari_i32(ret, ret, 24); 963 } 964 } 965 966 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 967 { 968 if (TCG_TARGET_HAS_ext16s_i32) { 969 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg); 970 } else { 971 tcg_gen_shli_i32(ret, arg, 16); 972 tcg_gen_sari_i32(ret, ret, 16); 973 } 974 } 975 976 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 977 { 978 if (TCG_TARGET_HAS_ext8u_i32) { 979 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg); 980 } else { 981 tcg_gen_andi_i32(ret, arg, 0xffu); 982 } 983 } 984 985 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 986 { 987 if (TCG_TARGET_HAS_ext16u_i32) { 988 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg); 989 } else { 990 tcg_gen_andi_i32(ret, arg, 0xffffu); 991 } 992 } 993 994 /* Note: we assume the two high bytes are set to zero */ 995 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg) 996 { 997 if (TCG_TARGET_HAS_bswap16_i32) { 998 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg); 999 } else { 1000 TCGv_i32 t0 = tcg_temp_new_i32(); 1001 1002 tcg_gen_ext8u_i32(t0, arg); 1003 tcg_gen_shli_i32(t0, t0, 8); 1004 tcg_gen_shri_i32(ret, arg, 8); 1005 tcg_gen_or_i32(ret, ret, t0); 1006 tcg_temp_free_i32(t0); 1007 } 1008 } 1009 1010 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1011 { 1012 if (TCG_TARGET_HAS_bswap32_i32) { 1013 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg); 1014 } else { 1015 TCGv_i32 t0 = tcg_temp_new_i32(); 1016 TCGv_i32 t1 = tcg_temp_new_i32(); 1017 TCGv_i32 t2 = tcg_const_i32(0x00ff00ff); 1018 1019 /* arg = abcd */ 1020 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */ 1021 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */ 1022 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */ 1023 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */ 1024 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */ 1025 1026 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */ 1027 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */ 1028 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */ 1029 1030 tcg_temp_free_i32(t0); 1031 tcg_temp_free_i32(t1); 1032 tcg_temp_free_i32(t2); 1033 } 1034 } 1035 1036 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1037 { 1038 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1039 } 1040 1041 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1042 { 1043 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1044 } 1045 1046 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1047 { 1048 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1049 } 1050 1051 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1052 { 1053 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1054 } 1055 1056 /* 64-bit ops */ 1057 1058 #if TCG_TARGET_REG_BITS == 32 1059 /* These are all inline for TCG_TARGET_REG_BITS == 64. */ 1060 1061 void tcg_gen_discard_i64(TCGv_i64 arg) 1062 { 1063 tcg_gen_discard_i32(TCGV_LOW(arg)); 1064 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1065 } 1066 1067 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1068 { 1069 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1070 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1071 } 1072 1073 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1074 { 1075 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1076 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1077 } 1078 1079 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1080 { 1081 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1082 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1083 } 1084 1085 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1086 { 1087 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1088 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1089 } 1090 1091 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1092 { 1093 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1094 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1095 } 1096 1097 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1098 { 1099 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1100 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1101 } 1102 1103 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1104 { 1105 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1106 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1107 } 1108 1109 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1110 { 1111 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1112 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1113 } 1114 1115 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1116 { 1117 /* Since arg2 and ret have different types, 1118 they cannot be the same temporary */ 1119 #ifdef HOST_WORDS_BIGENDIAN 1120 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1121 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1122 #else 1123 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1124 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1125 #endif 1126 } 1127 1128 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1129 { 1130 #ifdef HOST_WORDS_BIGENDIAN 1131 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1132 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1133 #else 1134 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1135 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1136 #endif 1137 } 1138 1139 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1140 { 1141 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1142 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1143 } 1144 1145 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1146 { 1147 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1148 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1149 } 1150 1151 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1152 { 1153 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1154 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1155 } 1156 1157 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1158 { 1159 gen_helper_shl_i64(ret, arg1, arg2); 1160 } 1161 1162 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1163 { 1164 gen_helper_shr_i64(ret, arg1, arg2); 1165 } 1166 1167 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1168 { 1169 gen_helper_sar_i64(ret, arg1, arg2); 1170 } 1171 1172 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1173 { 1174 TCGv_i64 t0; 1175 TCGv_i32 t1; 1176 1177 t0 = tcg_temp_new_i64(); 1178 t1 = tcg_temp_new_i32(); 1179 1180 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1181 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1182 1183 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1184 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1185 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1186 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1187 1188 tcg_gen_mov_i64(ret, t0); 1189 tcg_temp_free_i64(t0); 1190 tcg_temp_free_i32(t1); 1191 } 1192 #endif /* TCG_TARGET_REG_SIZE == 32 */ 1193 1194 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1195 { 1196 /* some cases can be optimized here */ 1197 if (arg2 == 0) { 1198 tcg_gen_mov_i64(ret, arg1); 1199 } else { 1200 TCGv_i64 t0 = tcg_const_i64(arg2); 1201 tcg_gen_add_i64(ret, arg1, t0); 1202 tcg_temp_free_i64(t0); 1203 } 1204 } 1205 1206 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1207 { 1208 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) { 1209 /* Don't recurse with tcg_gen_neg_i64. */ 1210 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2); 1211 } else { 1212 TCGv_i64 t0 = tcg_const_i64(arg1); 1213 tcg_gen_sub_i64(ret, t0, arg2); 1214 tcg_temp_free_i64(t0); 1215 } 1216 } 1217 1218 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1219 { 1220 /* some cases can be optimized here */ 1221 if (arg2 == 0) { 1222 tcg_gen_mov_i64(ret, arg1); 1223 } else { 1224 TCGv_i64 t0 = tcg_const_i64(arg2); 1225 tcg_gen_sub_i64(ret, arg1, t0); 1226 tcg_temp_free_i64(t0); 1227 } 1228 } 1229 1230 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1231 { 1232 TCGv_i64 t0; 1233 1234 if (TCG_TARGET_REG_BITS == 32) { 1235 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1236 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1237 return; 1238 } 1239 1240 /* Some cases can be optimized here. */ 1241 switch (arg2) { 1242 case 0: 1243 tcg_gen_movi_i64(ret, 0); 1244 return; 1245 case -1: 1246 tcg_gen_mov_i64(ret, arg1); 1247 return; 1248 case 0xff: 1249 /* Don't recurse with tcg_gen_ext8u_i64. */ 1250 if (TCG_TARGET_HAS_ext8u_i64) { 1251 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1); 1252 return; 1253 } 1254 break; 1255 case 0xffff: 1256 if (TCG_TARGET_HAS_ext16u_i64) { 1257 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1); 1258 return; 1259 } 1260 break; 1261 case 0xffffffffu: 1262 if (TCG_TARGET_HAS_ext32u_i64) { 1263 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1); 1264 return; 1265 } 1266 break; 1267 } 1268 t0 = tcg_const_i64(arg2); 1269 tcg_gen_and_i64(ret, arg1, t0); 1270 tcg_temp_free_i64(t0); 1271 } 1272 1273 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1274 { 1275 if (TCG_TARGET_REG_BITS == 32) { 1276 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1277 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1278 return; 1279 } 1280 /* Some cases can be optimized here. */ 1281 if (arg2 == -1) { 1282 tcg_gen_movi_i64(ret, -1); 1283 } else if (arg2 == 0) { 1284 tcg_gen_mov_i64(ret, arg1); 1285 } else { 1286 TCGv_i64 t0 = tcg_const_i64(arg2); 1287 tcg_gen_or_i64(ret, arg1, t0); 1288 tcg_temp_free_i64(t0); 1289 } 1290 } 1291 1292 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1293 { 1294 if (TCG_TARGET_REG_BITS == 32) { 1295 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1296 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1297 return; 1298 } 1299 /* Some cases can be optimized here. */ 1300 if (arg2 == 0) { 1301 tcg_gen_mov_i64(ret, arg1); 1302 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) { 1303 /* Don't recurse with tcg_gen_not_i64. */ 1304 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1); 1305 } else { 1306 TCGv_i64 t0 = tcg_const_i64(arg2); 1307 tcg_gen_xor_i64(ret, arg1, t0); 1308 tcg_temp_free_i64(t0); 1309 } 1310 } 1311 1312 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1313 unsigned c, bool right, bool arith) 1314 { 1315 tcg_debug_assert(c < 64); 1316 if (c == 0) { 1317 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1318 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1319 } else if (c >= 32) { 1320 c -= 32; 1321 if (right) { 1322 if (arith) { 1323 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1324 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1325 } else { 1326 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1327 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1328 } 1329 } else { 1330 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1331 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1332 } 1333 } else { 1334 TCGv_i32 t0, t1; 1335 1336 t0 = tcg_temp_new_i32(); 1337 t1 = tcg_temp_new_i32(); 1338 if (right) { 1339 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c); 1340 if (arith) { 1341 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c); 1342 } else { 1343 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c); 1344 } 1345 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1346 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0); 1347 tcg_gen_mov_i32(TCGV_HIGH(ret), t1); 1348 } else { 1349 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1350 /* Note: ret can be the same as arg1, so we use t1 */ 1351 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c); 1352 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1353 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0); 1354 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1355 } 1356 tcg_temp_free_i32(t0); 1357 tcg_temp_free_i32(t1); 1358 } 1359 } 1360 1361 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1362 { 1363 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1364 if (TCG_TARGET_REG_BITS == 32) { 1365 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1366 } else if (arg2 == 0) { 1367 tcg_gen_mov_i64(ret, arg1); 1368 } else { 1369 TCGv_i64 t0 = tcg_const_i64(arg2); 1370 tcg_gen_shl_i64(ret, arg1, t0); 1371 tcg_temp_free_i64(t0); 1372 } 1373 } 1374 1375 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1376 { 1377 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1378 if (TCG_TARGET_REG_BITS == 32) { 1379 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1380 } else if (arg2 == 0) { 1381 tcg_gen_mov_i64(ret, arg1); 1382 } else { 1383 TCGv_i64 t0 = tcg_const_i64(arg2); 1384 tcg_gen_shr_i64(ret, arg1, t0); 1385 tcg_temp_free_i64(t0); 1386 } 1387 } 1388 1389 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1390 { 1391 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1392 if (TCG_TARGET_REG_BITS == 32) { 1393 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1394 } else if (arg2 == 0) { 1395 tcg_gen_mov_i64(ret, arg1); 1396 } else { 1397 TCGv_i64 t0 = tcg_const_i64(arg2); 1398 tcg_gen_sar_i64(ret, arg1, t0); 1399 tcg_temp_free_i64(t0); 1400 } 1401 } 1402 1403 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1404 { 1405 if (cond == TCG_COND_ALWAYS) { 1406 tcg_gen_br(l); 1407 } else if (cond != TCG_COND_NEVER) { 1408 if (TCG_TARGET_REG_BITS == 32) { 1409 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1410 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1411 TCGV_HIGH(arg2), cond, label_arg(l)); 1412 } else { 1413 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1414 label_arg(l)); 1415 } 1416 } 1417 } 1418 1419 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1420 { 1421 if (cond == TCG_COND_ALWAYS) { 1422 tcg_gen_br(l); 1423 } else if (cond != TCG_COND_NEVER) { 1424 TCGv_i64 t0 = tcg_const_i64(arg2); 1425 tcg_gen_brcond_i64(cond, arg1, t0, l); 1426 tcg_temp_free_i64(t0); 1427 } 1428 } 1429 1430 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1431 TCGv_i64 arg1, TCGv_i64 arg2) 1432 { 1433 if (cond == TCG_COND_ALWAYS) { 1434 tcg_gen_movi_i64(ret, 1); 1435 } else if (cond == TCG_COND_NEVER) { 1436 tcg_gen_movi_i64(ret, 0); 1437 } else { 1438 if (TCG_TARGET_REG_BITS == 32) { 1439 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1440 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1441 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1442 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1443 } else { 1444 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1445 } 1446 } 1447 } 1448 1449 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1450 TCGv_i64 arg1, int64_t arg2) 1451 { 1452 TCGv_i64 t0 = tcg_const_i64(arg2); 1453 tcg_gen_setcond_i64(cond, ret, arg1, t0); 1454 tcg_temp_free_i64(t0); 1455 } 1456 1457 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1458 { 1459 if (arg2 == 0) { 1460 tcg_gen_movi_i64(ret, 0); 1461 } else if (is_power_of_2(arg2)) { 1462 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 1463 } else { 1464 TCGv_i64 t0 = tcg_const_i64(arg2); 1465 tcg_gen_mul_i64(ret, arg1, t0); 1466 tcg_temp_free_i64(t0); 1467 } 1468 } 1469 1470 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1471 { 1472 if (TCG_TARGET_HAS_div_i64) { 1473 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2); 1474 } else if (TCG_TARGET_HAS_div2_i64) { 1475 TCGv_i64 t0 = tcg_temp_new_i64(); 1476 tcg_gen_sari_i64(t0, arg1, 63); 1477 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2); 1478 tcg_temp_free_i64(t0); 1479 } else { 1480 gen_helper_div_i64(ret, arg1, arg2); 1481 } 1482 } 1483 1484 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1485 { 1486 if (TCG_TARGET_HAS_rem_i64) { 1487 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2); 1488 } else if (TCG_TARGET_HAS_div_i64) { 1489 TCGv_i64 t0 = tcg_temp_new_i64(); 1490 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2); 1491 tcg_gen_mul_i64(t0, t0, arg2); 1492 tcg_gen_sub_i64(ret, arg1, t0); 1493 tcg_temp_free_i64(t0); 1494 } else if (TCG_TARGET_HAS_div2_i64) { 1495 TCGv_i64 t0 = tcg_temp_new_i64(); 1496 tcg_gen_sari_i64(t0, arg1, 63); 1497 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2); 1498 tcg_temp_free_i64(t0); 1499 } else { 1500 gen_helper_rem_i64(ret, arg1, arg2); 1501 } 1502 } 1503 1504 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1505 { 1506 if (TCG_TARGET_HAS_div_i64) { 1507 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2); 1508 } else if (TCG_TARGET_HAS_div2_i64) { 1509 TCGv_i64 t0 = tcg_temp_new_i64(); 1510 tcg_gen_movi_i64(t0, 0); 1511 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2); 1512 tcg_temp_free_i64(t0); 1513 } else { 1514 gen_helper_divu_i64(ret, arg1, arg2); 1515 } 1516 } 1517 1518 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1519 { 1520 if (TCG_TARGET_HAS_rem_i64) { 1521 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2); 1522 } else if (TCG_TARGET_HAS_div_i64) { 1523 TCGv_i64 t0 = tcg_temp_new_i64(); 1524 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2); 1525 tcg_gen_mul_i64(t0, t0, arg2); 1526 tcg_gen_sub_i64(ret, arg1, t0); 1527 tcg_temp_free_i64(t0); 1528 } else if (TCG_TARGET_HAS_div2_i64) { 1529 TCGv_i64 t0 = tcg_temp_new_i64(); 1530 tcg_gen_movi_i64(t0, 0); 1531 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2); 1532 tcg_temp_free_i64(t0); 1533 } else { 1534 gen_helper_remu_i64(ret, arg1, arg2); 1535 } 1536 } 1537 1538 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 1539 { 1540 if (TCG_TARGET_REG_BITS == 32) { 1541 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1542 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1543 } else if (TCG_TARGET_HAS_ext8s_i64) { 1544 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg); 1545 } else { 1546 tcg_gen_shli_i64(ret, arg, 56); 1547 tcg_gen_sari_i64(ret, ret, 56); 1548 } 1549 } 1550 1551 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 1552 { 1553 if (TCG_TARGET_REG_BITS == 32) { 1554 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1555 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1556 } else if (TCG_TARGET_HAS_ext16s_i64) { 1557 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg); 1558 } else { 1559 tcg_gen_shli_i64(ret, arg, 48); 1560 tcg_gen_sari_i64(ret, ret, 48); 1561 } 1562 } 1563 1564 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 1565 { 1566 if (TCG_TARGET_REG_BITS == 32) { 1567 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1568 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1569 } else if (TCG_TARGET_HAS_ext32s_i64) { 1570 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg); 1571 } else { 1572 tcg_gen_shli_i64(ret, arg, 32); 1573 tcg_gen_sari_i64(ret, ret, 32); 1574 } 1575 } 1576 1577 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 1578 { 1579 if (TCG_TARGET_REG_BITS == 32) { 1580 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1581 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1582 } else if (TCG_TARGET_HAS_ext8u_i64) { 1583 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg); 1584 } else { 1585 tcg_gen_andi_i64(ret, arg, 0xffu); 1586 } 1587 } 1588 1589 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 1590 { 1591 if (TCG_TARGET_REG_BITS == 32) { 1592 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1593 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1594 } else if (TCG_TARGET_HAS_ext16u_i64) { 1595 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg); 1596 } else { 1597 tcg_gen_andi_i64(ret, arg, 0xffffu); 1598 } 1599 } 1600 1601 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 1602 { 1603 if (TCG_TARGET_REG_BITS == 32) { 1604 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1605 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1606 } else if (TCG_TARGET_HAS_ext32u_i64) { 1607 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg); 1608 } else { 1609 tcg_gen_andi_i64(ret, arg, 0xffffffffu); 1610 } 1611 } 1612 1613 /* Note: we assume the six high bytes are set to zero */ 1614 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg) 1615 { 1616 if (TCG_TARGET_REG_BITS == 32) { 1617 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1618 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1619 } else if (TCG_TARGET_HAS_bswap16_i64) { 1620 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg); 1621 } else { 1622 TCGv_i64 t0 = tcg_temp_new_i64(); 1623 1624 tcg_gen_ext8u_i64(t0, arg); 1625 tcg_gen_shli_i64(t0, t0, 8); 1626 tcg_gen_shri_i64(ret, arg, 8); 1627 tcg_gen_or_i64(ret, ret, t0); 1628 tcg_temp_free_i64(t0); 1629 } 1630 } 1631 1632 /* Note: we assume the four high bytes are set to zero */ 1633 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg) 1634 { 1635 if (TCG_TARGET_REG_BITS == 32) { 1636 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1637 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1638 } else if (TCG_TARGET_HAS_bswap32_i64) { 1639 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg); 1640 } else { 1641 TCGv_i64 t0 = tcg_temp_new_i64(); 1642 TCGv_i64 t1 = tcg_temp_new_i64(); 1643 TCGv_i64 t2 = tcg_const_i64(0x00ff00ff); 1644 1645 /* arg = ....abcd */ 1646 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .....abc */ 1647 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */ 1648 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */ 1649 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */ 1650 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */ 1651 1652 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */ 1653 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */ 1654 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */ 1655 tcg_gen_or_i64(ret, t0, t1); /* ret = ....dcba */ 1656 1657 tcg_temp_free_i64(t0); 1658 tcg_temp_free_i64(t1); 1659 tcg_temp_free_i64(t2); 1660 } 1661 } 1662 1663 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 1664 { 1665 if (TCG_TARGET_REG_BITS == 32) { 1666 TCGv_i32 t0, t1; 1667 t0 = tcg_temp_new_i32(); 1668 t1 = tcg_temp_new_i32(); 1669 1670 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 1671 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 1672 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1673 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 1674 tcg_temp_free_i32(t0); 1675 tcg_temp_free_i32(t1); 1676 } else if (TCG_TARGET_HAS_bswap64_i64) { 1677 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg); 1678 } else { 1679 TCGv_i64 t0 = tcg_temp_new_i64(); 1680 TCGv_i64 t1 = tcg_temp_new_i64(); 1681 TCGv_i64 t2 = tcg_temp_new_i64(); 1682 1683 /* arg = abcdefgh */ 1684 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull); 1685 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */ 1686 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */ 1687 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */ 1688 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */ 1689 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */ 1690 1691 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull); 1692 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */ 1693 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */ 1694 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */ 1695 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */ 1696 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */ 1697 1698 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */ 1699 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */ 1700 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */ 1701 1702 tcg_temp_free_i64(t0); 1703 tcg_temp_free_i64(t1); 1704 tcg_temp_free_i64(t2); 1705 } 1706 } 1707 1708 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 1709 { 1710 if (TCG_TARGET_REG_BITS == 32) { 1711 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1712 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1713 } else if (TCG_TARGET_HAS_not_i64) { 1714 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg); 1715 } else { 1716 tcg_gen_xori_i64(ret, arg, -1); 1717 } 1718 } 1719 1720 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1721 { 1722 if (TCG_TARGET_REG_BITS == 32) { 1723 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1724 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1725 } else if (TCG_TARGET_HAS_andc_i64) { 1726 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2); 1727 } else { 1728 TCGv_i64 t0 = tcg_temp_new_i64(); 1729 tcg_gen_not_i64(t0, arg2); 1730 tcg_gen_and_i64(ret, arg1, t0); 1731 tcg_temp_free_i64(t0); 1732 } 1733 } 1734 1735 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1736 { 1737 if (TCG_TARGET_REG_BITS == 32) { 1738 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1739 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1740 } else if (TCG_TARGET_HAS_eqv_i64) { 1741 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2); 1742 } else { 1743 tcg_gen_xor_i64(ret, arg1, arg2); 1744 tcg_gen_not_i64(ret, ret); 1745 } 1746 } 1747 1748 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1749 { 1750 if (TCG_TARGET_REG_BITS == 32) { 1751 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1752 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1753 } else if (TCG_TARGET_HAS_nand_i64) { 1754 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2); 1755 } else { 1756 tcg_gen_and_i64(ret, arg1, arg2); 1757 tcg_gen_not_i64(ret, ret); 1758 } 1759 } 1760 1761 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1762 { 1763 if (TCG_TARGET_REG_BITS == 32) { 1764 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1765 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1766 } else if (TCG_TARGET_HAS_nor_i64) { 1767 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2); 1768 } else { 1769 tcg_gen_or_i64(ret, arg1, arg2); 1770 tcg_gen_not_i64(ret, ret); 1771 } 1772 } 1773 1774 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1775 { 1776 if (TCG_TARGET_REG_BITS == 32) { 1777 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1778 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1779 } else if (TCG_TARGET_HAS_orc_i64) { 1780 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2); 1781 } else { 1782 TCGv_i64 t0 = tcg_temp_new_i64(); 1783 tcg_gen_not_i64(t0, arg2); 1784 tcg_gen_or_i64(ret, arg1, t0); 1785 tcg_temp_free_i64(t0); 1786 } 1787 } 1788 1789 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1790 { 1791 if (TCG_TARGET_HAS_clz_i64) { 1792 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2); 1793 } else { 1794 gen_helper_clz_i64(ret, arg1, arg2); 1795 } 1796 } 1797 1798 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1799 { 1800 if (TCG_TARGET_REG_BITS == 32 1801 && TCG_TARGET_HAS_clz_i32 1802 && arg2 <= 0xffffffffu) { 1803 TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32); 1804 tcg_gen_clz_i32(t, TCGV_LOW(arg1), t); 1805 tcg_gen_addi_i32(t, t, 32); 1806 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 1807 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1808 tcg_temp_free_i32(t); 1809 } else { 1810 TCGv_i64 t = tcg_const_i64(arg2); 1811 tcg_gen_clz_i64(ret, arg1, t); 1812 tcg_temp_free_i64(t); 1813 } 1814 } 1815 1816 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1817 { 1818 if (TCG_TARGET_HAS_ctz_i64) { 1819 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); 1820 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) { 1821 TCGv_i64 z, t = tcg_temp_new_i64(); 1822 1823 if (TCG_TARGET_HAS_ctpop_i64) { 1824 tcg_gen_subi_i64(t, arg1, 1); 1825 tcg_gen_andc_i64(t, t, arg1); 1826 tcg_gen_ctpop_i64(t, t); 1827 } else { 1828 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */ 1829 tcg_gen_neg_i64(t, arg1); 1830 tcg_gen_and_i64(t, t, arg1); 1831 tcg_gen_clzi_i64(t, t, 64); 1832 tcg_gen_xori_i64(t, t, 63); 1833 } 1834 z = tcg_const_i64(0); 1835 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 1836 tcg_temp_free_i64(t); 1837 tcg_temp_free_i64(z); 1838 } else { 1839 gen_helper_ctz_i64(ret, arg1, arg2); 1840 } 1841 } 1842 1843 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1844 { 1845 if (TCG_TARGET_REG_BITS == 32 1846 && TCG_TARGET_HAS_ctz_i32 1847 && arg2 <= 0xffffffffu) { 1848 TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32); 1849 tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32); 1850 tcg_gen_addi_i32(t32, t32, 32); 1851 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 1852 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1853 tcg_temp_free_i32(t32); 1854 } else if (!TCG_TARGET_HAS_ctz_i64 1855 && TCG_TARGET_HAS_ctpop_i64 1856 && arg2 == 64) { 1857 /* This equivalence has the advantage of not requiring a fixup. */ 1858 TCGv_i64 t = tcg_temp_new_i64(); 1859 tcg_gen_subi_i64(t, arg1, 1); 1860 tcg_gen_andc_i64(t, t, arg1); 1861 tcg_gen_ctpop_i64(ret, t); 1862 tcg_temp_free_i64(t); 1863 } else { 1864 TCGv_i64 t64 = tcg_const_i64(arg2); 1865 tcg_gen_ctz_i64(ret, arg1, t64); 1866 tcg_temp_free_i64(t64); 1867 } 1868 } 1869 1870 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 1871 { 1872 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) { 1873 TCGv_i64 t = tcg_temp_new_i64(); 1874 tcg_gen_sari_i64(t, arg, 63); 1875 tcg_gen_xor_i64(t, t, arg); 1876 tcg_gen_clzi_i64(t, t, 64); 1877 tcg_gen_subi_i64(ret, t, 1); 1878 tcg_temp_free_i64(t); 1879 } else { 1880 gen_helper_clrsb_i64(ret, arg); 1881 } 1882 } 1883 1884 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 1885 { 1886 if (TCG_TARGET_HAS_ctpop_i64) { 1887 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 1888 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 1889 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1890 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1891 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 1892 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1893 } else { 1894 gen_helper_ctpop_i64(ret, arg1); 1895 } 1896 } 1897 1898 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1899 { 1900 if (TCG_TARGET_HAS_rot_i64) { 1901 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2); 1902 } else { 1903 TCGv_i64 t0, t1; 1904 t0 = tcg_temp_new_i64(); 1905 t1 = tcg_temp_new_i64(); 1906 tcg_gen_shl_i64(t0, arg1, arg2); 1907 tcg_gen_subfi_i64(t1, 64, arg2); 1908 tcg_gen_shr_i64(t1, arg1, t1); 1909 tcg_gen_or_i64(ret, t0, t1); 1910 tcg_temp_free_i64(t0); 1911 tcg_temp_free_i64(t1); 1912 } 1913 } 1914 1915 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1916 { 1917 tcg_debug_assert(arg2 < 64); 1918 /* some cases can be optimized here */ 1919 if (arg2 == 0) { 1920 tcg_gen_mov_i64(ret, arg1); 1921 } else if (TCG_TARGET_HAS_rot_i64) { 1922 TCGv_i64 t0 = tcg_const_i64(arg2); 1923 tcg_gen_rotl_i64(ret, arg1, t0); 1924 tcg_temp_free_i64(t0); 1925 } else { 1926 TCGv_i64 t0, t1; 1927 t0 = tcg_temp_new_i64(); 1928 t1 = tcg_temp_new_i64(); 1929 tcg_gen_shli_i64(t0, arg1, arg2); 1930 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 1931 tcg_gen_or_i64(ret, t0, t1); 1932 tcg_temp_free_i64(t0); 1933 tcg_temp_free_i64(t1); 1934 } 1935 } 1936 1937 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1938 { 1939 if (TCG_TARGET_HAS_rot_i64) { 1940 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2); 1941 } else { 1942 TCGv_i64 t0, t1; 1943 t0 = tcg_temp_new_i64(); 1944 t1 = tcg_temp_new_i64(); 1945 tcg_gen_shr_i64(t0, arg1, arg2); 1946 tcg_gen_subfi_i64(t1, 64, arg2); 1947 tcg_gen_shl_i64(t1, arg1, t1); 1948 tcg_gen_or_i64(ret, t0, t1); 1949 tcg_temp_free_i64(t0); 1950 tcg_temp_free_i64(t1); 1951 } 1952 } 1953 1954 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1955 { 1956 tcg_debug_assert(arg2 < 64); 1957 /* some cases can be optimized here */ 1958 if (arg2 == 0) { 1959 tcg_gen_mov_i64(ret, arg1); 1960 } else { 1961 tcg_gen_rotli_i64(ret, arg1, 64 - arg2); 1962 } 1963 } 1964 1965 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 1966 unsigned int ofs, unsigned int len) 1967 { 1968 uint64_t mask; 1969 TCGv_i64 t1; 1970 1971 tcg_debug_assert(ofs < 64); 1972 tcg_debug_assert(len > 0); 1973 tcg_debug_assert(len <= 64); 1974 tcg_debug_assert(ofs + len <= 64); 1975 1976 if (len == 64) { 1977 tcg_gen_mov_i64(ret, arg2); 1978 return; 1979 } 1980 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 1981 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 1982 return; 1983 } 1984 1985 if (TCG_TARGET_REG_BITS == 32) { 1986 if (ofs >= 32) { 1987 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 1988 TCGV_LOW(arg2), ofs - 32, len); 1989 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1990 return; 1991 } 1992 if (ofs + len <= 32) { 1993 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 1994 TCGV_LOW(arg2), ofs, len); 1995 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1996 return; 1997 } 1998 } 1999 2000 mask = (1ull << len) - 1; 2001 t1 = tcg_temp_new_i64(); 2002 2003 if (ofs + len < 64) { 2004 tcg_gen_andi_i64(t1, arg2, mask); 2005 tcg_gen_shli_i64(t1, t1, ofs); 2006 } else { 2007 tcg_gen_shli_i64(t1, arg2, ofs); 2008 } 2009 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2010 tcg_gen_or_i64(ret, ret, t1); 2011 2012 tcg_temp_free_i64(t1); 2013 } 2014 2015 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2016 unsigned int ofs, unsigned int len) 2017 { 2018 tcg_debug_assert(ofs < 64); 2019 tcg_debug_assert(len > 0); 2020 tcg_debug_assert(len <= 64); 2021 tcg_debug_assert(ofs + len <= 64); 2022 2023 if (ofs + len == 64) { 2024 tcg_gen_shli_i64(ret, arg, ofs); 2025 } else if (ofs == 0) { 2026 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2027 } else if (TCG_TARGET_HAS_deposit_i64 2028 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2029 TCGv_i64 zero = tcg_const_i64(0); 2030 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2031 tcg_temp_free_i64(zero); 2032 } else { 2033 if (TCG_TARGET_REG_BITS == 32) { 2034 if (ofs >= 32) { 2035 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2036 ofs - 32, len); 2037 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2038 return; 2039 } 2040 if (ofs + len <= 32) { 2041 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2042 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2043 return; 2044 } 2045 } 2046 /* To help two-operand hosts we prefer to zero-extend first, 2047 which allows ARG to stay live. */ 2048 switch (len) { 2049 case 32: 2050 if (TCG_TARGET_HAS_ext32u_i64) { 2051 tcg_gen_ext32u_i64(ret, arg); 2052 tcg_gen_shli_i64(ret, ret, ofs); 2053 return; 2054 } 2055 break; 2056 case 16: 2057 if (TCG_TARGET_HAS_ext16u_i64) { 2058 tcg_gen_ext16u_i64(ret, arg); 2059 tcg_gen_shli_i64(ret, ret, ofs); 2060 return; 2061 } 2062 break; 2063 case 8: 2064 if (TCG_TARGET_HAS_ext8u_i64) { 2065 tcg_gen_ext8u_i64(ret, arg); 2066 tcg_gen_shli_i64(ret, ret, ofs); 2067 return; 2068 } 2069 break; 2070 } 2071 /* Otherwise prefer zero-extension over AND for code size. */ 2072 switch (ofs + len) { 2073 case 32: 2074 if (TCG_TARGET_HAS_ext32u_i64) { 2075 tcg_gen_shli_i64(ret, arg, ofs); 2076 tcg_gen_ext32u_i64(ret, ret); 2077 return; 2078 } 2079 break; 2080 case 16: 2081 if (TCG_TARGET_HAS_ext16u_i64) { 2082 tcg_gen_shli_i64(ret, arg, ofs); 2083 tcg_gen_ext16u_i64(ret, ret); 2084 return; 2085 } 2086 break; 2087 case 8: 2088 if (TCG_TARGET_HAS_ext8u_i64) { 2089 tcg_gen_shli_i64(ret, arg, ofs); 2090 tcg_gen_ext8u_i64(ret, ret); 2091 return; 2092 } 2093 break; 2094 } 2095 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2096 tcg_gen_shli_i64(ret, ret, ofs); 2097 } 2098 } 2099 2100 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2101 unsigned int ofs, unsigned int len) 2102 { 2103 tcg_debug_assert(ofs < 64); 2104 tcg_debug_assert(len > 0); 2105 tcg_debug_assert(len <= 64); 2106 tcg_debug_assert(ofs + len <= 64); 2107 2108 /* Canonicalize certain special cases, even if extract is supported. */ 2109 if (ofs + len == 64) { 2110 tcg_gen_shri_i64(ret, arg, 64 - len); 2111 return; 2112 } 2113 if (ofs == 0) { 2114 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2115 return; 2116 } 2117 2118 if (TCG_TARGET_REG_BITS == 32) { 2119 /* Look for a 32-bit extract within one of the two words. */ 2120 if (ofs >= 32) { 2121 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2122 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2123 return; 2124 } 2125 if (ofs + len <= 32) { 2126 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2127 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2128 return; 2129 } 2130 /* The field is split across two words. One double-word 2131 shift is better than two double-word shifts. */ 2132 goto do_shift_and; 2133 } 2134 2135 if (TCG_TARGET_HAS_extract_i64 2136 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2137 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2138 return; 2139 } 2140 2141 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2142 switch (ofs + len) { 2143 case 32: 2144 if (TCG_TARGET_HAS_ext32u_i64) { 2145 tcg_gen_ext32u_i64(ret, arg); 2146 tcg_gen_shri_i64(ret, ret, ofs); 2147 return; 2148 } 2149 break; 2150 case 16: 2151 if (TCG_TARGET_HAS_ext16u_i64) { 2152 tcg_gen_ext16u_i64(ret, arg); 2153 tcg_gen_shri_i64(ret, ret, ofs); 2154 return; 2155 } 2156 break; 2157 case 8: 2158 if (TCG_TARGET_HAS_ext8u_i64) { 2159 tcg_gen_ext8u_i64(ret, arg); 2160 tcg_gen_shri_i64(ret, ret, ofs); 2161 return; 2162 } 2163 break; 2164 } 2165 2166 /* ??? Ideally we'd know what values are available for immediate AND. 2167 Assume that 8 bits are available, plus the special cases of 16 and 32, 2168 so that we get ext8u, ext16u, and ext32u. */ 2169 switch (len) { 2170 case 1 ... 8: case 16: case 32: 2171 do_shift_and: 2172 tcg_gen_shri_i64(ret, arg, ofs); 2173 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2174 break; 2175 default: 2176 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2177 tcg_gen_shri_i64(ret, ret, 64 - len); 2178 break; 2179 } 2180 } 2181 2182 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2183 unsigned int ofs, unsigned int len) 2184 { 2185 tcg_debug_assert(ofs < 64); 2186 tcg_debug_assert(len > 0); 2187 tcg_debug_assert(len <= 64); 2188 tcg_debug_assert(ofs + len <= 64); 2189 2190 /* Canonicalize certain special cases, even if sextract is supported. */ 2191 if (ofs + len == 64) { 2192 tcg_gen_sari_i64(ret, arg, 64 - len); 2193 return; 2194 } 2195 if (ofs == 0) { 2196 switch (len) { 2197 case 32: 2198 tcg_gen_ext32s_i64(ret, arg); 2199 return; 2200 case 16: 2201 tcg_gen_ext16s_i64(ret, arg); 2202 return; 2203 case 8: 2204 tcg_gen_ext8s_i64(ret, arg); 2205 return; 2206 } 2207 } 2208 2209 if (TCG_TARGET_REG_BITS == 32) { 2210 /* Look for a 32-bit extract within one of the two words. */ 2211 if (ofs >= 32) { 2212 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2213 } else if (ofs + len <= 32) { 2214 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2215 } else if (ofs == 0) { 2216 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2217 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2218 return; 2219 } else if (len > 32) { 2220 TCGv_i32 t = tcg_temp_new_i32(); 2221 /* Extract the bits for the high word normally. */ 2222 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2223 /* Shift the field down for the low part. */ 2224 tcg_gen_shri_i64(ret, arg, ofs); 2225 /* Overwrite the shift into the high part. */ 2226 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2227 tcg_temp_free_i32(t); 2228 return; 2229 } else { 2230 /* Shift the field down for the low part, such that the 2231 field sits at the MSB. */ 2232 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2233 /* Shift the field down from the MSB, sign extending. */ 2234 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2235 } 2236 /* Sign-extend the field from 32 bits. */ 2237 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2238 return; 2239 } 2240 2241 if (TCG_TARGET_HAS_sextract_i64 2242 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2243 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2244 return; 2245 } 2246 2247 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2248 switch (ofs + len) { 2249 case 32: 2250 if (TCG_TARGET_HAS_ext32s_i64) { 2251 tcg_gen_ext32s_i64(ret, arg); 2252 tcg_gen_sari_i64(ret, ret, ofs); 2253 return; 2254 } 2255 break; 2256 case 16: 2257 if (TCG_TARGET_HAS_ext16s_i64) { 2258 tcg_gen_ext16s_i64(ret, arg); 2259 tcg_gen_sari_i64(ret, ret, ofs); 2260 return; 2261 } 2262 break; 2263 case 8: 2264 if (TCG_TARGET_HAS_ext8s_i64) { 2265 tcg_gen_ext8s_i64(ret, arg); 2266 tcg_gen_sari_i64(ret, ret, ofs); 2267 return; 2268 } 2269 break; 2270 } 2271 switch (len) { 2272 case 32: 2273 if (TCG_TARGET_HAS_ext32s_i64) { 2274 tcg_gen_shri_i64(ret, arg, ofs); 2275 tcg_gen_ext32s_i64(ret, ret); 2276 return; 2277 } 2278 break; 2279 case 16: 2280 if (TCG_TARGET_HAS_ext16s_i64) { 2281 tcg_gen_shri_i64(ret, arg, ofs); 2282 tcg_gen_ext16s_i64(ret, ret); 2283 return; 2284 } 2285 break; 2286 case 8: 2287 if (TCG_TARGET_HAS_ext8s_i64) { 2288 tcg_gen_shri_i64(ret, arg, ofs); 2289 tcg_gen_ext8s_i64(ret, ret); 2290 return; 2291 } 2292 break; 2293 } 2294 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2295 tcg_gen_sari_i64(ret, ret, 64 - len); 2296 } 2297 2298 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2299 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2300 { 2301 if (cond == TCG_COND_ALWAYS) { 2302 tcg_gen_mov_i64(ret, v1); 2303 } else if (cond == TCG_COND_NEVER) { 2304 tcg_gen_mov_i64(ret, v2); 2305 } else if (TCG_TARGET_REG_BITS == 32) { 2306 TCGv_i32 t0 = tcg_temp_new_i32(); 2307 TCGv_i32 t1 = tcg_temp_new_i32(); 2308 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2309 TCGV_LOW(c1), TCGV_HIGH(c1), 2310 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2311 2312 if (TCG_TARGET_HAS_movcond_i32) { 2313 tcg_gen_movi_i32(t1, 0); 2314 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1, 2315 TCGV_LOW(v1), TCGV_LOW(v2)); 2316 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1, 2317 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2318 } else { 2319 tcg_gen_neg_i32(t0, t0); 2320 2321 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0); 2322 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0); 2323 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1); 2324 2325 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0); 2326 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0); 2327 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1); 2328 } 2329 tcg_temp_free_i32(t0); 2330 tcg_temp_free_i32(t1); 2331 } else if (TCG_TARGET_HAS_movcond_i64) { 2332 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2333 } else { 2334 TCGv_i64 t0 = tcg_temp_new_i64(); 2335 TCGv_i64 t1 = tcg_temp_new_i64(); 2336 tcg_gen_setcond_i64(cond, t0, c1, c2); 2337 tcg_gen_neg_i64(t0, t0); 2338 tcg_gen_and_i64(t1, v1, t0); 2339 tcg_gen_andc_i64(ret, v2, t0); 2340 tcg_gen_or_i64(ret, ret, t1); 2341 tcg_temp_free_i64(t0); 2342 tcg_temp_free_i64(t1); 2343 } 2344 } 2345 2346 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2347 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2348 { 2349 if (TCG_TARGET_HAS_add2_i64) { 2350 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2351 } else { 2352 TCGv_i64 t0 = tcg_temp_new_i64(); 2353 TCGv_i64 t1 = tcg_temp_new_i64(); 2354 tcg_gen_add_i64(t0, al, bl); 2355 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2356 tcg_gen_add_i64(rh, ah, bh); 2357 tcg_gen_add_i64(rh, rh, t1); 2358 tcg_gen_mov_i64(rl, t0); 2359 tcg_temp_free_i64(t0); 2360 tcg_temp_free_i64(t1); 2361 } 2362 } 2363 2364 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2365 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2366 { 2367 if (TCG_TARGET_HAS_sub2_i64) { 2368 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2369 } else { 2370 TCGv_i64 t0 = tcg_temp_new_i64(); 2371 TCGv_i64 t1 = tcg_temp_new_i64(); 2372 tcg_gen_sub_i64(t0, al, bl); 2373 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2374 tcg_gen_sub_i64(rh, ah, bh); 2375 tcg_gen_sub_i64(rh, rh, t1); 2376 tcg_gen_mov_i64(rl, t0); 2377 tcg_temp_free_i64(t0); 2378 tcg_temp_free_i64(t1); 2379 } 2380 } 2381 2382 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2383 { 2384 if (TCG_TARGET_HAS_mulu2_i64) { 2385 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2386 } else if (TCG_TARGET_HAS_muluh_i64) { 2387 TCGv_i64 t = tcg_temp_new_i64(); 2388 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2389 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2); 2390 tcg_gen_mov_i64(rl, t); 2391 tcg_temp_free_i64(t); 2392 } else { 2393 TCGv_i64 t0 = tcg_temp_new_i64(); 2394 tcg_gen_mul_i64(t0, arg1, arg2); 2395 gen_helper_muluh_i64(rh, arg1, arg2); 2396 tcg_gen_mov_i64(rl, t0); 2397 tcg_temp_free_i64(t0); 2398 } 2399 } 2400 2401 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2402 { 2403 if (TCG_TARGET_HAS_muls2_i64) { 2404 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 2405 } else if (TCG_TARGET_HAS_mulsh_i64) { 2406 TCGv_i64 t = tcg_temp_new_i64(); 2407 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2408 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2); 2409 tcg_gen_mov_i64(rl, t); 2410 tcg_temp_free_i64(t); 2411 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) { 2412 TCGv_i64 t0 = tcg_temp_new_i64(); 2413 TCGv_i64 t1 = tcg_temp_new_i64(); 2414 TCGv_i64 t2 = tcg_temp_new_i64(); 2415 TCGv_i64 t3 = tcg_temp_new_i64(); 2416 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2417 /* Adjust for negative inputs. */ 2418 tcg_gen_sari_i64(t2, arg1, 63); 2419 tcg_gen_sari_i64(t3, arg2, 63); 2420 tcg_gen_and_i64(t2, t2, arg2); 2421 tcg_gen_and_i64(t3, t3, arg1); 2422 tcg_gen_sub_i64(rh, t1, t2); 2423 tcg_gen_sub_i64(rh, rh, t3); 2424 tcg_gen_mov_i64(rl, t0); 2425 tcg_temp_free_i64(t0); 2426 tcg_temp_free_i64(t1); 2427 tcg_temp_free_i64(t2); 2428 tcg_temp_free_i64(t3); 2429 } else { 2430 TCGv_i64 t0 = tcg_temp_new_i64(); 2431 tcg_gen_mul_i64(t0, arg1, arg2); 2432 gen_helper_mulsh_i64(rh, arg1, arg2); 2433 tcg_gen_mov_i64(rl, t0); 2434 tcg_temp_free_i64(t0); 2435 } 2436 } 2437 2438 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2439 { 2440 TCGv_i64 t0 = tcg_temp_new_i64(); 2441 TCGv_i64 t1 = tcg_temp_new_i64(); 2442 TCGv_i64 t2 = tcg_temp_new_i64(); 2443 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2444 /* Adjust for negative input for the signed arg1. */ 2445 tcg_gen_sari_i64(t2, arg1, 63); 2446 tcg_gen_and_i64(t2, t2, arg2); 2447 tcg_gen_sub_i64(rh, t1, t2); 2448 tcg_gen_mov_i64(rl, t0); 2449 tcg_temp_free_i64(t0); 2450 tcg_temp_free_i64(t1); 2451 tcg_temp_free_i64(t2); 2452 } 2453 2454 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2455 { 2456 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 2457 } 2458 2459 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2460 { 2461 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 2462 } 2463 2464 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2465 { 2466 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 2467 } 2468 2469 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2470 { 2471 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 2472 } 2473 2474 /* Size changing operations. */ 2475 2476 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2477 { 2478 if (TCG_TARGET_REG_BITS == 32) { 2479 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2480 } else if (TCG_TARGET_HAS_extrl_i64_i32) { 2481 tcg_gen_op2(INDEX_op_extrl_i64_i32, 2482 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2483 } else { 2484 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2485 } 2486 } 2487 2488 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2489 { 2490 if (TCG_TARGET_REG_BITS == 32) { 2491 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2492 } else if (TCG_TARGET_HAS_extrh_i64_i32) { 2493 tcg_gen_op2(INDEX_op_extrh_i64_i32, 2494 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2495 } else { 2496 TCGv_i64 t = tcg_temp_new_i64(); 2497 tcg_gen_shri_i64(t, arg, 32); 2498 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2499 tcg_temp_free_i64(t); 2500 } 2501 } 2502 2503 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2504 { 2505 if (TCG_TARGET_REG_BITS == 32) { 2506 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2507 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2508 } else { 2509 tcg_gen_op2(INDEX_op_extu_i32_i64, 2510 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2511 } 2512 } 2513 2514 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2515 { 2516 if (TCG_TARGET_REG_BITS == 32) { 2517 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2518 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2519 } else { 2520 tcg_gen_op2(INDEX_op_ext_i32_i64, 2521 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2522 } 2523 } 2524 2525 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 2526 { 2527 TCGv_i64 tmp; 2528 2529 if (TCG_TARGET_REG_BITS == 32) { 2530 tcg_gen_mov_i32(TCGV_LOW(dest), low); 2531 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 2532 return; 2533 } 2534 2535 tmp = tcg_temp_new_i64(); 2536 /* These extensions are only needed for type correctness. 2537 We may be able to do better given target specific information. */ 2538 tcg_gen_extu_i32_i64(tmp, high); 2539 tcg_gen_extu_i32_i64(dest, low); 2540 /* If deposit is available, use it. Otherwise use the extra 2541 knowledge that we have of the zero-extensions above. */ 2542 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) { 2543 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 2544 } else { 2545 tcg_gen_shli_i64(tmp, tmp, 32); 2546 tcg_gen_or_i64(dest, dest, tmp); 2547 } 2548 tcg_temp_free_i64(tmp); 2549 } 2550 2551 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 2552 { 2553 if (TCG_TARGET_REG_BITS == 32) { 2554 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 2555 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 2556 } else { 2557 tcg_gen_extrl_i64_i32(lo, arg); 2558 tcg_gen_extrh_i64_i32(hi, arg); 2559 } 2560 } 2561 2562 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 2563 { 2564 tcg_gen_ext32u_i64(lo, arg); 2565 tcg_gen_shri_i64(hi, arg, 32); 2566 } 2567 2568 /* QEMU specific operations. */ 2569 2570 void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx) 2571 { 2572 uintptr_t val = (uintptr_t)tb + idx; 2573 2574 if (tb == NULL) { 2575 tcg_debug_assert(idx == 0); 2576 } else if (idx <= TB_EXIT_IDXMAX) { 2577 #ifdef CONFIG_DEBUG_TCG 2578 /* This is an exit following a goto_tb. Verify that we have 2579 seen this numbered exit before, via tcg_gen_goto_tb. */ 2580 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); 2581 #endif 2582 /* When not chaining, exit without indicating a link. */ 2583 if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2584 val = 0; 2585 } 2586 } else { 2587 /* This is an exit via the exitreq label. */ 2588 tcg_debug_assert(idx == TB_EXIT_REQUESTED); 2589 } 2590 2591 tcg_gen_op1i(INDEX_op_exit_tb, val); 2592 } 2593 2594 void tcg_gen_goto_tb(unsigned idx) 2595 { 2596 /* We only support two chained exits. */ 2597 tcg_debug_assert(idx <= TB_EXIT_IDXMAX); 2598 #ifdef CONFIG_DEBUG_TCG 2599 /* Verify that we havn't seen this numbered exit before. */ 2600 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 2601 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 2602 #endif 2603 /* When not chaining, we simply fall through to the "fallback" exit. */ 2604 if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2605 tcg_gen_op1i(INDEX_op_goto_tb, idx); 2606 } 2607 } 2608 2609 void tcg_gen_lookup_and_goto_ptr(void) 2610 { 2611 if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2612 TCGv_ptr ptr = tcg_temp_new_ptr(); 2613 gen_helper_lookup_tb_ptr(ptr, cpu_env); 2614 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr)); 2615 tcg_temp_free_ptr(ptr); 2616 } else { 2617 tcg_gen_exit_tb(NULL, 0); 2618 } 2619 } 2620 2621 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st) 2622 { 2623 /* Trigger the asserts within as early as possible. */ 2624 (void)get_alignment_bits(op); 2625 2626 switch (op & MO_SIZE) { 2627 case MO_8: 2628 op &= ~MO_BSWAP; 2629 break; 2630 case MO_16: 2631 break; 2632 case MO_32: 2633 if (!is64) { 2634 op &= ~MO_SIGN; 2635 } 2636 break; 2637 case MO_64: 2638 if (!is64) { 2639 tcg_abort(); 2640 } 2641 break; 2642 } 2643 if (st) { 2644 op &= ~MO_SIGN; 2645 } 2646 return op; 2647 } 2648 2649 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr, 2650 TCGMemOp memop, TCGArg idx) 2651 { 2652 TCGMemOpIdx oi = make_memop_idx(memop, idx); 2653 #if TARGET_LONG_BITS == 32 2654 tcg_gen_op3i_i32(opc, val, addr, oi); 2655 #else 2656 if (TCG_TARGET_REG_BITS == 32) { 2657 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2658 } else { 2659 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi); 2660 } 2661 #endif 2662 } 2663 2664 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr, 2665 TCGMemOp memop, TCGArg idx) 2666 { 2667 TCGMemOpIdx oi = make_memop_idx(memop, idx); 2668 #if TARGET_LONG_BITS == 32 2669 if (TCG_TARGET_REG_BITS == 32) { 2670 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi); 2671 } else { 2672 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi); 2673 } 2674 #else 2675 if (TCG_TARGET_REG_BITS == 32) { 2676 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), 2677 TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2678 } else { 2679 tcg_gen_op3i_i64(opc, val, addr, oi); 2680 } 2681 #endif 2682 } 2683 2684 static void tcg_gen_req_mo(TCGBar type) 2685 { 2686 #ifdef TCG_GUEST_DEFAULT_MO 2687 type &= TCG_GUEST_DEFAULT_MO; 2688 #endif 2689 type &= ~TCG_TARGET_DEFAULT_MO; 2690 if (type) { 2691 tcg_gen_mb(type | TCG_BAR_SC); 2692 } 2693 } 2694 2695 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2696 { 2697 TCGMemOp orig_memop; 2698 2699 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2700 memop = tcg_canonicalize_memop(memop, 0, 0); 2701 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2702 addr, trace_mem_get_info(memop, 0)); 2703 2704 orig_memop = memop; 2705 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2706 memop &= ~MO_BSWAP; 2707 /* The bswap primitive requires zero-extended input. */ 2708 if ((memop & MO_SSIZE) == MO_SW) { 2709 memop &= ~MO_SIGN; 2710 } 2711 } 2712 2713 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx); 2714 2715 if ((orig_memop ^ memop) & MO_BSWAP) { 2716 switch (orig_memop & MO_SIZE) { 2717 case MO_16: 2718 tcg_gen_bswap16_i32(val, val); 2719 if (orig_memop & MO_SIGN) { 2720 tcg_gen_ext16s_i32(val, val); 2721 } 2722 break; 2723 case MO_32: 2724 tcg_gen_bswap32_i32(val, val); 2725 break; 2726 default: 2727 g_assert_not_reached(); 2728 } 2729 } 2730 } 2731 2732 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2733 { 2734 TCGv_i32 swap = NULL; 2735 2736 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2737 memop = tcg_canonicalize_memop(memop, 0, 1); 2738 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2739 addr, trace_mem_get_info(memop, 1)); 2740 2741 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2742 swap = tcg_temp_new_i32(); 2743 switch (memop & MO_SIZE) { 2744 case MO_16: 2745 tcg_gen_ext16u_i32(swap, val); 2746 tcg_gen_bswap16_i32(swap, swap); 2747 break; 2748 case MO_32: 2749 tcg_gen_bswap32_i32(swap, val); 2750 break; 2751 default: 2752 g_assert_not_reached(); 2753 } 2754 val = swap; 2755 memop &= ~MO_BSWAP; 2756 } 2757 2758 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); 2759 2760 if (swap) { 2761 tcg_temp_free_i32(swap); 2762 } 2763 } 2764 2765 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2766 { 2767 TCGMemOp orig_memop; 2768 2769 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 2770 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); 2771 if (memop & MO_SIGN) { 2772 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 2773 } else { 2774 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 2775 } 2776 return; 2777 } 2778 2779 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2780 memop = tcg_canonicalize_memop(memop, 1, 0); 2781 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2782 addr, trace_mem_get_info(memop, 0)); 2783 2784 orig_memop = memop; 2785 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2786 memop &= ~MO_BSWAP; 2787 /* The bswap primitive requires zero-extended input. */ 2788 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) { 2789 memop &= ~MO_SIGN; 2790 } 2791 } 2792 2793 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx); 2794 2795 if ((orig_memop ^ memop) & MO_BSWAP) { 2796 switch (orig_memop & MO_SIZE) { 2797 case MO_16: 2798 tcg_gen_bswap16_i64(val, val); 2799 if (orig_memop & MO_SIGN) { 2800 tcg_gen_ext16s_i64(val, val); 2801 } 2802 break; 2803 case MO_32: 2804 tcg_gen_bswap32_i64(val, val); 2805 if (orig_memop & MO_SIGN) { 2806 tcg_gen_ext32s_i64(val, val); 2807 } 2808 break; 2809 case MO_64: 2810 tcg_gen_bswap64_i64(val, val); 2811 break; 2812 default: 2813 g_assert_not_reached(); 2814 } 2815 } 2816 } 2817 2818 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2819 { 2820 TCGv_i64 swap = NULL; 2821 2822 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 2823 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); 2824 return; 2825 } 2826 2827 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2828 memop = tcg_canonicalize_memop(memop, 1, 1); 2829 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2830 addr, trace_mem_get_info(memop, 1)); 2831 2832 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2833 swap = tcg_temp_new_i64(); 2834 switch (memop & MO_SIZE) { 2835 case MO_16: 2836 tcg_gen_ext16u_i64(swap, val); 2837 tcg_gen_bswap16_i64(swap, swap); 2838 break; 2839 case MO_32: 2840 tcg_gen_ext32u_i64(swap, val); 2841 tcg_gen_bswap32_i64(swap, swap); 2842 break; 2843 case MO_64: 2844 tcg_gen_bswap64_i64(swap, val); 2845 break; 2846 default: 2847 g_assert_not_reached(); 2848 } 2849 val = swap; 2850 memop &= ~MO_BSWAP; 2851 } 2852 2853 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx); 2854 2855 if (swap) { 2856 tcg_temp_free_i64(swap); 2857 } 2858 } 2859 2860 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc) 2861 { 2862 switch (opc & MO_SSIZE) { 2863 case MO_SB: 2864 tcg_gen_ext8s_i32(ret, val); 2865 break; 2866 case MO_UB: 2867 tcg_gen_ext8u_i32(ret, val); 2868 break; 2869 case MO_SW: 2870 tcg_gen_ext16s_i32(ret, val); 2871 break; 2872 case MO_UW: 2873 tcg_gen_ext16u_i32(ret, val); 2874 break; 2875 default: 2876 tcg_gen_mov_i32(ret, val); 2877 break; 2878 } 2879 } 2880 2881 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc) 2882 { 2883 switch (opc & MO_SSIZE) { 2884 case MO_SB: 2885 tcg_gen_ext8s_i64(ret, val); 2886 break; 2887 case MO_UB: 2888 tcg_gen_ext8u_i64(ret, val); 2889 break; 2890 case MO_SW: 2891 tcg_gen_ext16s_i64(ret, val); 2892 break; 2893 case MO_UW: 2894 tcg_gen_ext16u_i64(ret, val); 2895 break; 2896 case MO_SL: 2897 tcg_gen_ext32s_i64(ret, val); 2898 break; 2899 case MO_UL: 2900 tcg_gen_ext32u_i64(ret, val); 2901 break; 2902 default: 2903 tcg_gen_mov_i64(ret, val); 2904 break; 2905 } 2906 } 2907 2908 #ifdef CONFIG_SOFTMMU 2909 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, 2910 TCGv_i32, TCGv_i32, TCGv_i32); 2911 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, 2912 TCGv_i64, TCGv_i64, TCGv_i32); 2913 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, 2914 TCGv_i32, TCGv_i32); 2915 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, 2916 TCGv_i64, TCGv_i32); 2917 #else 2918 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32); 2919 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64); 2920 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32); 2921 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64); 2922 #endif 2923 2924 #ifdef CONFIG_ATOMIC64 2925 # define WITH_ATOMIC64(X) X, 2926 #else 2927 # define WITH_ATOMIC64(X) 2928 #endif 2929 2930 static void * const table_cmpxchg[16] = { 2931 [MO_8] = gen_helper_atomic_cmpxchgb, 2932 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, 2933 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, 2934 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, 2935 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, 2936 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) 2937 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) 2938 }; 2939 2940 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, 2941 TCGv_i32 newv, TCGArg idx, TCGMemOp memop) 2942 { 2943 memop = tcg_canonicalize_memop(memop, 0, 0); 2944 2945 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { 2946 TCGv_i32 t1 = tcg_temp_new_i32(); 2947 TCGv_i32 t2 = tcg_temp_new_i32(); 2948 2949 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); 2950 2951 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 2952 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); 2953 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 2954 tcg_temp_free_i32(t2); 2955 2956 if (memop & MO_SIGN) { 2957 tcg_gen_ext_i32(retv, t1, memop); 2958 } else { 2959 tcg_gen_mov_i32(retv, t1); 2960 } 2961 tcg_temp_free_i32(t1); 2962 } else { 2963 gen_atomic_cx_i32 gen; 2964 2965 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 2966 tcg_debug_assert(gen != NULL); 2967 2968 #ifdef CONFIG_SOFTMMU 2969 { 2970 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 2971 gen(retv, cpu_env, addr, cmpv, newv, oi); 2972 tcg_temp_free_i32(oi); 2973 } 2974 #else 2975 gen(retv, cpu_env, addr, cmpv, newv); 2976 #endif 2977 2978 if (memop & MO_SIGN) { 2979 tcg_gen_ext_i32(retv, retv, memop); 2980 } 2981 } 2982 } 2983 2984 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, 2985 TCGv_i64 newv, TCGArg idx, TCGMemOp memop) 2986 { 2987 memop = tcg_canonicalize_memop(memop, 1, 0); 2988 2989 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { 2990 TCGv_i64 t1 = tcg_temp_new_i64(); 2991 TCGv_i64 t2 = tcg_temp_new_i64(); 2992 2993 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); 2994 2995 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 2996 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); 2997 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 2998 tcg_temp_free_i64(t2); 2999 3000 if (memop & MO_SIGN) { 3001 tcg_gen_ext_i64(retv, t1, memop); 3002 } else { 3003 tcg_gen_mov_i64(retv, t1); 3004 } 3005 tcg_temp_free_i64(t1); 3006 } else if ((memop & MO_SIZE) == MO_64) { 3007 #ifdef CONFIG_ATOMIC64 3008 gen_atomic_cx_i64 gen; 3009 3010 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 3011 tcg_debug_assert(gen != NULL); 3012 3013 #ifdef CONFIG_SOFTMMU 3014 { 3015 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx)); 3016 gen(retv, cpu_env, addr, cmpv, newv, oi); 3017 tcg_temp_free_i32(oi); 3018 } 3019 #else 3020 gen(retv, cpu_env, addr, cmpv, newv); 3021 #endif 3022 #else 3023 gen_helper_exit_atomic(cpu_env); 3024 /* Produce a result, so that we have a well-formed opcode stream 3025 with respect to uses of the result in the (dead) code following. */ 3026 tcg_gen_movi_i64(retv, 0); 3027 #endif /* CONFIG_ATOMIC64 */ 3028 } else { 3029 TCGv_i32 c32 = tcg_temp_new_i32(); 3030 TCGv_i32 n32 = tcg_temp_new_i32(); 3031 TCGv_i32 r32 = tcg_temp_new_i32(); 3032 3033 tcg_gen_extrl_i64_i32(c32, cmpv); 3034 tcg_gen_extrl_i64_i32(n32, newv); 3035 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN); 3036 tcg_temp_free_i32(c32); 3037 tcg_temp_free_i32(n32); 3038 3039 tcg_gen_extu_i32_i64(retv, r32); 3040 tcg_temp_free_i32(r32); 3041 3042 if (memop & MO_SIGN) { 3043 tcg_gen_ext_i64(retv, retv, memop); 3044 } 3045 } 3046 } 3047 3048 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 3049 TCGArg idx, TCGMemOp memop, bool new_val, 3050 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) 3051 { 3052 TCGv_i32 t1 = tcg_temp_new_i32(); 3053 TCGv_i32 t2 = tcg_temp_new_i32(); 3054 3055 memop = tcg_canonicalize_memop(memop, 0, 0); 3056 3057 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 3058 gen(t2, t1, val); 3059 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 3060 3061 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); 3062 tcg_temp_free_i32(t1); 3063 tcg_temp_free_i32(t2); 3064 } 3065 3066 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 3067 TCGArg idx, TCGMemOp memop, void * const table[]) 3068 { 3069 gen_atomic_op_i32 gen; 3070 3071 memop = tcg_canonicalize_memop(memop, 0, 0); 3072 3073 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 3074 tcg_debug_assert(gen != NULL); 3075 3076 #ifdef CONFIG_SOFTMMU 3077 { 3078 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 3079 gen(ret, cpu_env, addr, val, oi); 3080 tcg_temp_free_i32(oi); 3081 } 3082 #else 3083 gen(ret, cpu_env, addr, val); 3084 #endif 3085 3086 if (memop & MO_SIGN) { 3087 tcg_gen_ext_i32(ret, ret, memop); 3088 } 3089 } 3090 3091 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 3092 TCGArg idx, TCGMemOp memop, bool new_val, 3093 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) 3094 { 3095 TCGv_i64 t1 = tcg_temp_new_i64(); 3096 TCGv_i64 t2 = tcg_temp_new_i64(); 3097 3098 memop = tcg_canonicalize_memop(memop, 1, 0); 3099 3100 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 3101 gen(t2, t1, val); 3102 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 3103 3104 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); 3105 tcg_temp_free_i64(t1); 3106 tcg_temp_free_i64(t2); 3107 } 3108 3109 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 3110 TCGArg idx, TCGMemOp memop, void * const table[]) 3111 { 3112 memop = tcg_canonicalize_memop(memop, 1, 0); 3113 3114 if ((memop & MO_SIZE) == MO_64) { 3115 #ifdef CONFIG_ATOMIC64 3116 gen_atomic_op_i64 gen; 3117 3118 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 3119 tcg_debug_assert(gen != NULL); 3120 3121 #ifdef CONFIG_SOFTMMU 3122 { 3123 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 3124 gen(ret, cpu_env, addr, val, oi); 3125 tcg_temp_free_i32(oi); 3126 } 3127 #else 3128 gen(ret, cpu_env, addr, val); 3129 #endif 3130 #else 3131 gen_helper_exit_atomic(cpu_env); 3132 /* Produce a result, so that we have a well-formed opcode stream 3133 with respect to uses of the result in the (dead) code following. */ 3134 tcg_gen_movi_i64(ret, 0); 3135 #endif /* CONFIG_ATOMIC64 */ 3136 } else { 3137 TCGv_i32 v32 = tcg_temp_new_i32(); 3138 TCGv_i32 r32 = tcg_temp_new_i32(); 3139 3140 tcg_gen_extrl_i64_i32(v32, val); 3141 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); 3142 tcg_temp_free_i32(v32); 3143 3144 tcg_gen_extu_i32_i64(ret, r32); 3145 tcg_temp_free_i32(r32); 3146 3147 if (memop & MO_SIGN) { 3148 tcg_gen_ext_i64(ret, ret, memop); 3149 } 3150 } 3151 } 3152 3153 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ 3154 static void * const table_##NAME[16] = { \ 3155 [MO_8] = gen_helper_atomic_##NAME##b, \ 3156 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ 3157 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ 3158 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ 3159 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ 3160 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ 3161 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ 3162 }; \ 3163 void tcg_gen_atomic_##NAME##_i32 \ 3164 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \ 3165 { \ 3166 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ 3167 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ 3168 } else { \ 3169 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ 3170 tcg_gen_##OP##_i32); \ 3171 } \ 3172 } \ 3173 void tcg_gen_atomic_##NAME##_i64 \ 3174 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \ 3175 { \ 3176 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ 3177 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ 3178 } else { \ 3179 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ 3180 tcg_gen_##OP##_i64); \ 3181 } \ 3182 } 3183 3184 GEN_ATOMIC_HELPER(fetch_add, add, 0) 3185 GEN_ATOMIC_HELPER(fetch_and, and, 0) 3186 GEN_ATOMIC_HELPER(fetch_or, or, 0) 3187 GEN_ATOMIC_HELPER(fetch_xor, xor, 0) 3188 GEN_ATOMIC_HELPER(fetch_smin, smin, 0) 3189 GEN_ATOMIC_HELPER(fetch_umin, umin, 0) 3190 GEN_ATOMIC_HELPER(fetch_smax, smax, 0) 3191 GEN_ATOMIC_HELPER(fetch_umax, umax, 0) 3192 3193 GEN_ATOMIC_HELPER(add_fetch, add, 1) 3194 GEN_ATOMIC_HELPER(and_fetch, and, 1) 3195 GEN_ATOMIC_HELPER(or_fetch, or, 1) 3196 GEN_ATOMIC_HELPER(xor_fetch, xor, 1) 3197 GEN_ATOMIC_HELPER(smin_fetch, smin, 1) 3198 GEN_ATOMIC_HELPER(umin_fetch, umin, 1) 3199 GEN_ATOMIC_HELPER(smax_fetch, smax, 1) 3200 GEN_ATOMIC_HELPER(umax_fetch, umax, 1) 3201 3202 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) 3203 { 3204 tcg_gen_mov_i32(r, b); 3205 } 3206 3207 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) 3208 { 3209 tcg_gen_mov_i64(r, b); 3210 } 3211 3212 GEN_ATOMIC_HELPER(xchg, mov2, 0) 3213 3214 #undef GEN_ATOMIC_HELPER 3215