1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "qemu-common.h" 27 #include "cpu.h" 28 #include "exec/exec-all.h" 29 #include "tcg.h" 30 #include "tcg-op.h" 31 #include "tcg-mo.h" 32 #include "trace-tcg.h" 33 #include "trace/mem.h" 34 35 /* Reduce the number of ifdefs below. This assumes that all uses of 36 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that 37 the compiler can eliminate. */ 38 #if TCG_TARGET_REG_BITS == 64 39 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); 40 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); 41 #define TCGV_LOW TCGV_LOW_link_error 42 #define TCGV_HIGH TCGV_HIGH_link_error 43 #endif 44 45 void tcg_gen_op1(TCGOpcode opc, TCGArg a1) 46 { 47 TCGOp *op = tcg_emit_op(opc); 48 op->args[0] = a1; 49 } 50 51 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2) 52 { 53 TCGOp *op = tcg_emit_op(opc); 54 op->args[0] = a1; 55 op->args[1] = a2; 56 } 57 58 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) 59 { 60 TCGOp *op = tcg_emit_op(opc); 61 op->args[0] = a1; 62 op->args[1] = a2; 63 op->args[2] = a3; 64 } 65 66 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4) 67 { 68 TCGOp *op = tcg_emit_op(opc); 69 op->args[0] = a1; 70 op->args[1] = a2; 71 op->args[2] = a3; 72 op->args[3] = a4; 73 } 74 75 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 76 TCGArg a4, TCGArg a5) 77 { 78 TCGOp *op = tcg_emit_op(opc); 79 op->args[0] = a1; 80 op->args[1] = a2; 81 op->args[2] = a3; 82 op->args[3] = a4; 83 op->args[4] = a5; 84 } 85 86 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 87 TCGArg a4, TCGArg a5, TCGArg a6) 88 { 89 TCGOp *op = tcg_emit_op(opc); 90 op->args[0] = a1; 91 op->args[1] = a2; 92 op->args[2] = a3; 93 op->args[3] = a4; 94 op->args[4] = a5; 95 op->args[5] = a6; 96 } 97 98 void tcg_gen_mb(TCGBar mb_type) 99 { 100 if (tcg_ctx->tb_cflags & CF_PARALLEL) { 101 tcg_gen_op1(INDEX_op_mb, mb_type); 102 } 103 } 104 105 /* 32 bit ops */ 106 107 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 108 { 109 /* some cases can be optimized here */ 110 if (arg2 == 0) { 111 tcg_gen_mov_i32(ret, arg1); 112 } else { 113 TCGv_i32 t0 = tcg_const_i32(arg2); 114 tcg_gen_add_i32(ret, arg1, t0); 115 tcg_temp_free_i32(t0); 116 } 117 } 118 119 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 120 { 121 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) { 122 /* Don't recurse with tcg_gen_neg_i32. */ 123 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2); 124 } else { 125 TCGv_i32 t0 = tcg_const_i32(arg1); 126 tcg_gen_sub_i32(ret, t0, arg2); 127 tcg_temp_free_i32(t0); 128 } 129 } 130 131 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 132 { 133 /* some cases can be optimized here */ 134 if (arg2 == 0) { 135 tcg_gen_mov_i32(ret, arg1); 136 } else { 137 TCGv_i32 t0 = tcg_const_i32(arg2); 138 tcg_gen_sub_i32(ret, arg1, t0); 139 tcg_temp_free_i32(t0); 140 } 141 } 142 143 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 144 { 145 TCGv_i32 t0; 146 /* Some cases can be optimized here. */ 147 switch (arg2) { 148 case 0: 149 tcg_gen_movi_i32(ret, 0); 150 return; 151 case -1: 152 tcg_gen_mov_i32(ret, arg1); 153 return; 154 case 0xff: 155 /* Don't recurse with tcg_gen_ext8u_i32. */ 156 if (TCG_TARGET_HAS_ext8u_i32) { 157 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1); 158 return; 159 } 160 break; 161 case 0xffff: 162 if (TCG_TARGET_HAS_ext16u_i32) { 163 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1); 164 return; 165 } 166 break; 167 } 168 t0 = tcg_const_i32(arg2); 169 tcg_gen_and_i32(ret, arg1, t0); 170 tcg_temp_free_i32(t0); 171 } 172 173 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 174 { 175 /* Some cases can be optimized here. */ 176 if (arg2 == -1) { 177 tcg_gen_movi_i32(ret, -1); 178 } else if (arg2 == 0) { 179 tcg_gen_mov_i32(ret, arg1); 180 } else { 181 TCGv_i32 t0 = tcg_const_i32(arg2); 182 tcg_gen_or_i32(ret, arg1, t0); 183 tcg_temp_free_i32(t0); 184 } 185 } 186 187 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 188 { 189 /* Some cases can be optimized here. */ 190 if (arg2 == 0) { 191 tcg_gen_mov_i32(ret, arg1); 192 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) { 193 /* Don't recurse with tcg_gen_not_i32. */ 194 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1); 195 } else { 196 TCGv_i32 t0 = tcg_const_i32(arg2); 197 tcg_gen_xor_i32(ret, arg1, t0); 198 tcg_temp_free_i32(t0); 199 } 200 } 201 202 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 203 { 204 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 205 if (arg2 == 0) { 206 tcg_gen_mov_i32(ret, arg1); 207 } else { 208 TCGv_i32 t0 = tcg_const_i32(arg2); 209 tcg_gen_shl_i32(ret, arg1, t0); 210 tcg_temp_free_i32(t0); 211 } 212 } 213 214 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 215 { 216 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 217 if (arg2 == 0) { 218 tcg_gen_mov_i32(ret, arg1); 219 } else { 220 TCGv_i32 t0 = tcg_const_i32(arg2); 221 tcg_gen_shr_i32(ret, arg1, t0); 222 tcg_temp_free_i32(t0); 223 } 224 } 225 226 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 227 { 228 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 229 if (arg2 == 0) { 230 tcg_gen_mov_i32(ret, arg1); 231 } else { 232 TCGv_i32 t0 = tcg_const_i32(arg2); 233 tcg_gen_sar_i32(ret, arg1, t0); 234 tcg_temp_free_i32(t0); 235 } 236 } 237 238 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 239 { 240 if (cond == TCG_COND_ALWAYS) { 241 tcg_gen_br(l); 242 } else if (cond != TCG_COND_NEVER) { 243 l->refs++; 244 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l)); 245 } 246 } 247 248 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 249 { 250 if (cond == TCG_COND_ALWAYS) { 251 tcg_gen_br(l); 252 } else if (cond != TCG_COND_NEVER) { 253 TCGv_i32 t0 = tcg_const_i32(arg2); 254 tcg_gen_brcond_i32(cond, arg1, t0, l); 255 tcg_temp_free_i32(t0); 256 } 257 } 258 259 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 260 TCGv_i32 arg1, TCGv_i32 arg2) 261 { 262 if (cond == TCG_COND_ALWAYS) { 263 tcg_gen_movi_i32(ret, 1); 264 } else if (cond == TCG_COND_NEVER) { 265 tcg_gen_movi_i32(ret, 0); 266 } else { 267 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 268 } 269 } 270 271 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 272 TCGv_i32 arg1, int32_t arg2) 273 { 274 TCGv_i32 t0 = tcg_const_i32(arg2); 275 tcg_gen_setcond_i32(cond, ret, arg1, t0); 276 tcg_temp_free_i32(t0); 277 } 278 279 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 280 { 281 if (arg2 == 0) { 282 tcg_gen_movi_i32(ret, 0); 283 } else if (is_power_of_2(arg2)) { 284 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 285 } else { 286 TCGv_i32 t0 = tcg_const_i32(arg2); 287 tcg_gen_mul_i32(ret, arg1, t0); 288 tcg_temp_free_i32(t0); 289 } 290 } 291 292 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 293 { 294 if (TCG_TARGET_HAS_div_i32) { 295 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2); 296 } else if (TCG_TARGET_HAS_div2_i32) { 297 TCGv_i32 t0 = tcg_temp_new_i32(); 298 tcg_gen_sari_i32(t0, arg1, 31); 299 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2); 300 tcg_temp_free_i32(t0); 301 } else { 302 gen_helper_div_i32(ret, arg1, arg2); 303 } 304 } 305 306 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 307 { 308 if (TCG_TARGET_HAS_rem_i32) { 309 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2); 310 } else if (TCG_TARGET_HAS_div_i32) { 311 TCGv_i32 t0 = tcg_temp_new_i32(); 312 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2); 313 tcg_gen_mul_i32(t0, t0, arg2); 314 tcg_gen_sub_i32(ret, arg1, t0); 315 tcg_temp_free_i32(t0); 316 } else if (TCG_TARGET_HAS_div2_i32) { 317 TCGv_i32 t0 = tcg_temp_new_i32(); 318 tcg_gen_sari_i32(t0, arg1, 31); 319 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2); 320 tcg_temp_free_i32(t0); 321 } else { 322 gen_helper_rem_i32(ret, arg1, arg2); 323 } 324 } 325 326 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 327 { 328 if (TCG_TARGET_HAS_div_i32) { 329 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2); 330 } else if (TCG_TARGET_HAS_div2_i32) { 331 TCGv_i32 t0 = tcg_temp_new_i32(); 332 tcg_gen_movi_i32(t0, 0); 333 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2); 334 tcg_temp_free_i32(t0); 335 } else { 336 gen_helper_divu_i32(ret, arg1, arg2); 337 } 338 } 339 340 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 341 { 342 if (TCG_TARGET_HAS_rem_i32) { 343 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2); 344 } else if (TCG_TARGET_HAS_div_i32) { 345 TCGv_i32 t0 = tcg_temp_new_i32(); 346 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2); 347 tcg_gen_mul_i32(t0, t0, arg2); 348 tcg_gen_sub_i32(ret, arg1, t0); 349 tcg_temp_free_i32(t0); 350 } else if (TCG_TARGET_HAS_div2_i32) { 351 TCGv_i32 t0 = tcg_temp_new_i32(); 352 tcg_gen_movi_i32(t0, 0); 353 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2); 354 tcg_temp_free_i32(t0); 355 } else { 356 gen_helper_remu_i32(ret, arg1, arg2); 357 } 358 } 359 360 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 361 { 362 if (TCG_TARGET_HAS_andc_i32) { 363 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2); 364 } else { 365 TCGv_i32 t0 = tcg_temp_new_i32(); 366 tcg_gen_not_i32(t0, arg2); 367 tcg_gen_and_i32(ret, arg1, t0); 368 tcg_temp_free_i32(t0); 369 } 370 } 371 372 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 373 { 374 if (TCG_TARGET_HAS_eqv_i32) { 375 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2); 376 } else { 377 tcg_gen_xor_i32(ret, arg1, arg2); 378 tcg_gen_not_i32(ret, ret); 379 } 380 } 381 382 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 383 { 384 if (TCG_TARGET_HAS_nand_i32) { 385 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2); 386 } else { 387 tcg_gen_and_i32(ret, arg1, arg2); 388 tcg_gen_not_i32(ret, ret); 389 } 390 } 391 392 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 393 { 394 if (TCG_TARGET_HAS_nor_i32) { 395 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2); 396 } else { 397 tcg_gen_or_i32(ret, arg1, arg2); 398 tcg_gen_not_i32(ret, ret); 399 } 400 } 401 402 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 403 { 404 if (TCG_TARGET_HAS_orc_i32) { 405 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2); 406 } else { 407 TCGv_i32 t0 = tcg_temp_new_i32(); 408 tcg_gen_not_i32(t0, arg2); 409 tcg_gen_or_i32(ret, arg1, t0); 410 tcg_temp_free_i32(t0); 411 } 412 } 413 414 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 415 { 416 if (TCG_TARGET_HAS_clz_i32) { 417 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2); 418 } else if (TCG_TARGET_HAS_clz_i64) { 419 TCGv_i64 t1 = tcg_temp_new_i64(); 420 TCGv_i64 t2 = tcg_temp_new_i64(); 421 tcg_gen_extu_i32_i64(t1, arg1); 422 tcg_gen_extu_i32_i64(t2, arg2); 423 tcg_gen_addi_i64(t2, t2, 32); 424 tcg_gen_clz_i64(t1, t1, t2); 425 tcg_gen_extrl_i64_i32(ret, t1); 426 tcg_temp_free_i64(t1); 427 tcg_temp_free_i64(t2); 428 tcg_gen_subi_i32(ret, ret, 32); 429 } else { 430 gen_helper_clz_i32(ret, arg1, arg2); 431 } 432 } 433 434 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 435 { 436 TCGv_i32 t = tcg_const_i32(arg2); 437 tcg_gen_clz_i32(ret, arg1, t); 438 tcg_temp_free_i32(t); 439 } 440 441 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 442 { 443 if (TCG_TARGET_HAS_ctz_i32) { 444 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); 445 } else if (TCG_TARGET_HAS_ctz_i64) { 446 TCGv_i64 t1 = tcg_temp_new_i64(); 447 TCGv_i64 t2 = tcg_temp_new_i64(); 448 tcg_gen_extu_i32_i64(t1, arg1); 449 tcg_gen_extu_i32_i64(t2, arg2); 450 tcg_gen_ctz_i64(t1, t1, t2); 451 tcg_gen_extrl_i64_i32(ret, t1); 452 tcg_temp_free_i64(t1); 453 tcg_temp_free_i64(t2); 454 } else if (TCG_TARGET_HAS_ctpop_i32 455 || TCG_TARGET_HAS_ctpop_i64 456 || TCG_TARGET_HAS_clz_i32 457 || TCG_TARGET_HAS_clz_i64) { 458 TCGv_i32 z, t = tcg_temp_new_i32(); 459 460 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 461 tcg_gen_subi_i32(t, arg1, 1); 462 tcg_gen_andc_i32(t, t, arg1); 463 tcg_gen_ctpop_i32(t, t); 464 } else { 465 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */ 466 tcg_gen_neg_i32(t, arg1); 467 tcg_gen_and_i32(t, t, arg1); 468 tcg_gen_clzi_i32(t, t, 32); 469 tcg_gen_xori_i32(t, t, 31); 470 } 471 z = tcg_const_i32(0); 472 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 473 tcg_temp_free_i32(t); 474 tcg_temp_free_i32(z); 475 } else { 476 gen_helper_ctz_i32(ret, arg1, arg2); 477 } 478 } 479 480 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 481 { 482 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 483 /* This equivalence has the advantage of not requiring a fixup. */ 484 TCGv_i32 t = tcg_temp_new_i32(); 485 tcg_gen_subi_i32(t, arg1, 1); 486 tcg_gen_andc_i32(t, t, arg1); 487 tcg_gen_ctpop_i32(ret, t); 488 tcg_temp_free_i32(t); 489 } else { 490 TCGv_i32 t = tcg_const_i32(arg2); 491 tcg_gen_ctz_i32(ret, arg1, t); 492 tcg_temp_free_i32(t); 493 } 494 } 495 496 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 497 { 498 if (TCG_TARGET_HAS_clz_i32) { 499 TCGv_i32 t = tcg_temp_new_i32(); 500 tcg_gen_sari_i32(t, arg, 31); 501 tcg_gen_xor_i32(t, t, arg); 502 tcg_gen_clzi_i32(t, t, 32); 503 tcg_gen_subi_i32(ret, t, 1); 504 tcg_temp_free_i32(t); 505 } else { 506 gen_helper_clrsb_i32(ret, arg); 507 } 508 } 509 510 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 511 { 512 if (TCG_TARGET_HAS_ctpop_i32) { 513 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 514 } else if (TCG_TARGET_HAS_ctpop_i64) { 515 TCGv_i64 t = tcg_temp_new_i64(); 516 tcg_gen_extu_i32_i64(t, arg1); 517 tcg_gen_ctpop_i64(t, t); 518 tcg_gen_extrl_i64_i32(ret, t); 519 tcg_temp_free_i64(t); 520 } else { 521 gen_helper_ctpop_i32(ret, arg1); 522 } 523 } 524 525 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 526 { 527 if (TCG_TARGET_HAS_rot_i32) { 528 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2); 529 } else { 530 TCGv_i32 t0, t1; 531 532 t0 = tcg_temp_new_i32(); 533 t1 = tcg_temp_new_i32(); 534 tcg_gen_shl_i32(t0, arg1, arg2); 535 tcg_gen_subfi_i32(t1, 32, arg2); 536 tcg_gen_shr_i32(t1, arg1, t1); 537 tcg_gen_or_i32(ret, t0, t1); 538 tcg_temp_free_i32(t0); 539 tcg_temp_free_i32(t1); 540 } 541 } 542 543 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 544 { 545 tcg_debug_assert(arg2 < 32); 546 /* some cases can be optimized here */ 547 if (arg2 == 0) { 548 tcg_gen_mov_i32(ret, arg1); 549 } else if (TCG_TARGET_HAS_rot_i32) { 550 TCGv_i32 t0 = tcg_const_i32(arg2); 551 tcg_gen_rotl_i32(ret, arg1, t0); 552 tcg_temp_free_i32(t0); 553 } else { 554 TCGv_i32 t0, t1; 555 t0 = tcg_temp_new_i32(); 556 t1 = tcg_temp_new_i32(); 557 tcg_gen_shli_i32(t0, arg1, arg2); 558 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 559 tcg_gen_or_i32(ret, t0, t1); 560 tcg_temp_free_i32(t0); 561 tcg_temp_free_i32(t1); 562 } 563 } 564 565 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 566 { 567 if (TCG_TARGET_HAS_rot_i32) { 568 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2); 569 } else { 570 TCGv_i32 t0, t1; 571 572 t0 = tcg_temp_new_i32(); 573 t1 = tcg_temp_new_i32(); 574 tcg_gen_shr_i32(t0, arg1, arg2); 575 tcg_gen_subfi_i32(t1, 32, arg2); 576 tcg_gen_shl_i32(t1, arg1, t1); 577 tcg_gen_or_i32(ret, t0, t1); 578 tcg_temp_free_i32(t0); 579 tcg_temp_free_i32(t1); 580 } 581 } 582 583 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 584 { 585 tcg_debug_assert(arg2 < 32); 586 /* some cases can be optimized here */ 587 if (arg2 == 0) { 588 tcg_gen_mov_i32(ret, arg1); 589 } else { 590 tcg_gen_rotli_i32(ret, arg1, 32 - arg2); 591 } 592 } 593 594 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 595 unsigned int ofs, unsigned int len) 596 { 597 uint32_t mask; 598 TCGv_i32 t1; 599 600 tcg_debug_assert(ofs < 32); 601 tcg_debug_assert(len > 0); 602 tcg_debug_assert(len <= 32); 603 tcg_debug_assert(ofs + len <= 32); 604 605 if (len == 32) { 606 tcg_gen_mov_i32(ret, arg2); 607 return; 608 } 609 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 610 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 611 return; 612 } 613 614 mask = (1u << len) - 1; 615 t1 = tcg_temp_new_i32(); 616 617 if (ofs + len < 32) { 618 tcg_gen_andi_i32(t1, arg2, mask); 619 tcg_gen_shli_i32(t1, t1, ofs); 620 } else { 621 tcg_gen_shli_i32(t1, arg2, ofs); 622 } 623 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 624 tcg_gen_or_i32(ret, ret, t1); 625 626 tcg_temp_free_i32(t1); 627 } 628 629 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 630 unsigned int ofs, unsigned int len) 631 { 632 tcg_debug_assert(ofs < 32); 633 tcg_debug_assert(len > 0); 634 tcg_debug_assert(len <= 32); 635 tcg_debug_assert(ofs + len <= 32); 636 637 if (ofs + len == 32) { 638 tcg_gen_shli_i32(ret, arg, ofs); 639 } else if (ofs == 0) { 640 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 641 } else if (TCG_TARGET_HAS_deposit_i32 642 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 643 TCGv_i32 zero = tcg_const_i32(0); 644 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 645 tcg_temp_free_i32(zero); 646 } else { 647 /* To help two-operand hosts we prefer to zero-extend first, 648 which allows ARG to stay live. */ 649 switch (len) { 650 case 16: 651 if (TCG_TARGET_HAS_ext16u_i32) { 652 tcg_gen_ext16u_i32(ret, arg); 653 tcg_gen_shli_i32(ret, ret, ofs); 654 return; 655 } 656 break; 657 case 8: 658 if (TCG_TARGET_HAS_ext8u_i32) { 659 tcg_gen_ext8u_i32(ret, arg); 660 tcg_gen_shli_i32(ret, ret, ofs); 661 return; 662 } 663 break; 664 } 665 /* Otherwise prefer zero-extension over AND for code size. */ 666 switch (ofs + len) { 667 case 16: 668 if (TCG_TARGET_HAS_ext16u_i32) { 669 tcg_gen_shli_i32(ret, arg, ofs); 670 tcg_gen_ext16u_i32(ret, ret); 671 return; 672 } 673 break; 674 case 8: 675 if (TCG_TARGET_HAS_ext8u_i32) { 676 tcg_gen_shli_i32(ret, arg, ofs); 677 tcg_gen_ext8u_i32(ret, ret); 678 return; 679 } 680 break; 681 } 682 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 683 tcg_gen_shli_i32(ret, ret, ofs); 684 } 685 } 686 687 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 688 unsigned int ofs, unsigned int len) 689 { 690 tcg_debug_assert(ofs < 32); 691 tcg_debug_assert(len > 0); 692 tcg_debug_assert(len <= 32); 693 tcg_debug_assert(ofs + len <= 32); 694 695 /* Canonicalize certain special cases, even if extract is supported. */ 696 if (ofs + len == 32) { 697 tcg_gen_shri_i32(ret, arg, 32 - len); 698 return; 699 } 700 if (ofs == 0) { 701 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 702 return; 703 } 704 705 if (TCG_TARGET_HAS_extract_i32 706 && TCG_TARGET_extract_i32_valid(ofs, len)) { 707 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 708 return; 709 } 710 711 /* Assume that zero-extension, if available, is cheaper than a shift. */ 712 switch (ofs + len) { 713 case 16: 714 if (TCG_TARGET_HAS_ext16u_i32) { 715 tcg_gen_ext16u_i32(ret, arg); 716 tcg_gen_shri_i32(ret, ret, ofs); 717 return; 718 } 719 break; 720 case 8: 721 if (TCG_TARGET_HAS_ext8u_i32) { 722 tcg_gen_ext8u_i32(ret, arg); 723 tcg_gen_shri_i32(ret, ret, ofs); 724 return; 725 } 726 break; 727 } 728 729 /* ??? Ideally we'd know what values are available for immediate AND. 730 Assume that 8 bits are available, plus the special case of 16, 731 so that we get ext8u, ext16u. */ 732 switch (len) { 733 case 1 ... 8: case 16: 734 tcg_gen_shri_i32(ret, arg, ofs); 735 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 736 break; 737 default: 738 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 739 tcg_gen_shri_i32(ret, ret, 32 - len); 740 break; 741 } 742 } 743 744 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 745 unsigned int ofs, unsigned int len) 746 { 747 tcg_debug_assert(ofs < 32); 748 tcg_debug_assert(len > 0); 749 tcg_debug_assert(len <= 32); 750 tcg_debug_assert(ofs + len <= 32); 751 752 /* Canonicalize certain special cases, even if extract is supported. */ 753 if (ofs + len == 32) { 754 tcg_gen_sari_i32(ret, arg, 32 - len); 755 return; 756 } 757 if (ofs == 0) { 758 switch (len) { 759 case 16: 760 tcg_gen_ext16s_i32(ret, arg); 761 return; 762 case 8: 763 tcg_gen_ext8s_i32(ret, arg); 764 return; 765 } 766 } 767 768 if (TCG_TARGET_HAS_sextract_i32 769 && TCG_TARGET_extract_i32_valid(ofs, len)) { 770 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 771 return; 772 } 773 774 /* Assume that sign-extension, if available, is cheaper than a shift. */ 775 switch (ofs + len) { 776 case 16: 777 if (TCG_TARGET_HAS_ext16s_i32) { 778 tcg_gen_ext16s_i32(ret, arg); 779 tcg_gen_sari_i32(ret, ret, ofs); 780 return; 781 } 782 break; 783 case 8: 784 if (TCG_TARGET_HAS_ext8s_i32) { 785 tcg_gen_ext8s_i32(ret, arg); 786 tcg_gen_sari_i32(ret, ret, ofs); 787 return; 788 } 789 break; 790 } 791 switch (len) { 792 case 16: 793 if (TCG_TARGET_HAS_ext16s_i32) { 794 tcg_gen_shri_i32(ret, arg, ofs); 795 tcg_gen_ext16s_i32(ret, ret); 796 return; 797 } 798 break; 799 case 8: 800 if (TCG_TARGET_HAS_ext8s_i32) { 801 tcg_gen_shri_i32(ret, arg, ofs); 802 tcg_gen_ext8s_i32(ret, ret); 803 return; 804 } 805 break; 806 } 807 808 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 809 tcg_gen_sari_i32(ret, ret, 32 - len); 810 } 811 812 /* 813 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs. 814 * Unlike tcg_gen_extract_i32 above, len is fixed at 32. 815 */ 816 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah, 817 unsigned int ofs) 818 { 819 tcg_debug_assert(ofs <= 32); 820 if (ofs == 0) { 821 tcg_gen_mov_i32(ret, al); 822 } else if (ofs == 32) { 823 tcg_gen_mov_i32(ret, ah); 824 } else if (al == ah) { 825 tcg_gen_rotri_i32(ret, al, ofs); 826 } else if (TCG_TARGET_HAS_extract2_i32) { 827 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs); 828 } else { 829 TCGv_i32 t0 = tcg_temp_new_i32(); 830 tcg_gen_shri_i32(t0, al, ofs); 831 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs); 832 tcg_temp_free_i32(t0); 833 } 834 } 835 836 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 837 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 838 { 839 if (cond == TCG_COND_ALWAYS) { 840 tcg_gen_mov_i32(ret, v1); 841 } else if (cond == TCG_COND_NEVER) { 842 tcg_gen_mov_i32(ret, v2); 843 } else if (TCG_TARGET_HAS_movcond_i32) { 844 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 845 } else { 846 TCGv_i32 t0 = tcg_temp_new_i32(); 847 TCGv_i32 t1 = tcg_temp_new_i32(); 848 tcg_gen_setcond_i32(cond, t0, c1, c2); 849 tcg_gen_neg_i32(t0, t0); 850 tcg_gen_and_i32(t1, v1, t0); 851 tcg_gen_andc_i32(ret, v2, t0); 852 tcg_gen_or_i32(ret, ret, t1); 853 tcg_temp_free_i32(t0); 854 tcg_temp_free_i32(t1); 855 } 856 } 857 858 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 859 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 860 { 861 if (TCG_TARGET_HAS_add2_i32) { 862 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 863 } else { 864 TCGv_i64 t0 = tcg_temp_new_i64(); 865 TCGv_i64 t1 = tcg_temp_new_i64(); 866 tcg_gen_concat_i32_i64(t0, al, ah); 867 tcg_gen_concat_i32_i64(t1, bl, bh); 868 tcg_gen_add_i64(t0, t0, t1); 869 tcg_gen_extr_i64_i32(rl, rh, t0); 870 tcg_temp_free_i64(t0); 871 tcg_temp_free_i64(t1); 872 } 873 } 874 875 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 876 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 877 { 878 if (TCG_TARGET_HAS_sub2_i32) { 879 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 880 } else { 881 TCGv_i64 t0 = tcg_temp_new_i64(); 882 TCGv_i64 t1 = tcg_temp_new_i64(); 883 tcg_gen_concat_i32_i64(t0, al, ah); 884 tcg_gen_concat_i32_i64(t1, bl, bh); 885 tcg_gen_sub_i64(t0, t0, t1); 886 tcg_gen_extr_i64_i32(rl, rh, t0); 887 tcg_temp_free_i64(t0); 888 tcg_temp_free_i64(t1); 889 } 890 } 891 892 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 893 { 894 if (TCG_TARGET_HAS_mulu2_i32) { 895 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 896 } else if (TCG_TARGET_HAS_muluh_i32) { 897 TCGv_i32 t = tcg_temp_new_i32(); 898 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 899 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2); 900 tcg_gen_mov_i32(rl, t); 901 tcg_temp_free_i32(t); 902 } else { 903 TCGv_i64 t0 = tcg_temp_new_i64(); 904 TCGv_i64 t1 = tcg_temp_new_i64(); 905 tcg_gen_extu_i32_i64(t0, arg1); 906 tcg_gen_extu_i32_i64(t1, arg2); 907 tcg_gen_mul_i64(t0, t0, t1); 908 tcg_gen_extr_i64_i32(rl, rh, t0); 909 tcg_temp_free_i64(t0); 910 tcg_temp_free_i64(t1); 911 } 912 } 913 914 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 915 { 916 if (TCG_TARGET_HAS_muls2_i32) { 917 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 918 } else if (TCG_TARGET_HAS_mulsh_i32) { 919 TCGv_i32 t = tcg_temp_new_i32(); 920 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 921 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2); 922 tcg_gen_mov_i32(rl, t); 923 tcg_temp_free_i32(t); 924 } else if (TCG_TARGET_REG_BITS == 32) { 925 TCGv_i32 t0 = tcg_temp_new_i32(); 926 TCGv_i32 t1 = tcg_temp_new_i32(); 927 TCGv_i32 t2 = tcg_temp_new_i32(); 928 TCGv_i32 t3 = tcg_temp_new_i32(); 929 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 930 /* Adjust for negative inputs. */ 931 tcg_gen_sari_i32(t2, arg1, 31); 932 tcg_gen_sari_i32(t3, arg2, 31); 933 tcg_gen_and_i32(t2, t2, arg2); 934 tcg_gen_and_i32(t3, t3, arg1); 935 tcg_gen_sub_i32(rh, t1, t2); 936 tcg_gen_sub_i32(rh, rh, t3); 937 tcg_gen_mov_i32(rl, t0); 938 tcg_temp_free_i32(t0); 939 tcg_temp_free_i32(t1); 940 tcg_temp_free_i32(t2); 941 tcg_temp_free_i32(t3); 942 } else { 943 TCGv_i64 t0 = tcg_temp_new_i64(); 944 TCGv_i64 t1 = tcg_temp_new_i64(); 945 tcg_gen_ext_i32_i64(t0, arg1); 946 tcg_gen_ext_i32_i64(t1, arg2); 947 tcg_gen_mul_i64(t0, t0, t1); 948 tcg_gen_extr_i64_i32(rl, rh, t0); 949 tcg_temp_free_i64(t0); 950 tcg_temp_free_i64(t1); 951 } 952 } 953 954 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 955 { 956 if (TCG_TARGET_REG_BITS == 32) { 957 TCGv_i32 t0 = tcg_temp_new_i32(); 958 TCGv_i32 t1 = tcg_temp_new_i32(); 959 TCGv_i32 t2 = tcg_temp_new_i32(); 960 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 961 /* Adjust for negative input for the signed arg1. */ 962 tcg_gen_sari_i32(t2, arg1, 31); 963 tcg_gen_and_i32(t2, t2, arg2); 964 tcg_gen_sub_i32(rh, t1, t2); 965 tcg_gen_mov_i32(rl, t0); 966 tcg_temp_free_i32(t0); 967 tcg_temp_free_i32(t1); 968 tcg_temp_free_i32(t2); 969 } else { 970 TCGv_i64 t0 = tcg_temp_new_i64(); 971 TCGv_i64 t1 = tcg_temp_new_i64(); 972 tcg_gen_ext_i32_i64(t0, arg1); 973 tcg_gen_extu_i32_i64(t1, arg2); 974 tcg_gen_mul_i64(t0, t0, t1); 975 tcg_gen_extr_i64_i32(rl, rh, t0); 976 tcg_temp_free_i64(t0); 977 tcg_temp_free_i64(t1); 978 } 979 } 980 981 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 982 { 983 if (TCG_TARGET_HAS_ext8s_i32) { 984 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg); 985 } else { 986 tcg_gen_shli_i32(ret, arg, 24); 987 tcg_gen_sari_i32(ret, ret, 24); 988 } 989 } 990 991 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 992 { 993 if (TCG_TARGET_HAS_ext16s_i32) { 994 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg); 995 } else { 996 tcg_gen_shli_i32(ret, arg, 16); 997 tcg_gen_sari_i32(ret, ret, 16); 998 } 999 } 1000 1001 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 1002 { 1003 if (TCG_TARGET_HAS_ext8u_i32) { 1004 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg); 1005 } else { 1006 tcg_gen_andi_i32(ret, arg, 0xffu); 1007 } 1008 } 1009 1010 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 1011 { 1012 if (TCG_TARGET_HAS_ext16u_i32) { 1013 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg); 1014 } else { 1015 tcg_gen_andi_i32(ret, arg, 0xffffu); 1016 } 1017 } 1018 1019 /* Note: we assume the two high bytes are set to zero */ 1020 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg) 1021 { 1022 if (TCG_TARGET_HAS_bswap16_i32) { 1023 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg); 1024 } else { 1025 TCGv_i32 t0 = tcg_temp_new_i32(); 1026 1027 tcg_gen_ext8u_i32(t0, arg); 1028 tcg_gen_shli_i32(t0, t0, 8); 1029 tcg_gen_shri_i32(ret, arg, 8); 1030 tcg_gen_or_i32(ret, ret, t0); 1031 tcg_temp_free_i32(t0); 1032 } 1033 } 1034 1035 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1036 { 1037 if (TCG_TARGET_HAS_bswap32_i32) { 1038 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg); 1039 } else { 1040 TCGv_i32 t0 = tcg_temp_new_i32(); 1041 TCGv_i32 t1 = tcg_temp_new_i32(); 1042 TCGv_i32 t2 = tcg_const_i32(0x00ff00ff); 1043 1044 /* arg = abcd */ 1045 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */ 1046 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */ 1047 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */ 1048 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */ 1049 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */ 1050 1051 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */ 1052 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */ 1053 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */ 1054 1055 tcg_temp_free_i32(t0); 1056 tcg_temp_free_i32(t1); 1057 tcg_temp_free_i32(t2); 1058 } 1059 } 1060 1061 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1062 { 1063 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1064 } 1065 1066 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1067 { 1068 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1069 } 1070 1071 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1072 { 1073 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1074 } 1075 1076 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1077 { 1078 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1079 } 1080 1081 /* 64-bit ops */ 1082 1083 #if TCG_TARGET_REG_BITS == 32 1084 /* These are all inline for TCG_TARGET_REG_BITS == 64. */ 1085 1086 void tcg_gen_discard_i64(TCGv_i64 arg) 1087 { 1088 tcg_gen_discard_i32(TCGV_LOW(arg)); 1089 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1090 } 1091 1092 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1093 { 1094 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1095 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1096 } 1097 1098 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1099 { 1100 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1101 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1102 } 1103 1104 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1105 { 1106 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1107 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1108 } 1109 1110 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1111 { 1112 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1113 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1114 } 1115 1116 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1117 { 1118 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1119 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1120 } 1121 1122 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1123 { 1124 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1125 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1126 } 1127 1128 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1129 { 1130 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1131 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1132 } 1133 1134 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1135 { 1136 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1137 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1138 } 1139 1140 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1141 { 1142 /* Since arg2 and ret have different types, 1143 they cannot be the same temporary */ 1144 #ifdef HOST_WORDS_BIGENDIAN 1145 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1146 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1147 #else 1148 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1149 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1150 #endif 1151 } 1152 1153 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1154 { 1155 #ifdef HOST_WORDS_BIGENDIAN 1156 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1157 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1158 #else 1159 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1160 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1161 #endif 1162 } 1163 1164 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1165 { 1166 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1167 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1168 } 1169 1170 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1171 { 1172 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1173 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1174 } 1175 1176 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1177 { 1178 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1179 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1180 } 1181 1182 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1183 { 1184 gen_helper_shl_i64(ret, arg1, arg2); 1185 } 1186 1187 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1188 { 1189 gen_helper_shr_i64(ret, arg1, arg2); 1190 } 1191 1192 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1193 { 1194 gen_helper_sar_i64(ret, arg1, arg2); 1195 } 1196 1197 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1198 { 1199 TCGv_i64 t0; 1200 TCGv_i32 t1; 1201 1202 t0 = tcg_temp_new_i64(); 1203 t1 = tcg_temp_new_i32(); 1204 1205 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1206 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1207 1208 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1209 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1210 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1211 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1212 1213 tcg_gen_mov_i64(ret, t0); 1214 tcg_temp_free_i64(t0); 1215 tcg_temp_free_i32(t1); 1216 } 1217 #endif /* TCG_TARGET_REG_SIZE == 32 */ 1218 1219 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1220 { 1221 /* some cases can be optimized here */ 1222 if (arg2 == 0) { 1223 tcg_gen_mov_i64(ret, arg1); 1224 } else { 1225 TCGv_i64 t0 = tcg_const_i64(arg2); 1226 tcg_gen_add_i64(ret, arg1, t0); 1227 tcg_temp_free_i64(t0); 1228 } 1229 } 1230 1231 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1232 { 1233 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) { 1234 /* Don't recurse with tcg_gen_neg_i64. */ 1235 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2); 1236 } else { 1237 TCGv_i64 t0 = tcg_const_i64(arg1); 1238 tcg_gen_sub_i64(ret, t0, arg2); 1239 tcg_temp_free_i64(t0); 1240 } 1241 } 1242 1243 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1244 { 1245 /* some cases can be optimized here */ 1246 if (arg2 == 0) { 1247 tcg_gen_mov_i64(ret, arg1); 1248 } else { 1249 TCGv_i64 t0 = tcg_const_i64(arg2); 1250 tcg_gen_sub_i64(ret, arg1, t0); 1251 tcg_temp_free_i64(t0); 1252 } 1253 } 1254 1255 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1256 { 1257 TCGv_i64 t0; 1258 1259 if (TCG_TARGET_REG_BITS == 32) { 1260 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1261 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1262 return; 1263 } 1264 1265 /* Some cases can be optimized here. */ 1266 switch (arg2) { 1267 case 0: 1268 tcg_gen_movi_i64(ret, 0); 1269 return; 1270 case -1: 1271 tcg_gen_mov_i64(ret, arg1); 1272 return; 1273 case 0xff: 1274 /* Don't recurse with tcg_gen_ext8u_i64. */ 1275 if (TCG_TARGET_HAS_ext8u_i64) { 1276 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1); 1277 return; 1278 } 1279 break; 1280 case 0xffff: 1281 if (TCG_TARGET_HAS_ext16u_i64) { 1282 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1); 1283 return; 1284 } 1285 break; 1286 case 0xffffffffu: 1287 if (TCG_TARGET_HAS_ext32u_i64) { 1288 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1); 1289 return; 1290 } 1291 break; 1292 } 1293 t0 = tcg_const_i64(arg2); 1294 tcg_gen_and_i64(ret, arg1, t0); 1295 tcg_temp_free_i64(t0); 1296 } 1297 1298 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1299 { 1300 if (TCG_TARGET_REG_BITS == 32) { 1301 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1302 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1303 return; 1304 } 1305 /* Some cases can be optimized here. */ 1306 if (arg2 == -1) { 1307 tcg_gen_movi_i64(ret, -1); 1308 } else if (arg2 == 0) { 1309 tcg_gen_mov_i64(ret, arg1); 1310 } else { 1311 TCGv_i64 t0 = tcg_const_i64(arg2); 1312 tcg_gen_or_i64(ret, arg1, t0); 1313 tcg_temp_free_i64(t0); 1314 } 1315 } 1316 1317 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1318 { 1319 if (TCG_TARGET_REG_BITS == 32) { 1320 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1321 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1322 return; 1323 } 1324 /* Some cases can be optimized here. */ 1325 if (arg2 == 0) { 1326 tcg_gen_mov_i64(ret, arg1); 1327 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) { 1328 /* Don't recurse with tcg_gen_not_i64. */ 1329 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1); 1330 } else { 1331 TCGv_i64 t0 = tcg_const_i64(arg2); 1332 tcg_gen_xor_i64(ret, arg1, t0); 1333 tcg_temp_free_i64(t0); 1334 } 1335 } 1336 1337 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1338 unsigned c, bool right, bool arith) 1339 { 1340 tcg_debug_assert(c < 64); 1341 if (c == 0) { 1342 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1343 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1344 } else if (c >= 32) { 1345 c -= 32; 1346 if (right) { 1347 if (arith) { 1348 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1349 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1350 } else { 1351 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1352 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1353 } 1354 } else { 1355 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1356 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1357 } 1358 } else { 1359 TCGv_i32 t0, t1; 1360 1361 t0 = tcg_temp_new_i32(); 1362 t1 = tcg_temp_new_i32(); 1363 if (right) { 1364 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c); 1365 if (arith) { 1366 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c); 1367 } else { 1368 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c); 1369 } 1370 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1371 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0); 1372 tcg_gen_mov_i32(TCGV_HIGH(ret), t1); 1373 } else { 1374 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1375 /* Note: ret can be the same as arg1, so we use t1 */ 1376 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c); 1377 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1378 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0); 1379 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1380 } 1381 tcg_temp_free_i32(t0); 1382 tcg_temp_free_i32(t1); 1383 } 1384 } 1385 1386 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1387 { 1388 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1389 if (TCG_TARGET_REG_BITS == 32) { 1390 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1391 } else if (arg2 == 0) { 1392 tcg_gen_mov_i64(ret, arg1); 1393 } else { 1394 TCGv_i64 t0 = tcg_const_i64(arg2); 1395 tcg_gen_shl_i64(ret, arg1, t0); 1396 tcg_temp_free_i64(t0); 1397 } 1398 } 1399 1400 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1401 { 1402 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1403 if (TCG_TARGET_REG_BITS == 32) { 1404 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1405 } else if (arg2 == 0) { 1406 tcg_gen_mov_i64(ret, arg1); 1407 } else { 1408 TCGv_i64 t0 = tcg_const_i64(arg2); 1409 tcg_gen_shr_i64(ret, arg1, t0); 1410 tcg_temp_free_i64(t0); 1411 } 1412 } 1413 1414 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1415 { 1416 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1417 if (TCG_TARGET_REG_BITS == 32) { 1418 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1419 } else if (arg2 == 0) { 1420 tcg_gen_mov_i64(ret, arg1); 1421 } else { 1422 TCGv_i64 t0 = tcg_const_i64(arg2); 1423 tcg_gen_sar_i64(ret, arg1, t0); 1424 tcg_temp_free_i64(t0); 1425 } 1426 } 1427 1428 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1429 { 1430 if (cond == TCG_COND_ALWAYS) { 1431 tcg_gen_br(l); 1432 } else if (cond != TCG_COND_NEVER) { 1433 l->refs++; 1434 if (TCG_TARGET_REG_BITS == 32) { 1435 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1436 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1437 TCGV_HIGH(arg2), cond, label_arg(l)); 1438 } else { 1439 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1440 label_arg(l)); 1441 } 1442 } 1443 } 1444 1445 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1446 { 1447 if (cond == TCG_COND_ALWAYS) { 1448 tcg_gen_br(l); 1449 } else if (cond != TCG_COND_NEVER) { 1450 TCGv_i64 t0 = tcg_const_i64(arg2); 1451 tcg_gen_brcond_i64(cond, arg1, t0, l); 1452 tcg_temp_free_i64(t0); 1453 } 1454 } 1455 1456 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1457 TCGv_i64 arg1, TCGv_i64 arg2) 1458 { 1459 if (cond == TCG_COND_ALWAYS) { 1460 tcg_gen_movi_i64(ret, 1); 1461 } else if (cond == TCG_COND_NEVER) { 1462 tcg_gen_movi_i64(ret, 0); 1463 } else { 1464 if (TCG_TARGET_REG_BITS == 32) { 1465 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1466 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1467 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1468 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1469 } else { 1470 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1471 } 1472 } 1473 } 1474 1475 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1476 TCGv_i64 arg1, int64_t arg2) 1477 { 1478 TCGv_i64 t0 = tcg_const_i64(arg2); 1479 tcg_gen_setcond_i64(cond, ret, arg1, t0); 1480 tcg_temp_free_i64(t0); 1481 } 1482 1483 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1484 { 1485 if (arg2 == 0) { 1486 tcg_gen_movi_i64(ret, 0); 1487 } else if (is_power_of_2(arg2)) { 1488 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 1489 } else { 1490 TCGv_i64 t0 = tcg_const_i64(arg2); 1491 tcg_gen_mul_i64(ret, arg1, t0); 1492 tcg_temp_free_i64(t0); 1493 } 1494 } 1495 1496 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1497 { 1498 if (TCG_TARGET_HAS_div_i64) { 1499 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2); 1500 } else if (TCG_TARGET_HAS_div2_i64) { 1501 TCGv_i64 t0 = tcg_temp_new_i64(); 1502 tcg_gen_sari_i64(t0, arg1, 63); 1503 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2); 1504 tcg_temp_free_i64(t0); 1505 } else { 1506 gen_helper_div_i64(ret, arg1, arg2); 1507 } 1508 } 1509 1510 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1511 { 1512 if (TCG_TARGET_HAS_rem_i64) { 1513 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2); 1514 } else if (TCG_TARGET_HAS_div_i64) { 1515 TCGv_i64 t0 = tcg_temp_new_i64(); 1516 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2); 1517 tcg_gen_mul_i64(t0, t0, arg2); 1518 tcg_gen_sub_i64(ret, arg1, t0); 1519 tcg_temp_free_i64(t0); 1520 } else if (TCG_TARGET_HAS_div2_i64) { 1521 TCGv_i64 t0 = tcg_temp_new_i64(); 1522 tcg_gen_sari_i64(t0, arg1, 63); 1523 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2); 1524 tcg_temp_free_i64(t0); 1525 } else { 1526 gen_helper_rem_i64(ret, arg1, arg2); 1527 } 1528 } 1529 1530 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1531 { 1532 if (TCG_TARGET_HAS_div_i64) { 1533 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2); 1534 } else if (TCG_TARGET_HAS_div2_i64) { 1535 TCGv_i64 t0 = tcg_temp_new_i64(); 1536 tcg_gen_movi_i64(t0, 0); 1537 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2); 1538 tcg_temp_free_i64(t0); 1539 } else { 1540 gen_helper_divu_i64(ret, arg1, arg2); 1541 } 1542 } 1543 1544 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1545 { 1546 if (TCG_TARGET_HAS_rem_i64) { 1547 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2); 1548 } else if (TCG_TARGET_HAS_div_i64) { 1549 TCGv_i64 t0 = tcg_temp_new_i64(); 1550 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2); 1551 tcg_gen_mul_i64(t0, t0, arg2); 1552 tcg_gen_sub_i64(ret, arg1, t0); 1553 tcg_temp_free_i64(t0); 1554 } else if (TCG_TARGET_HAS_div2_i64) { 1555 TCGv_i64 t0 = tcg_temp_new_i64(); 1556 tcg_gen_movi_i64(t0, 0); 1557 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2); 1558 tcg_temp_free_i64(t0); 1559 } else { 1560 gen_helper_remu_i64(ret, arg1, arg2); 1561 } 1562 } 1563 1564 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 1565 { 1566 if (TCG_TARGET_REG_BITS == 32) { 1567 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1568 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1569 } else if (TCG_TARGET_HAS_ext8s_i64) { 1570 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg); 1571 } else { 1572 tcg_gen_shli_i64(ret, arg, 56); 1573 tcg_gen_sari_i64(ret, ret, 56); 1574 } 1575 } 1576 1577 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 1578 { 1579 if (TCG_TARGET_REG_BITS == 32) { 1580 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1581 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1582 } else if (TCG_TARGET_HAS_ext16s_i64) { 1583 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg); 1584 } else { 1585 tcg_gen_shli_i64(ret, arg, 48); 1586 tcg_gen_sari_i64(ret, ret, 48); 1587 } 1588 } 1589 1590 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 1591 { 1592 if (TCG_TARGET_REG_BITS == 32) { 1593 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1594 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1595 } else if (TCG_TARGET_HAS_ext32s_i64) { 1596 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg); 1597 } else { 1598 tcg_gen_shli_i64(ret, arg, 32); 1599 tcg_gen_sari_i64(ret, ret, 32); 1600 } 1601 } 1602 1603 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 1604 { 1605 if (TCG_TARGET_REG_BITS == 32) { 1606 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1607 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1608 } else if (TCG_TARGET_HAS_ext8u_i64) { 1609 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg); 1610 } else { 1611 tcg_gen_andi_i64(ret, arg, 0xffu); 1612 } 1613 } 1614 1615 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 1616 { 1617 if (TCG_TARGET_REG_BITS == 32) { 1618 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1619 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1620 } else if (TCG_TARGET_HAS_ext16u_i64) { 1621 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg); 1622 } else { 1623 tcg_gen_andi_i64(ret, arg, 0xffffu); 1624 } 1625 } 1626 1627 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 1628 { 1629 if (TCG_TARGET_REG_BITS == 32) { 1630 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1631 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1632 } else if (TCG_TARGET_HAS_ext32u_i64) { 1633 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg); 1634 } else { 1635 tcg_gen_andi_i64(ret, arg, 0xffffffffu); 1636 } 1637 } 1638 1639 /* Note: we assume the six high bytes are set to zero */ 1640 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg) 1641 { 1642 if (TCG_TARGET_REG_BITS == 32) { 1643 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1644 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1645 } else if (TCG_TARGET_HAS_bswap16_i64) { 1646 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg); 1647 } else { 1648 TCGv_i64 t0 = tcg_temp_new_i64(); 1649 1650 tcg_gen_ext8u_i64(t0, arg); 1651 tcg_gen_shli_i64(t0, t0, 8); 1652 tcg_gen_shri_i64(ret, arg, 8); 1653 tcg_gen_or_i64(ret, ret, t0); 1654 tcg_temp_free_i64(t0); 1655 } 1656 } 1657 1658 /* Note: we assume the four high bytes are set to zero */ 1659 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg) 1660 { 1661 if (TCG_TARGET_REG_BITS == 32) { 1662 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1663 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1664 } else if (TCG_TARGET_HAS_bswap32_i64) { 1665 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg); 1666 } else { 1667 TCGv_i64 t0 = tcg_temp_new_i64(); 1668 TCGv_i64 t1 = tcg_temp_new_i64(); 1669 TCGv_i64 t2 = tcg_const_i64(0x00ff00ff); 1670 1671 /* arg = ....abcd */ 1672 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .....abc */ 1673 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */ 1674 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */ 1675 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */ 1676 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */ 1677 1678 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */ 1679 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */ 1680 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */ 1681 tcg_gen_or_i64(ret, t0, t1); /* ret = ....dcba */ 1682 1683 tcg_temp_free_i64(t0); 1684 tcg_temp_free_i64(t1); 1685 tcg_temp_free_i64(t2); 1686 } 1687 } 1688 1689 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 1690 { 1691 if (TCG_TARGET_REG_BITS == 32) { 1692 TCGv_i32 t0, t1; 1693 t0 = tcg_temp_new_i32(); 1694 t1 = tcg_temp_new_i32(); 1695 1696 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 1697 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 1698 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1699 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 1700 tcg_temp_free_i32(t0); 1701 tcg_temp_free_i32(t1); 1702 } else if (TCG_TARGET_HAS_bswap64_i64) { 1703 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg); 1704 } else { 1705 TCGv_i64 t0 = tcg_temp_new_i64(); 1706 TCGv_i64 t1 = tcg_temp_new_i64(); 1707 TCGv_i64 t2 = tcg_temp_new_i64(); 1708 1709 /* arg = abcdefgh */ 1710 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull); 1711 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */ 1712 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */ 1713 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */ 1714 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */ 1715 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */ 1716 1717 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull); 1718 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */ 1719 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */ 1720 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */ 1721 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */ 1722 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */ 1723 1724 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */ 1725 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */ 1726 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */ 1727 1728 tcg_temp_free_i64(t0); 1729 tcg_temp_free_i64(t1); 1730 tcg_temp_free_i64(t2); 1731 } 1732 } 1733 1734 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 1735 { 1736 if (TCG_TARGET_REG_BITS == 32) { 1737 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1738 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1739 } else if (TCG_TARGET_HAS_not_i64) { 1740 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg); 1741 } else { 1742 tcg_gen_xori_i64(ret, arg, -1); 1743 } 1744 } 1745 1746 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1747 { 1748 if (TCG_TARGET_REG_BITS == 32) { 1749 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1750 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1751 } else if (TCG_TARGET_HAS_andc_i64) { 1752 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2); 1753 } else { 1754 TCGv_i64 t0 = tcg_temp_new_i64(); 1755 tcg_gen_not_i64(t0, arg2); 1756 tcg_gen_and_i64(ret, arg1, t0); 1757 tcg_temp_free_i64(t0); 1758 } 1759 } 1760 1761 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1762 { 1763 if (TCG_TARGET_REG_BITS == 32) { 1764 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1765 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1766 } else if (TCG_TARGET_HAS_eqv_i64) { 1767 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2); 1768 } else { 1769 tcg_gen_xor_i64(ret, arg1, arg2); 1770 tcg_gen_not_i64(ret, ret); 1771 } 1772 } 1773 1774 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1775 { 1776 if (TCG_TARGET_REG_BITS == 32) { 1777 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1778 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1779 } else if (TCG_TARGET_HAS_nand_i64) { 1780 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2); 1781 } else { 1782 tcg_gen_and_i64(ret, arg1, arg2); 1783 tcg_gen_not_i64(ret, ret); 1784 } 1785 } 1786 1787 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1788 { 1789 if (TCG_TARGET_REG_BITS == 32) { 1790 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1791 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1792 } else if (TCG_TARGET_HAS_nor_i64) { 1793 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2); 1794 } else { 1795 tcg_gen_or_i64(ret, arg1, arg2); 1796 tcg_gen_not_i64(ret, ret); 1797 } 1798 } 1799 1800 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1801 { 1802 if (TCG_TARGET_REG_BITS == 32) { 1803 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1804 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1805 } else if (TCG_TARGET_HAS_orc_i64) { 1806 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2); 1807 } else { 1808 TCGv_i64 t0 = tcg_temp_new_i64(); 1809 tcg_gen_not_i64(t0, arg2); 1810 tcg_gen_or_i64(ret, arg1, t0); 1811 tcg_temp_free_i64(t0); 1812 } 1813 } 1814 1815 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1816 { 1817 if (TCG_TARGET_HAS_clz_i64) { 1818 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2); 1819 } else { 1820 gen_helper_clz_i64(ret, arg1, arg2); 1821 } 1822 } 1823 1824 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1825 { 1826 if (TCG_TARGET_REG_BITS == 32 1827 && TCG_TARGET_HAS_clz_i32 1828 && arg2 <= 0xffffffffu) { 1829 TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32); 1830 tcg_gen_clz_i32(t, TCGV_LOW(arg1), t); 1831 tcg_gen_addi_i32(t, t, 32); 1832 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 1833 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1834 tcg_temp_free_i32(t); 1835 } else { 1836 TCGv_i64 t = tcg_const_i64(arg2); 1837 tcg_gen_clz_i64(ret, arg1, t); 1838 tcg_temp_free_i64(t); 1839 } 1840 } 1841 1842 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1843 { 1844 if (TCG_TARGET_HAS_ctz_i64) { 1845 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); 1846 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) { 1847 TCGv_i64 z, t = tcg_temp_new_i64(); 1848 1849 if (TCG_TARGET_HAS_ctpop_i64) { 1850 tcg_gen_subi_i64(t, arg1, 1); 1851 tcg_gen_andc_i64(t, t, arg1); 1852 tcg_gen_ctpop_i64(t, t); 1853 } else { 1854 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */ 1855 tcg_gen_neg_i64(t, arg1); 1856 tcg_gen_and_i64(t, t, arg1); 1857 tcg_gen_clzi_i64(t, t, 64); 1858 tcg_gen_xori_i64(t, t, 63); 1859 } 1860 z = tcg_const_i64(0); 1861 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 1862 tcg_temp_free_i64(t); 1863 tcg_temp_free_i64(z); 1864 } else { 1865 gen_helper_ctz_i64(ret, arg1, arg2); 1866 } 1867 } 1868 1869 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1870 { 1871 if (TCG_TARGET_REG_BITS == 32 1872 && TCG_TARGET_HAS_ctz_i32 1873 && arg2 <= 0xffffffffu) { 1874 TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32); 1875 tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32); 1876 tcg_gen_addi_i32(t32, t32, 32); 1877 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 1878 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1879 tcg_temp_free_i32(t32); 1880 } else if (!TCG_TARGET_HAS_ctz_i64 1881 && TCG_TARGET_HAS_ctpop_i64 1882 && arg2 == 64) { 1883 /* This equivalence has the advantage of not requiring a fixup. */ 1884 TCGv_i64 t = tcg_temp_new_i64(); 1885 tcg_gen_subi_i64(t, arg1, 1); 1886 tcg_gen_andc_i64(t, t, arg1); 1887 tcg_gen_ctpop_i64(ret, t); 1888 tcg_temp_free_i64(t); 1889 } else { 1890 TCGv_i64 t64 = tcg_const_i64(arg2); 1891 tcg_gen_ctz_i64(ret, arg1, t64); 1892 tcg_temp_free_i64(t64); 1893 } 1894 } 1895 1896 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 1897 { 1898 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) { 1899 TCGv_i64 t = tcg_temp_new_i64(); 1900 tcg_gen_sari_i64(t, arg, 63); 1901 tcg_gen_xor_i64(t, t, arg); 1902 tcg_gen_clzi_i64(t, t, 64); 1903 tcg_gen_subi_i64(ret, t, 1); 1904 tcg_temp_free_i64(t); 1905 } else { 1906 gen_helper_clrsb_i64(ret, arg); 1907 } 1908 } 1909 1910 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 1911 { 1912 if (TCG_TARGET_HAS_ctpop_i64) { 1913 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 1914 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 1915 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1916 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1917 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 1918 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1919 } else { 1920 gen_helper_ctpop_i64(ret, arg1); 1921 } 1922 } 1923 1924 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1925 { 1926 if (TCG_TARGET_HAS_rot_i64) { 1927 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2); 1928 } else { 1929 TCGv_i64 t0, t1; 1930 t0 = tcg_temp_new_i64(); 1931 t1 = tcg_temp_new_i64(); 1932 tcg_gen_shl_i64(t0, arg1, arg2); 1933 tcg_gen_subfi_i64(t1, 64, arg2); 1934 tcg_gen_shr_i64(t1, arg1, t1); 1935 tcg_gen_or_i64(ret, t0, t1); 1936 tcg_temp_free_i64(t0); 1937 tcg_temp_free_i64(t1); 1938 } 1939 } 1940 1941 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1942 { 1943 tcg_debug_assert(arg2 < 64); 1944 /* some cases can be optimized here */ 1945 if (arg2 == 0) { 1946 tcg_gen_mov_i64(ret, arg1); 1947 } else if (TCG_TARGET_HAS_rot_i64) { 1948 TCGv_i64 t0 = tcg_const_i64(arg2); 1949 tcg_gen_rotl_i64(ret, arg1, t0); 1950 tcg_temp_free_i64(t0); 1951 } else { 1952 TCGv_i64 t0, t1; 1953 t0 = tcg_temp_new_i64(); 1954 t1 = tcg_temp_new_i64(); 1955 tcg_gen_shli_i64(t0, arg1, arg2); 1956 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 1957 tcg_gen_or_i64(ret, t0, t1); 1958 tcg_temp_free_i64(t0); 1959 tcg_temp_free_i64(t1); 1960 } 1961 } 1962 1963 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1964 { 1965 if (TCG_TARGET_HAS_rot_i64) { 1966 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2); 1967 } else { 1968 TCGv_i64 t0, t1; 1969 t0 = tcg_temp_new_i64(); 1970 t1 = tcg_temp_new_i64(); 1971 tcg_gen_shr_i64(t0, arg1, arg2); 1972 tcg_gen_subfi_i64(t1, 64, arg2); 1973 tcg_gen_shl_i64(t1, arg1, t1); 1974 tcg_gen_or_i64(ret, t0, t1); 1975 tcg_temp_free_i64(t0); 1976 tcg_temp_free_i64(t1); 1977 } 1978 } 1979 1980 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1981 { 1982 tcg_debug_assert(arg2 < 64); 1983 /* some cases can be optimized here */ 1984 if (arg2 == 0) { 1985 tcg_gen_mov_i64(ret, arg1); 1986 } else { 1987 tcg_gen_rotli_i64(ret, arg1, 64 - arg2); 1988 } 1989 } 1990 1991 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 1992 unsigned int ofs, unsigned int len) 1993 { 1994 uint64_t mask; 1995 TCGv_i64 t1; 1996 1997 tcg_debug_assert(ofs < 64); 1998 tcg_debug_assert(len > 0); 1999 tcg_debug_assert(len <= 64); 2000 tcg_debug_assert(ofs + len <= 64); 2001 2002 if (len == 64) { 2003 tcg_gen_mov_i64(ret, arg2); 2004 return; 2005 } 2006 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2007 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 2008 return; 2009 } 2010 2011 if (TCG_TARGET_REG_BITS == 32) { 2012 if (ofs >= 32) { 2013 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 2014 TCGV_LOW(arg2), ofs - 32, len); 2015 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2016 return; 2017 } 2018 if (ofs + len <= 32) { 2019 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 2020 TCGV_LOW(arg2), ofs, len); 2021 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2022 return; 2023 } 2024 } 2025 2026 mask = (1ull << len) - 1; 2027 t1 = tcg_temp_new_i64(); 2028 2029 if (ofs + len < 64) { 2030 tcg_gen_andi_i64(t1, arg2, mask); 2031 tcg_gen_shli_i64(t1, t1, ofs); 2032 } else { 2033 tcg_gen_shli_i64(t1, arg2, ofs); 2034 } 2035 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2036 tcg_gen_or_i64(ret, ret, t1); 2037 2038 tcg_temp_free_i64(t1); 2039 } 2040 2041 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2042 unsigned int ofs, unsigned int len) 2043 { 2044 tcg_debug_assert(ofs < 64); 2045 tcg_debug_assert(len > 0); 2046 tcg_debug_assert(len <= 64); 2047 tcg_debug_assert(ofs + len <= 64); 2048 2049 if (ofs + len == 64) { 2050 tcg_gen_shli_i64(ret, arg, ofs); 2051 } else if (ofs == 0) { 2052 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2053 } else if (TCG_TARGET_HAS_deposit_i64 2054 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2055 TCGv_i64 zero = tcg_const_i64(0); 2056 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2057 tcg_temp_free_i64(zero); 2058 } else { 2059 if (TCG_TARGET_REG_BITS == 32) { 2060 if (ofs >= 32) { 2061 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2062 ofs - 32, len); 2063 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2064 return; 2065 } 2066 if (ofs + len <= 32) { 2067 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2068 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2069 return; 2070 } 2071 } 2072 /* To help two-operand hosts we prefer to zero-extend first, 2073 which allows ARG to stay live. */ 2074 switch (len) { 2075 case 32: 2076 if (TCG_TARGET_HAS_ext32u_i64) { 2077 tcg_gen_ext32u_i64(ret, arg); 2078 tcg_gen_shli_i64(ret, ret, ofs); 2079 return; 2080 } 2081 break; 2082 case 16: 2083 if (TCG_TARGET_HAS_ext16u_i64) { 2084 tcg_gen_ext16u_i64(ret, arg); 2085 tcg_gen_shli_i64(ret, ret, ofs); 2086 return; 2087 } 2088 break; 2089 case 8: 2090 if (TCG_TARGET_HAS_ext8u_i64) { 2091 tcg_gen_ext8u_i64(ret, arg); 2092 tcg_gen_shli_i64(ret, ret, ofs); 2093 return; 2094 } 2095 break; 2096 } 2097 /* Otherwise prefer zero-extension over AND for code size. */ 2098 switch (ofs + len) { 2099 case 32: 2100 if (TCG_TARGET_HAS_ext32u_i64) { 2101 tcg_gen_shli_i64(ret, arg, ofs); 2102 tcg_gen_ext32u_i64(ret, ret); 2103 return; 2104 } 2105 break; 2106 case 16: 2107 if (TCG_TARGET_HAS_ext16u_i64) { 2108 tcg_gen_shli_i64(ret, arg, ofs); 2109 tcg_gen_ext16u_i64(ret, ret); 2110 return; 2111 } 2112 break; 2113 case 8: 2114 if (TCG_TARGET_HAS_ext8u_i64) { 2115 tcg_gen_shli_i64(ret, arg, ofs); 2116 tcg_gen_ext8u_i64(ret, ret); 2117 return; 2118 } 2119 break; 2120 } 2121 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2122 tcg_gen_shli_i64(ret, ret, ofs); 2123 } 2124 } 2125 2126 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2127 unsigned int ofs, unsigned int len) 2128 { 2129 tcg_debug_assert(ofs < 64); 2130 tcg_debug_assert(len > 0); 2131 tcg_debug_assert(len <= 64); 2132 tcg_debug_assert(ofs + len <= 64); 2133 2134 /* Canonicalize certain special cases, even if extract is supported. */ 2135 if (ofs + len == 64) { 2136 tcg_gen_shri_i64(ret, arg, 64 - len); 2137 return; 2138 } 2139 if (ofs == 0) { 2140 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2141 return; 2142 } 2143 2144 if (TCG_TARGET_REG_BITS == 32) { 2145 /* Look for a 32-bit extract within one of the two words. */ 2146 if (ofs >= 32) { 2147 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2148 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2149 return; 2150 } 2151 if (ofs + len <= 32) { 2152 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2153 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2154 return; 2155 } 2156 /* The field is split across two words. One double-word 2157 shift is better than two double-word shifts. */ 2158 goto do_shift_and; 2159 } 2160 2161 if (TCG_TARGET_HAS_extract_i64 2162 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2163 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2164 return; 2165 } 2166 2167 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2168 switch (ofs + len) { 2169 case 32: 2170 if (TCG_TARGET_HAS_ext32u_i64) { 2171 tcg_gen_ext32u_i64(ret, arg); 2172 tcg_gen_shri_i64(ret, ret, ofs); 2173 return; 2174 } 2175 break; 2176 case 16: 2177 if (TCG_TARGET_HAS_ext16u_i64) { 2178 tcg_gen_ext16u_i64(ret, arg); 2179 tcg_gen_shri_i64(ret, ret, ofs); 2180 return; 2181 } 2182 break; 2183 case 8: 2184 if (TCG_TARGET_HAS_ext8u_i64) { 2185 tcg_gen_ext8u_i64(ret, arg); 2186 tcg_gen_shri_i64(ret, ret, ofs); 2187 return; 2188 } 2189 break; 2190 } 2191 2192 /* ??? Ideally we'd know what values are available for immediate AND. 2193 Assume that 8 bits are available, plus the special cases of 16 and 32, 2194 so that we get ext8u, ext16u, and ext32u. */ 2195 switch (len) { 2196 case 1 ... 8: case 16: case 32: 2197 do_shift_and: 2198 tcg_gen_shri_i64(ret, arg, ofs); 2199 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2200 break; 2201 default: 2202 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2203 tcg_gen_shri_i64(ret, ret, 64 - len); 2204 break; 2205 } 2206 } 2207 2208 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2209 unsigned int ofs, unsigned int len) 2210 { 2211 tcg_debug_assert(ofs < 64); 2212 tcg_debug_assert(len > 0); 2213 tcg_debug_assert(len <= 64); 2214 tcg_debug_assert(ofs + len <= 64); 2215 2216 /* Canonicalize certain special cases, even if sextract is supported. */ 2217 if (ofs + len == 64) { 2218 tcg_gen_sari_i64(ret, arg, 64 - len); 2219 return; 2220 } 2221 if (ofs == 0) { 2222 switch (len) { 2223 case 32: 2224 tcg_gen_ext32s_i64(ret, arg); 2225 return; 2226 case 16: 2227 tcg_gen_ext16s_i64(ret, arg); 2228 return; 2229 case 8: 2230 tcg_gen_ext8s_i64(ret, arg); 2231 return; 2232 } 2233 } 2234 2235 if (TCG_TARGET_REG_BITS == 32) { 2236 /* Look for a 32-bit extract within one of the two words. */ 2237 if (ofs >= 32) { 2238 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2239 } else if (ofs + len <= 32) { 2240 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2241 } else if (ofs == 0) { 2242 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2243 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2244 return; 2245 } else if (len > 32) { 2246 TCGv_i32 t = tcg_temp_new_i32(); 2247 /* Extract the bits for the high word normally. */ 2248 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2249 /* Shift the field down for the low part. */ 2250 tcg_gen_shri_i64(ret, arg, ofs); 2251 /* Overwrite the shift into the high part. */ 2252 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2253 tcg_temp_free_i32(t); 2254 return; 2255 } else { 2256 /* Shift the field down for the low part, such that the 2257 field sits at the MSB. */ 2258 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2259 /* Shift the field down from the MSB, sign extending. */ 2260 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2261 } 2262 /* Sign-extend the field from 32 bits. */ 2263 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2264 return; 2265 } 2266 2267 if (TCG_TARGET_HAS_sextract_i64 2268 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2269 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2270 return; 2271 } 2272 2273 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2274 switch (ofs + len) { 2275 case 32: 2276 if (TCG_TARGET_HAS_ext32s_i64) { 2277 tcg_gen_ext32s_i64(ret, arg); 2278 tcg_gen_sari_i64(ret, ret, ofs); 2279 return; 2280 } 2281 break; 2282 case 16: 2283 if (TCG_TARGET_HAS_ext16s_i64) { 2284 tcg_gen_ext16s_i64(ret, arg); 2285 tcg_gen_sari_i64(ret, ret, ofs); 2286 return; 2287 } 2288 break; 2289 case 8: 2290 if (TCG_TARGET_HAS_ext8s_i64) { 2291 tcg_gen_ext8s_i64(ret, arg); 2292 tcg_gen_sari_i64(ret, ret, ofs); 2293 return; 2294 } 2295 break; 2296 } 2297 switch (len) { 2298 case 32: 2299 if (TCG_TARGET_HAS_ext32s_i64) { 2300 tcg_gen_shri_i64(ret, arg, ofs); 2301 tcg_gen_ext32s_i64(ret, ret); 2302 return; 2303 } 2304 break; 2305 case 16: 2306 if (TCG_TARGET_HAS_ext16s_i64) { 2307 tcg_gen_shri_i64(ret, arg, ofs); 2308 tcg_gen_ext16s_i64(ret, ret); 2309 return; 2310 } 2311 break; 2312 case 8: 2313 if (TCG_TARGET_HAS_ext8s_i64) { 2314 tcg_gen_shri_i64(ret, arg, ofs); 2315 tcg_gen_ext8s_i64(ret, ret); 2316 return; 2317 } 2318 break; 2319 } 2320 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2321 tcg_gen_sari_i64(ret, ret, 64 - len); 2322 } 2323 2324 /* 2325 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs. 2326 * Unlike tcg_gen_extract_i64 above, len is fixed at 64. 2327 */ 2328 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah, 2329 unsigned int ofs) 2330 { 2331 tcg_debug_assert(ofs <= 64); 2332 if (ofs == 0) { 2333 tcg_gen_mov_i64(ret, al); 2334 } else if (ofs == 64) { 2335 tcg_gen_mov_i64(ret, ah); 2336 } else if (al == ah) { 2337 tcg_gen_rotri_i64(ret, al, ofs); 2338 } else if (TCG_TARGET_HAS_extract2_i64) { 2339 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs); 2340 } else { 2341 TCGv_i64 t0 = tcg_temp_new_i64(); 2342 tcg_gen_shri_i64(t0, al, ofs); 2343 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs); 2344 tcg_temp_free_i64(t0); 2345 } 2346 } 2347 2348 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2349 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2350 { 2351 if (cond == TCG_COND_ALWAYS) { 2352 tcg_gen_mov_i64(ret, v1); 2353 } else if (cond == TCG_COND_NEVER) { 2354 tcg_gen_mov_i64(ret, v2); 2355 } else if (TCG_TARGET_REG_BITS == 32) { 2356 TCGv_i32 t0 = tcg_temp_new_i32(); 2357 TCGv_i32 t1 = tcg_temp_new_i32(); 2358 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2359 TCGV_LOW(c1), TCGV_HIGH(c1), 2360 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2361 2362 if (TCG_TARGET_HAS_movcond_i32) { 2363 tcg_gen_movi_i32(t1, 0); 2364 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1, 2365 TCGV_LOW(v1), TCGV_LOW(v2)); 2366 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1, 2367 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2368 } else { 2369 tcg_gen_neg_i32(t0, t0); 2370 2371 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0); 2372 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0); 2373 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1); 2374 2375 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0); 2376 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0); 2377 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1); 2378 } 2379 tcg_temp_free_i32(t0); 2380 tcg_temp_free_i32(t1); 2381 } else if (TCG_TARGET_HAS_movcond_i64) { 2382 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2383 } else { 2384 TCGv_i64 t0 = tcg_temp_new_i64(); 2385 TCGv_i64 t1 = tcg_temp_new_i64(); 2386 tcg_gen_setcond_i64(cond, t0, c1, c2); 2387 tcg_gen_neg_i64(t0, t0); 2388 tcg_gen_and_i64(t1, v1, t0); 2389 tcg_gen_andc_i64(ret, v2, t0); 2390 tcg_gen_or_i64(ret, ret, t1); 2391 tcg_temp_free_i64(t0); 2392 tcg_temp_free_i64(t1); 2393 } 2394 } 2395 2396 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2397 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2398 { 2399 if (TCG_TARGET_HAS_add2_i64) { 2400 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2401 } else { 2402 TCGv_i64 t0 = tcg_temp_new_i64(); 2403 TCGv_i64 t1 = tcg_temp_new_i64(); 2404 tcg_gen_add_i64(t0, al, bl); 2405 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2406 tcg_gen_add_i64(rh, ah, bh); 2407 tcg_gen_add_i64(rh, rh, t1); 2408 tcg_gen_mov_i64(rl, t0); 2409 tcg_temp_free_i64(t0); 2410 tcg_temp_free_i64(t1); 2411 } 2412 } 2413 2414 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2415 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2416 { 2417 if (TCG_TARGET_HAS_sub2_i64) { 2418 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2419 } else { 2420 TCGv_i64 t0 = tcg_temp_new_i64(); 2421 TCGv_i64 t1 = tcg_temp_new_i64(); 2422 tcg_gen_sub_i64(t0, al, bl); 2423 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2424 tcg_gen_sub_i64(rh, ah, bh); 2425 tcg_gen_sub_i64(rh, rh, t1); 2426 tcg_gen_mov_i64(rl, t0); 2427 tcg_temp_free_i64(t0); 2428 tcg_temp_free_i64(t1); 2429 } 2430 } 2431 2432 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2433 { 2434 if (TCG_TARGET_HAS_mulu2_i64) { 2435 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2436 } else if (TCG_TARGET_HAS_muluh_i64) { 2437 TCGv_i64 t = tcg_temp_new_i64(); 2438 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2439 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2); 2440 tcg_gen_mov_i64(rl, t); 2441 tcg_temp_free_i64(t); 2442 } else { 2443 TCGv_i64 t0 = tcg_temp_new_i64(); 2444 tcg_gen_mul_i64(t0, arg1, arg2); 2445 gen_helper_muluh_i64(rh, arg1, arg2); 2446 tcg_gen_mov_i64(rl, t0); 2447 tcg_temp_free_i64(t0); 2448 } 2449 } 2450 2451 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2452 { 2453 if (TCG_TARGET_HAS_muls2_i64) { 2454 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 2455 } else if (TCG_TARGET_HAS_mulsh_i64) { 2456 TCGv_i64 t = tcg_temp_new_i64(); 2457 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2458 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2); 2459 tcg_gen_mov_i64(rl, t); 2460 tcg_temp_free_i64(t); 2461 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) { 2462 TCGv_i64 t0 = tcg_temp_new_i64(); 2463 TCGv_i64 t1 = tcg_temp_new_i64(); 2464 TCGv_i64 t2 = tcg_temp_new_i64(); 2465 TCGv_i64 t3 = tcg_temp_new_i64(); 2466 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2467 /* Adjust for negative inputs. */ 2468 tcg_gen_sari_i64(t2, arg1, 63); 2469 tcg_gen_sari_i64(t3, arg2, 63); 2470 tcg_gen_and_i64(t2, t2, arg2); 2471 tcg_gen_and_i64(t3, t3, arg1); 2472 tcg_gen_sub_i64(rh, t1, t2); 2473 tcg_gen_sub_i64(rh, rh, t3); 2474 tcg_gen_mov_i64(rl, t0); 2475 tcg_temp_free_i64(t0); 2476 tcg_temp_free_i64(t1); 2477 tcg_temp_free_i64(t2); 2478 tcg_temp_free_i64(t3); 2479 } else { 2480 TCGv_i64 t0 = tcg_temp_new_i64(); 2481 tcg_gen_mul_i64(t0, arg1, arg2); 2482 gen_helper_mulsh_i64(rh, arg1, arg2); 2483 tcg_gen_mov_i64(rl, t0); 2484 tcg_temp_free_i64(t0); 2485 } 2486 } 2487 2488 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2489 { 2490 TCGv_i64 t0 = tcg_temp_new_i64(); 2491 TCGv_i64 t1 = tcg_temp_new_i64(); 2492 TCGv_i64 t2 = tcg_temp_new_i64(); 2493 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2494 /* Adjust for negative input for the signed arg1. */ 2495 tcg_gen_sari_i64(t2, arg1, 63); 2496 tcg_gen_and_i64(t2, t2, arg2); 2497 tcg_gen_sub_i64(rh, t1, t2); 2498 tcg_gen_mov_i64(rl, t0); 2499 tcg_temp_free_i64(t0); 2500 tcg_temp_free_i64(t1); 2501 tcg_temp_free_i64(t2); 2502 } 2503 2504 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2505 { 2506 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 2507 } 2508 2509 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2510 { 2511 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 2512 } 2513 2514 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2515 { 2516 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 2517 } 2518 2519 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2520 { 2521 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 2522 } 2523 2524 /* Size changing operations. */ 2525 2526 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2527 { 2528 if (TCG_TARGET_REG_BITS == 32) { 2529 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2530 } else if (TCG_TARGET_HAS_extrl_i64_i32) { 2531 tcg_gen_op2(INDEX_op_extrl_i64_i32, 2532 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2533 } else { 2534 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2535 } 2536 } 2537 2538 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2539 { 2540 if (TCG_TARGET_REG_BITS == 32) { 2541 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2542 } else if (TCG_TARGET_HAS_extrh_i64_i32) { 2543 tcg_gen_op2(INDEX_op_extrh_i64_i32, 2544 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2545 } else { 2546 TCGv_i64 t = tcg_temp_new_i64(); 2547 tcg_gen_shri_i64(t, arg, 32); 2548 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2549 tcg_temp_free_i64(t); 2550 } 2551 } 2552 2553 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2554 { 2555 if (TCG_TARGET_REG_BITS == 32) { 2556 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2557 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2558 } else { 2559 tcg_gen_op2(INDEX_op_extu_i32_i64, 2560 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2561 } 2562 } 2563 2564 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2565 { 2566 if (TCG_TARGET_REG_BITS == 32) { 2567 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2568 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2569 } else { 2570 tcg_gen_op2(INDEX_op_ext_i32_i64, 2571 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2572 } 2573 } 2574 2575 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 2576 { 2577 TCGv_i64 tmp; 2578 2579 if (TCG_TARGET_REG_BITS == 32) { 2580 tcg_gen_mov_i32(TCGV_LOW(dest), low); 2581 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 2582 return; 2583 } 2584 2585 tmp = tcg_temp_new_i64(); 2586 /* These extensions are only needed for type correctness. 2587 We may be able to do better given target specific information. */ 2588 tcg_gen_extu_i32_i64(tmp, high); 2589 tcg_gen_extu_i32_i64(dest, low); 2590 /* If deposit is available, use it. Otherwise use the extra 2591 knowledge that we have of the zero-extensions above. */ 2592 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) { 2593 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 2594 } else { 2595 tcg_gen_shli_i64(tmp, tmp, 32); 2596 tcg_gen_or_i64(dest, dest, tmp); 2597 } 2598 tcg_temp_free_i64(tmp); 2599 } 2600 2601 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 2602 { 2603 if (TCG_TARGET_REG_BITS == 32) { 2604 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 2605 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 2606 } else { 2607 tcg_gen_extrl_i64_i32(lo, arg); 2608 tcg_gen_extrh_i64_i32(hi, arg); 2609 } 2610 } 2611 2612 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 2613 { 2614 tcg_gen_ext32u_i64(lo, arg); 2615 tcg_gen_shri_i64(hi, arg, 32); 2616 } 2617 2618 /* QEMU specific operations. */ 2619 2620 void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx) 2621 { 2622 uintptr_t val = (uintptr_t)tb + idx; 2623 2624 if (tb == NULL) { 2625 tcg_debug_assert(idx == 0); 2626 } else if (idx <= TB_EXIT_IDXMAX) { 2627 #ifdef CONFIG_DEBUG_TCG 2628 /* This is an exit following a goto_tb. Verify that we have 2629 seen this numbered exit before, via tcg_gen_goto_tb. */ 2630 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); 2631 #endif 2632 /* When not chaining, exit without indicating a link. */ 2633 if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2634 val = 0; 2635 } 2636 } else { 2637 /* This is an exit via the exitreq label. */ 2638 tcg_debug_assert(idx == TB_EXIT_REQUESTED); 2639 } 2640 2641 tcg_gen_op1i(INDEX_op_exit_tb, val); 2642 } 2643 2644 void tcg_gen_goto_tb(unsigned idx) 2645 { 2646 /* We only support two chained exits. */ 2647 tcg_debug_assert(idx <= TB_EXIT_IDXMAX); 2648 #ifdef CONFIG_DEBUG_TCG 2649 /* Verify that we havn't seen this numbered exit before. */ 2650 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 2651 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 2652 #endif 2653 /* When not chaining, we simply fall through to the "fallback" exit. */ 2654 if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2655 tcg_gen_op1i(INDEX_op_goto_tb, idx); 2656 } 2657 } 2658 2659 void tcg_gen_lookup_and_goto_ptr(void) 2660 { 2661 if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2662 TCGv_ptr ptr = tcg_temp_new_ptr(); 2663 gen_helper_lookup_tb_ptr(ptr, cpu_env); 2664 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr)); 2665 tcg_temp_free_ptr(ptr); 2666 } else { 2667 tcg_gen_exit_tb(NULL, 0); 2668 } 2669 } 2670 2671 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st) 2672 { 2673 /* Trigger the asserts within as early as possible. */ 2674 (void)get_alignment_bits(op); 2675 2676 switch (op & MO_SIZE) { 2677 case MO_8: 2678 op &= ~MO_BSWAP; 2679 break; 2680 case MO_16: 2681 break; 2682 case MO_32: 2683 if (!is64) { 2684 op &= ~MO_SIGN; 2685 } 2686 break; 2687 case MO_64: 2688 if (!is64) { 2689 tcg_abort(); 2690 } 2691 break; 2692 } 2693 if (st) { 2694 op &= ~MO_SIGN; 2695 } 2696 return op; 2697 } 2698 2699 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr, 2700 TCGMemOp memop, TCGArg idx) 2701 { 2702 TCGMemOpIdx oi = make_memop_idx(memop, idx); 2703 #if TARGET_LONG_BITS == 32 2704 tcg_gen_op3i_i32(opc, val, addr, oi); 2705 #else 2706 if (TCG_TARGET_REG_BITS == 32) { 2707 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2708 } else { 2709 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi); 2710 } 2711 #endif 2712 } 2713 2714 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr, 2715 TCGMemOp memop, TCGArg idx) 2716 { 2717 TCGMemOpIdx oi = make_memop_idx(memop, idx); 2718 #if TARGET_LONG_BITS == 32 2719 if (TCG_TARGET_REG_BITS == 32) { 2720 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi); 2721 } else { 2722 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi); 2723 } 2724 #else 2725 if (TCG_TARGET_REG_BITS == 32) { 2726 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), 2727 TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2728 } else { 2729 tcg_gen_op3i_i64(opc, val, addr, oi); 2730 } 2731 #endif 2732 } 2733 2734 static void tcg_gen_req_mo(TCGBar type) 2735 { 2736 #ifdef TCG_GUEST_DEFAULT_MO 2737 type &= TCG_GUEST_DEFAULT_MO; 2738 #endif 2739 type &= ~TCG_TARGET_DEFAULT_MO; 2740 if (type) { 2741 tcg_gen_mb(type | TCG_BAR_SC); 2742 } 2743 } 2744 2745 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2746 { 2747 TCGMemOp orig_memop; 2748 2749 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2750 memop = tcg_canonicalize_memop(memop, 0, 0); 2751 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2752 addr, trace_mem_get_info(memop, 0)); 2753 2754 orig_memop = memop; 2755 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2756 memop &= ~MO_BSWAP; 2757 /* The bswap primitive requires zero-extended input. */ 2758 if ((memop & MO_SSIZE) == MO_SW) { 2759 memop &= ~MO_SIGN; 2760 } 2761 } 2762 2763 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx); 2764 2765 if ((orig_memop ^ memop) & MO_BSWAP) { 2766 switch (orig_memop & MO_SIZE) { 2767 case MO_16: 2768 tcg_gen_bswap16_i32(val, val); 2769 if (orig_memop & MO_SIGN) { 2770 tcg_gen_ext16s_i32(val, val); 2771 } 2772 break; 2773 case MO_32: 2774 tcg_gen_bswap32_i32(val, val); 2775 break; 2776 default: 2777 g_assert_not_reached(); 2778 } 2779 } 2780 } 2781 2782 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2783 { 2784 TCGv_i32 swap = NULL; 2785 2786 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2787 memop = tcg_canonicalize_memop(memop, 0, 1); 2788 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2789 addr, trace_mem_get_info(memop, 1)); 2790 2791 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2792 swap = tcg_temp_new_i32(); 2793 switch (memop & MO_SIZE) { 2794 case MO_16: 2795 tcg_gen_ext16u_i32(swap, val); 2796 tcg_gen_bswap16_i32(swap, swap); 2797 break; 2798 case MO_32: 2799 tcg_gen_bswap32_i32(swap, val); 2800 break; 2801 default: 2802 g_assert_not_reached(); 2803 } 2804 val = swap; 2805 memop &= ~MO_BSWAP; 2806 } 2807 2808 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); 2809 2810 if (swap) { 2811 tcg_temp_free_i32(swap); 2812 } 2813 } 2814 2815 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2816 { 2817 TCGMemOp orig_memop; 2818 2819 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 2820 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); 2821 if (memop & MO_SIGN) { 2822 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 2823 } else { 2824 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 2825 } 2826 return; 2827 } 2828 2829 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2830 memop = tcg_canonicalize_memop(memop, 1, 0); 2831 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2832 addr, trace_mem_get_info(memop, 0)); 2833 2834 orig_memop = memop; 2835 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2836 memop &= ~MO_BSWAP; 2837 /* The bswap primitive requires zero-extended input. */ 2838 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) { 2839 memop &= ~MO_SIGN; 2840 } 2841 } 2842 2843 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx); 2844 2845 if ((orig_memop ^ memop) & MO_BSWAP) { 2846 switch (orig_memop & MO_SIZE) { 2847 case MO_16: 2848 tcg_gen_bswap16_i64(val, val); 2849 if (orig_memop & MO_SIGN) { 2850 tcg_gen_ext16s_i64(val, val); 2851 } 2852 break; 2853 case MO_32: 2854 tcg_gen_bswap32_i64(val, val); 2855 if (orig_memop & MO_SIGN) { 2856 tcg_gen_ext32s_i64(val, val); 2857 } 2858 break; 2859 case MO_64: 2860 tcg_gen_bswap64_i64(val, val); 2861 break; 2862 default: 2863 g_assert_not_reached(); 2864 } 2865 } 2866 } 2867 2868 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2869 { 2870 TCGv_i64 swap = NULL; 2871 2872 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 2873 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); 2874 return; 2875 } 2876 2877 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2878 memop = tcg_canonicalize_memop(memop, 1, 1); 2879 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2880 addr, trace_mem_get_info(memop, 1)); 2881 2882 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2883 swap = tcg_temp_new_i64(); 2884 switch (memop & MO_SIZE) { 2885 case MO_16: 2886 tcg_gen_ext16u_i64(swap, val); 2887 tcg_gen_bswap16_i64(swap, swap); 2888 break; 2889 case MO_32: 2890 tcg_gen_ext32u_i64(swap, val); 2891 tcg_gen_bswap32_i64(swap, swap); 2892 break; 2893 case MO_64: 2894 tcg_gen_bswap64_i64(swap, val); 2895 break; 2896 default: 2897 g_assert_not_reached(); 2898 } 2899 val = swap; 2900 memop &= ~MO_BSWAP; 2901 } 2902 2903 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx); 2904 2905 if (swap) { 2906 tcg_temp_free_i64(swap); 2907 } 2908 } 2909 2910 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc) 2911 { 2912 switch (opc & MO_SSIZE) { 2913 case MO_SB: 2914 tcg_gen_ext8s_i32(ret, val); 2915 break; 2916 case MO_UB: 2917 tcg_gen_ext8u_i32(ret, val); 2918 break; 2919 case MO_SW: 2920 tcg_gen_ext16s_i32(ret, val); 2921 break; 2922 case MO_UW: 2923 tcg_gen_ext16u_i32(ret, val); 2924 break; 2925 default: 2926 tcg_gen_mov_i32(ret, val); 2927 break; 2928 } 2929 } 2930 2931 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc) 2932 { 2933 switch (opc & MO_SSIZE) { 2934 case MO_SB: 2935 tcg_gen_ext8s_i64(ret, val); 2936 break; 2937 case MO_UB: 2938 tcg_gen_ext8u_i64(ret, val); 2939 break; 2940 case MO_SW: 2941 tcg_gen_ext16s_i64(ret, val); 2942 break; 2943 case MO_UW: 2944 tcg_gen_ext16u_i64(ret, val); 2945 break; 2946 case MO_SL: 2947 tcg_gen_ext32s_i64(ret, val); 2948 break; 2949 case MO_UL: 2950 tcg_gen_ext32u_i64(ret, val); 2951 break; 2952 default: 2953 tcg_gen_mov_i64(ret, val); 2954 break; 2955 } 2956 } 2957 2958 #ifdef CONFIG_SOFTMMU 2959 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, 2960 TCGv_i32, TCGv_i32, TCGv_i32); 2961 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, 2962 TCGv_i64, TCGv_i64, TCGv_i32); 2963 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, 2964 TCGv_i32, TCGv_i32); 2965 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, 2966 TCGv_i64, TCGv_i32); 2967 #else 2968 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32); 2969 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64); 2970 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32); 2971 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64); 2972 #endif 2973 2974 #ifdef CONFIG_ATOMIC64 2975 # define WITH_ATOMIC64(X) X, 2976 #else 2977 # define WITH_ATOMIC64(X) 2978 #endif 2979 2980 static void * const table_cmpxchg[16] = { 2981 [MO_8] = gen_helper_atomic_cmpxchgb, 2982 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, 2983 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, 2984 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, 2985 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, 2986 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) 2987 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) 2988 }; 2989 2990 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, 2991 TCGv_i32 newv, TCGArg idx, TCGMemOp memop) 2992 { 2993 memop = tcg_canonicalize_memop(memop, 0, 0); 2994 2995 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { 2996 TCGv_i32 t1 = tcg_temp_new_i32(); 2997 TCGv_i32 t2 = tcg_temp_new_i32(); 2998 2999 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); 3000 3001 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 3002 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); 3003 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 3004 tcg_temp_free_i32(t2); 3005 3006 if (memop & MO_SIGN) { 3007 tcg_gen_ext_i32(retv, t1, memop); 3008 } else { 3009 tcg_gen_mov_i32(retv, t1); 3010 } 3011 tcg_temp_free_i32(t1); 3012 } else { 3013 gen_atomic_cx_i32 gen; 3014 3015 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 3016 tcg_debug_assert(gen != NULL); 3017 3018 #ifdef CONFIG_SOFTMMU 3019 { 3020 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 3021 gen(retv, cpu_env, addr, cmpv, newv, oi); 3022 tcg_temp_free_i32(oi); 3023 } 3024 #else 3025 gen(retv, cpu_env, addr, cmpv, newv); 3026 #endif 3027 3028 if (memop & MO_SIGN) { 3029 tcg_gen_ext_i32(retv, retv, memop); 3030 } 3031 } 3032 } 3033 3034 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, 3035 TCGv_i64 newv, TCGArg idx, TCGMemOp memop) 3036 { 3037 memop = tcg_canonicalize_memop(memop, 1, 0); 3038 3039 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { 3040 TCGv_i64 t1 = tcg_temp_new_i64(); 3041 TCGv_i64 t2 = tcg_temp_new_i64(); 3042 3043 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); 3044 3045 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 3046 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); 3047 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 3048 tcg_temp_free_i64(t2); 3049 3050 if (memop & MO_SIGN) { 3051 tcg_gen_ext_i64(retv, t1, memop); 3052 } else { 3053 tcg_gen_mov_i64(retv, t1); 3054 } 3055 tcg_temp_free_i64(t1); 3056 } else if ((memop & MO_SIZE) == MO_64) { 3057 #ifdef CONFIG_ATOMIC64 3058 gen_atomic_cx_i64 gen; 3059 3060 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 3061 tcg_debug_assert(gen != NULL); 3062 3063 #ifdef CONFIG_SOFTMMU 3064 { 3065 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx)); 3066 gen(retv, cpu_env, addr, cmpv, newv, oi); 3067 tcg_temp_free_i32(oi); 3068 } 3069 #else 3070 gen(retv, cpu_env, addr, cmpv, newv); 3071 #endif 3072 #else 3073 gen_helper_exit_atomic(cpu_env); 3074 /* Produce a result, so that we have a well-formed opcode stream 3075 with respect to uses of the result in the (dead) code following. */ 3076 tcg_gen_movi_i64(retv, 0); 3077 #endif /* CONFIG_ATOMIC64 */ 3078 } else { 3079 TCGv_i32 c32 = tcg_temp_new_i32(); 3080 TCGv_i32 n32 = tcg_temp_new_i32(); 3081 TCGv_i32 r32 = tcg_temp_new_i32(); 3082 3083 tcg_gen_extrl_i64_i32(c32, cmpv); 3084 tcg_gen_extrl_i64_i32(n32, newv); 3085 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN); 3086 tcg_temp_free_i32(c32); 3087 tcg_temp_free_i32(n32); 3088 3089 tcg_gen_extu_i32_i64(retv, r32); 3090 tcg_temp_free_i32(r32); 3091 3092 if (memop & MO_SIGN) { 3093 tcg_gen_ext_i64(retv, retv, memop); 3094 } 3095 } 3096 } 3097 3098 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 3099 TCGArg idx, TCGMemOp memop, bool new_val, 3100 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) 3101 { 3102 TCGv_i32 t1 = tcg_temp_new_i32(); 3103 TCGv_i32 t2 = tcg_temp_new_i32(); 3104 3105 memop = tcg_canonicalize_memop(memop, 0, 0); 3106 3107 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 3108 gen(t2, t1, val); 3109 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 3110 3111 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); 3112 tcg_temp_free_i32(t1); 3113 tcg_temp_free_i32(t2); 3114 } 3115 3116 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 3117 TCGArg idx, TCGMemOp memop, void * const table[]) 3118 { 3119 gen_atomic_op_i32 gen; 3120 3121 memop = tcg_canonicalize_memop(memop, 0, 0); 3122 3123 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 3124 tcg_debug_assert(gen != NULL); 3125 3126 #ifdef CONFIG_SOFTMMU 3127 { 3128 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 3129 gen(ret, cpu_env, addr, val, oi); 3130 tcg_temp_free_i32(oi); 3131 } 3132 #else 3133 gen(ret, cpu_env, addr, val); 3134 #endif 3135 3136 if (memop & MO_SIGN) { 3137 tcg_gen_ext_i32(ret, ret, memop); 3138 } 3139 } 3140 3141 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 3142 TCGArg idx, TCGMemOp memop, bool new_val, 3143 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) 3144 { 3145 TCGv_i64 t1 = tcg_temp_new_i64(); 3146 TCGv_i64 t2 = tcg_temp_new_i64(); 3147 3148 memop = tcg_canonicalize_memop(memop, 1, 0); 3149 3150 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 3151 gen(t2, t1, val); 3152 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 3153 3154 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); 3155 tcg_temp_free_i64(t1); 3156 tcg_temp_free_i64(t2); 3157 } 3158 3159 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 3160 TCGArg idx, TCGMemOp memop, void * const table[]) 3161 { 3162 memop = tcg_canonicalize_memop(memop, 1, 0); 3163 3164 if ((memop & MO_SIZE) == MO_64) { 3165 #ifdef CONFIG_ATOMIC64 3166 gen_atomic_op_i64 gen; 3167 3168 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 3169 tcg_debug_assert(gen != NULL); 3170 3171 #ifdef CONFIG_SOFTMMU 3172 { 3173 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 3174 gen(ret, cpu_env, addr, val, oi); 3175 tcg_temp_free_i32(oi); 3176 } 3177 #else 3178 gen(ret, cpu_env, addr, val); 3179 #endif 3180 #else 3181 gen_helper_exit_atomic(cpu_env); 3182 /* Produce a result, so that we have a well-formed opcode stream 3183 with respect to uses of the result in the (dead) code following. */ 3184 tcg_gen_movi_i64(ret, 0); 3185 #endif /* CONFIG_ATOMIC64 */ 3186 } else { 3187 TCGv_i32 v32 = tcg_temp_new_i32(); 3188 TCGv_i32 r32 = tcg_temp_new_i32(); 3189 3190 tcg_gen_extrl_i64_i32(v32, val); 3191 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); 3192 tcg_temp_free_i32(v32); 3193 3194 tcg_gen_extu_i32_i64(ret, r32); 3195 tcg_temp_free_i32(r32); 3196 3197 if (memop & MO_SIGN) { 3198 tcg_gen_ext_i64(ret, ret, memop); 3199 } 3200 } 3201 } 3202 3203 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ 3204 static void * const table_##NAME[16] = { \ 3205 [MO_8] = gen_helper_atomic_##NAME##b, \ 3206 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ 3207 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ 3208 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ 3209 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ 3210 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ 3211 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ 3212 }; \ 3213 void tcg_gen_atomic_##NAME##_i32 \ 3214 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \ 3215 { \ 3216 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ 3217 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ 3218 } else { \ 3219 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ 3220 tcg_gen_##OP##_i32); \ 3221 } \ 3222 } \ 3223 void tcg_gen_atomic_##NAME##_i64 \ 3224 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \ 3225 { \ 3226 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ 3227 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ 3228 } else { \ 3229 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ 3230 tcg_gen_##OP##_i64); \ 3231 } \ 3232 } 3233 3234 GEN_ATOMIC_HELPER(fetch_add, add, 0) 3235 GEN_ATOMIC_HELPER(fetch_and, and, 0) 3236 GEN_ATOMIC_HELPER(fetch_or, or, 0) 3237 GEN_ATOMIC_HELPER(fetch_xor, xor, 0) 3238 GEN_ATOMIC_HELPER(fetch_smin, smin, 0) 3239 GEN_ATOMIC_HELPER(fetch_umin, umin, 0) 3240 GEN_ATOMIC_HELPER(fetch_smax, smax, 0) 3241 GEN_ATOMIC_HELPER(fetch_umax, umax, 0) 3242 3243 GEN_ATOMIC_HELPER(add_fetch, add, 1) 3244 GEN_ATOMIC_HELPER(and_fetch, and, 1) 3245 GEN_ATOMIC_HELPER(or_fetch, or, 1) 3246 GEN_ATOMIC_HELPER(xor_fetch, xor, 1) 3247 GEN_ATOMIC_HELPER(smin_fetch, smin, 1) 3248 GEN_ATOMIC_HELPER(umin_fetch, umin, 1) 3249 GEN_ATOMIC_HELPER(smax_fetch, smax, 1) 3250 GEN_ATOMIC_HELPER(umax_fetch, umax, 1) 3251 3252 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) 3253 { 3254 tcg_gen_mov_i32(r, b); 3255 } 3256 3257 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) 3258 { 3259 tcg_gen_mov_i64(r, b); 3260 } 3261 3262 GEN_ATOMIC_HELPER(xchg, mov2, 0) 3263 3264 #undef GEN_ATOMIC_HELPER 3265